From 49d683c34ce0ffed07bfebaaf49dc4a49350d5fc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Sep 2019 17:58:23 +0300 Subject: [PATCH 0001/2007] Auto version update to [19.14.2.1] [54425] --- dbms/cmake/version.cmake | 8 ++++---- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 332fa51ab26..e13c8f66311 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) -set(VERSION_PATCH 1) -set(VERSION_GITHASH adfc36917222bdb03eba069f0cad0f4f5b8f1c94) -set(VERSION_DESCRIBE v19.14.1.1-prestable) -set(VERSION_STRING 19.14.1.1) +set(VERSION_PATCH 2) +set(VERSION_GITHASH 6f1a8c37abe6ee4e7ee74c0b5cb9c05a87417b61) +set(VERSION_DESCRIBE v19.14.2.1-prestable) +set(VERSION_STRING 19.14.2.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index f1db1b81185..bab2c62d8df 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.13.1.1) unstable; urgency=low +clickhouse (19.14.2.1) unstable; urgency=low * Modified source code - -- clickhouse-release Tue, 23 Jul 2019 11:20:49 +0300 + -- clickhouse-release Fri, 06 Sep 2019 17:58:20 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 68cdf3f0204..cb28c075540 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.13.1.* +ARG version=19.14.2.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 934c1921a67..53c336d82c0 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.13.1.* +ARG version=19.14.2.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 5c2bd25b48c..9f26a36b913 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.13.1.* +ARG version=19.14.2.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From 9ed197c24f176a2ed820768d8e4cb8162d2fb234 Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Mon, 9 Sep 2019 19:59:51 +0300 Subject: [PATCH 0002/2007] Store offsets manually for each message (#6872) (cherry picked from commit 6c32fc3fc11a27a09b174f1d7b4dd8550b79e918) --- dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp | 7 ++++--- dbms/src/Storages/Kafka/StorageKafka.cpp | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 4614e581a3c..823eb632b7f 100644 --- a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -72,9 +72,7 @@ void ReadBufferFromKafkaConsumer::commit() PrintOffsets("Polled offset", consumer->get_offsets_position(consumer->get_assignment())); - /// Since we can poll more messages than we already processed - commit only processed messages. - if (!messages.empty()) - consumer->async_commit(*std::prev(current)); + consumer->async_commit(); PrintOffsets("Committed offset", consumer->get_offsets_committed(consumer->get_assignment())); @@ -186,6 +184,9 @@ bool ReadBufferFromKafkaConsumer::nextImpl() auto new_position = reinterpret_cast(const_cast(current->get_payload().get_data())); BufferBase::set(new_position, current->get_payload().get_size(), 0); + /// Since we can poll more messages than we already processed - commit only processed messages. + consumer->store_offset(*current); + ++current; return true; diff --git a/dbms/src/Storages/Kafka/StorageKafka.cpp b/dbms/src/Storages/Kafka/StorageKafka.cpp index 2b41fa9e772..ed067993a18 100644 --- a/dbms/src/Storages/Kafka/StorageKafka.cpp +++ b/dbms/src/Storages/Kafka/StorageKafka.cpp @@ -261,9 +261,10 @@ ConsumerBufferPtr StorageKafka::createReadBuffer() conf.set("metadata.broker.list", brokers); conf.set("group.id", group); conf.set("client.id", VERSION_FULL); - conf.set("auto.offset.reset", "smallest"); // If no offset stored for this group, read all messages from the start - conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished - conf.set("enable.partition.eof", "false"); // Ignore EOF messages + conf.set("auto.offset.reset", "smallest"); // If no offset stored for this group, read all messages from the start + conf.set("enable.auto.commit", "false"); // We manually commit offsets after a stream successfully finished + conf.set("enable.auto.offset.store", "false"); // Update offset automatically - to commit them all at once. + conf.set("enable.partition.eof", "false"); // Ignore EOF messages updateConfiguration(conf); // Create a consumer and subscribe to topics From cf97326931e79f7ff7af7b4c37f6bc0e6c12d3ff Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Sep 2019 13:32:19 +0300 Subject: [PATCH 0003/2007] Auto version update to [19.14.2.2] [54425] --- dbms/cmake/version.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index e13c8f66311..b99fd1eac6a 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -3,9 +3,9 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) set(VERSION_PATCH 2) -set(VERSION_GITHASH 6f1a8c37abe6ee4e7ee74c0b5cb9c05a87417b61) -set(VERSION_DESCRIBE v19.14.2.1-prestable) -set(VERSION_STRING 19.14.2.1) +set(VERSION_GITHASH 9ed197c24f176a2ed820768d8e4cb8162d2fb234) +set(VERSION_DESCRIBE v19.14.2.2-prestable) +set(VERSION_STRING 19.14.2.2) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") From 94788f7702248692d796aaf08b38376707aae74f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Sep 2019 13:32:37 +0300 Subject: [PATCH 0004/2007] Auto version update to [19.14.3.1] [54425] --- dbms/cmake/version.cmake | 6 +++--- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index b99fd1eac6a..d0c543f25c7 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) -set(VERSION_PATCH 2) +set(VERSION_PATCH 3) set(VERSION_GITHASH 9ed197c24f176a2ed820768d8e4cb8162d2fb234) -set(VERSION_DESCRIBE v19.14.2.2-prestable) -set(VERSION_STRING 19.14.2.2) +set(VERSION_DESCRIBE v19.14.3.1-prestable) +set(VERSION_STRING 19.14.3.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index bab2c62d8df..9ac29181056 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.14.2.1) unstable; urgency=low +clickhouse (19.14.3.1) unstable; urgency=low * Modified source code - -- clickhouse-release Fri, 06 Sep 2019 17:58:20 +0300 + -- clickhouse-release Tue, 10 Sep 2019 13:32:34 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index cb28c075540..30b09d7645e 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.2.* +ARG version=19.14.3.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 53c336d82c0..5a2abec6245 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.2.* +ARG version=19.14.3.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 9f26a36b913..16e23213084 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.2.* +ARG version=19.14.3.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From f38d2483f8f0e72f4d8cca0a2ad612de6318b941 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Sep 2019 20:49:19 +0300 Subject: [PATCH 0005/2007] Auto version update to [19.14.3.3] [54425] --- dbms/cmake/version.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index d0c543f25c7..bef2500f0e0 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -3,9 +3,9 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) set(VERSION_PATCH 3) -set(VERSION_GITHASH 9ed197c24f176a2ed820768d8e4cb8162d2fb234) -set(VERSION_DESCRIBE v19.14.3.1-prestable) -set(VERSION_STRING 19.14.3.1) +set(VERSION_GITHASH 94788f7702248692d796aaf08b38376707aae74f) +set(VERSION_DESCRIBE v19.14.3.3-stable) +set(VERSION_STRING 19.14.3.3) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") From 7797dee7f1051de3fe2b8791ec944c7ebb22e9c4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Sep 2019 20:49:38 +0300 Subject: [PATCH 0006/2007] Auto version update to [19.14.4.1] [54425] --- dbms/cmake/version.cmake | 6 +++--- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index bef2500f0e0..15f6373ba3f 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) -set(VERSION_PATCH 3) +set(VERSION_PATCH 4) set(VERSION_GITHASH 94788f7702248692d796aaf08b38376707aae74f) -set(VERSION_DESCRIBE v19.14.3.3-stable) -set(VERSION_STRING 19.14.3.3) +set(VERSION_DESCRIBE v19.14.4.1-stable) +set(VERSION_STRING 19.14.4.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index 9ac29181056..55716635681 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.14.3.1) unstable; urgency=low +clickhouse (19.14.4.1) unstable; urgency=low * Modified source code - -- clickhouse-release Tue, 10 Sep 2019 13:32:34 +0300 + -- clickhouse-release Tue, 10 Sep 2019 20:49:34 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 30b09d7645e..04904134763 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.3.* +ARG version=19.14.4.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 5a2abec6245..62eb5f41e8e 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.3.* +ARG version=19.14.4.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 16e23213084..49799749fbd 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.3.* +ARG version=19.14.4.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From 9f15ebf3fdfc73f116055409036a762b5d1fb7df Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 12 Sep 2019 03:20:38 +0300 Subject: [PATCH 0007/2007] Merge pull request #6909 from abyss7/issue-6902 Put delimiter only after consuming a message (cherry picked from commit cb2f4ebfaff95fbb0fb55f332e1d38867f0224c5) --- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 3 +-- dbms/tests/integration/test_storage_kafka/test.py | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index 823eb632b7f..083b471d4f1 100644 --- a/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/dbms/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -150,8 +150,6 @@ bool ReadBufferFromKafkaConsumer::nextImpl() return true; } - put_delimiter = (delimiter != 0); - if (current == messages.end()) { if (intermediate_commit) @@ -183,6 +181,7 @@ bool ReadBufferFromKafkaConsumer::nextImpl() // XXX: very fishy place with const casting. auto new_position = reinterpret_cast(const_cast(current->get_payload().get_data())); BufferBase::set(new_position, current->get_payload().get_size(), 0); + put_delimiter = (delimiter != 0); /// Since we can poll more messages than we already processed - commit only processed messages. consumer->store_offset(*current); diff --git a/dbms/tests/integration/test_storage_kafka/test.py b/dbms/tests/integration/test_storage_kafka/test.py index a3aa290c98d..c629ac9f22e 100644 --- a/dbms/tests/integration/test_storage_kafka/test.py +++ b/dbms/tests/integration/test_storage_kafka/test.py @@ -248,6 +248,21 @@ def test_kafka_tsv_with_delimiter(kafka_cluster): kafka_check_result(result, True) +@pytest.mark.timeout(180) +def test_kafka_select_empty(kafka_cluster): + instance.query(''' + CREATE TABLE test.kafka (key UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = 'empty', + kafka_group_name = 'empty', + kafka_format = 'TSV', + kafka_row_delimiter = '\\n'; + ''') + + assert int(instance.query('SELECT count() FROM test.kafka')) == 0 + + @pytest.mark.timeout(180) def test_kafka_json_without_delimiter(kafka_cluster): instance.query(''' From da44d9fd9beaae81c9c98e1f976d306c23fb8eaf Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 7 Sep 2019 03:01:42 +0300 Subject: [PATCH 0008/2007] Merge pull request #6853 from yandex/pipe-capacity-very-old-kernels Returned support for very old Linux kernels (that lack of F_GETPIPE_SZ fcntl) (cherry picked from commit 46e7b4f925f4715cf73220c4935ade845b13fa2b) --- dbms/src/Common/TraceCollector.cpp | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/dbms/src/Common/TraceCollector.cpp b/dbms/src/Common/TraceCollector.cpp index 6ed2074e47d..9451c3f88e5 100644 --- a/dbms/src/Common/TraceCollector.cpp +++ b/dbms/src/Common/TraceCollector.cpp @@ -49,15 +49,26 @@ TraceCollector::TraceCollector(std::shared_ptr & trace_log_) #if !defined(__FreeBSD__) /** Increase pipe size to avoid slowdown during fine-grained trace collection. */ - constexpr int max_pipe_capacity_to_set = 1048576; int pipe_size = fcntl(trace_pipe.fds_rw[1], F_GETPIPE_SZ); if (-1 == pipe_size) - throwFromErrno("Cannot get pipe capacity", ErrorCodes::CANNOT_FCNTL); - for (errno = 0; errno != EPERM && pipe_size < max_pipe_capacity_to_set; pipe_size *= 2) - if (-1 == fcntl(trace_pipe.fds_rw[1], F_SETPIPE_SZ, pipe_size * 2) && errno != EPERM) - throwFromErrno("Cannot increase pipe capacity to " + toString(pipe_size * 2), ErrorCodes::CANNOT_FCNTL); + { + if (errno == EINVAL) + { + LOG_INFO(log, "Cannot get pipe capacity, " << errnoToString(ErrorCodes::CANNOT_FCNTL) << ". Very old Linux kernels have no support for this fcntl."); + /// It will work nevertheless. + } + else + throwFromErrno("Cannot get pipe capacity", ErrorCodes::CANNOT_FCNTL); + } + else + { + constexpr int max_pipe_capacity_to_set = 1048576; + for (errno = 0; errno != EPERM && pipe_size < max_pipe_capacity_to_set; pipe_size *= 2) + if (-1 == fcntl(trace_pipe.fds_rw[1], F_SETPIPE_SZ, pipe_size * 2) && errno != EPERM) + throwFromErrno("Cannot increase pipe capacity to " + toString(pipe_size * 2), ErrorCodes::CANNOT_FCNTL); - LOG_TRACE(log, "Pipe capacity is " << formatReadableSizeWithBinarySuffix(std::min(pipe_size, max_pipe_capacity_to_set))); + LOG_TRACE(log, "Pipe capacity is " << formatReadableSizeWithBinarySuffix(std::min(pipe_size, max_pipe_capacity_to_set))); + } #endif thread = ThreadFromGlobalPool(&TraceCollector::run, this); From 1602daba5937311a5d70a45dbae78435da9a8200 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 12 Sep 2019 00:36:59 +0300 Subject: [PATCH 0009/2007] Merge pull request #6911 from yandex/fix-insert-select-data-loss Fix insert select data loss (cherry picked from commit 6fbf9ca7ab88f02b39328f0c5cc83ee8c5cfdfd6) --- dbms/src/DataStreams/copyData.cpp | 2 -- dbms/src/Interpreters/executeQuery.cpp | 15 ++++++++++++++- .../01009_insert_select_data_loss.reference | 2 ++ .../0_stateless/01009_insert_select_data_loss.sql | 5 +++++ 4 files changed, 21 insertions(+), 3 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01009_insert_select_data_loss.reference create mode 100644 dbms/tests/queries/0_stateless/01009_insert_select_data_loss.sql diff --git a/dbms/src/DataStreams/copyData.cpp b/dbms/src/DataStreams/copyData.cpp index 5000c87be7c..9d17596fc8d 100644 --- a/dbms/src/DataStreams/copyData.cpp +++ b/dbms/src/DataStreams/copyData.cpp @@ -28,8 +28,6 @@ void copyDataImpl(IBlockInputStream & from, IBlockOutputStream & to, TCancelCall break; to.write(block); - if (!block.rows()) - to.flush(); progress(block); } diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 3793f2f79c9..85130437155 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -35,6 +35,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -631,7 +632,19 @@ void executeQuery( if (set_query_id) set_query_id(context.getClientInfo().current_query_id); - copyData(*streams.in, *out); + if (ast->as()) + { + /// For Watch query, flush data if block is empty (to send data to client). + auto flush_callback = [&out](const Block & block) + { + if (block.rows() == 0) + out->flush(); + }; + + copyData(*streams.in, *out, [](){ return false; }, std::move(flush_callback)); + } + else + copyData(*streams.in, *out); } if (pipeline.initialized()) diff --git a/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.reference b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.reference new file mode 100644 index 00000000000..25e7f55667e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.reference @@ -0,0 +1,2 @@ +0 +10 diff --git a/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.sql b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.sql new file mode 100644 index 00000000000..9a754d94323 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01009_insert_select_data_loss.sql @@ -0,0 +1,5 @@ +drop table if exists tab; +create table tab (x UInt64) engine = MergeTree order by tuple(); + +insert into tab select number as n from numbers(20) any inner join (select number * 10 as n from numbers(2)) using(n) settings any_join_distinct_right_table_keys = 1, max_block_size = 5; +select * from tab order by x; From 242bdeda5ff371b46e93105ae6460d0d6a8783f2 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 6 Sep 2019 20:15:14 +0300 Subject: [PATCH 0010/2007] Merge pull request #6837 from filimonov/fix_some_pvs_reported_issues Fix some pvs reported issues (cherry picked from commit c90dfc1e48f83de8880f2c2e73dea20ab9618598) --- dbms/src/Interpreters/ClusterProxy/executeQuery.cpp | 2 +- dbms/src/Interpreters/Context.cpp | 4 ++-- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 7 +++---- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index dc0d3ef27b1..989595b3647 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -30,7 +30,7 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin /// Set as unchanged to avoid sending to remote server. new_settings.max_concurrent_queries_for_user.changed = false; new_settings.max_memory_usage_for_user.changed = false; - new_settings.max_memory_usage_for_all_queries = false; + new_settings.max_memory_usage_for_all_queries.changed = false; Context new_context(context); new_context.setSettings(new_settings); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 53b652681e2..f0ec292f647 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -1222,8 +1222,8 @@ void Context::setCurrentQueryId(const String & query_id) } words; } random; - random.words.a = thread_local_rng(); - random.words.b = thread_local_rng(); + random.words.a = thread_local_rng(); //-V656 + random.words.b = thread_local_rng(); //-V656 /// Use protected constructor. struct qUUID : Poco::UUID diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 05ac99196a4..a795fac596d 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -418,8 +418,6 @@ QueryPipeline InterpreterSelectQuery::executeWithProcessors() Block InterpreterSelectQuery::getSampleBlockImpl() { - FilterInfoPtr filter_info; - /// Need to create sets before analyzeExpressions(). Otherwise some sets for index won't be created. query_analyzer->makeSetsForIndex(getSelectQuery().where()); query_analyzer->makeSetsForIndex(getSelectQuery().prewhere()); @@ -431,8 +429,9 @@ Block InterpreterSelectQuery::getSampleBlockImpl() options.to_stage, context, storage, - true, - filter_info); + true, // only_types + {} // filter_info + ); if (options.to_stage == QueryProcessingStage::Enum::FetchColumns) { From d341531cb1c3a507dc9d5a0fada2648cf38a595a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 11 Sep 2019 02:03:20 +0300 Subject: [PATCH 0011/2007] Merge pull request #6882 from clemrodriguez/issue-6825 Fix too early MySQL connection closure in MySQLBlockInputStreamream.cpp (cherry picked from commit a446ec5e61ff068327a963030f9a2e3b315b762d) --- dbms/src/Formats/MySQLBlockInputStream.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/src/Formats/MySQLBlockInputStream.cpp b/dbms/src/Formats/MySQLBlockInputStream.cpp index dcf609f62ce..4f8291ffebe 100644 --- a/dbms/src/Formats/MySQLBlockInputStream.cpp +++ b/dbms/src/Formats/MySQLBlockInputStream.cpp @@ -131,8 +131,6 @@ Block MySQLBlockInputStream::readImpl() row = result.fetch(); } - if (auto_close) - entry.disconnect(); return description.sample_block.cloneWithColumns(std::move(columns)); } From 60cc8fb29f15a58604eec0168ef2dd5bb531f9aa Mon Sep 17 00:00:00 2001 From: Yuriy Baranov Date: Mon, 9 Sep 2019 09:43:28 +0300 Subject: [PATCH 0012/2007] Merge pull request #6865 from yurriy/mysql Releasing resources after query execution in MySQL compatibility server (cherry picked from commit f6e1a26806baeb55cdd2033fb5429128cac1158c) --- dbms/programs/server/MySQLHandler.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/programs/server/MySQLHandler.cpp b/dbms/programs/server/MySQLHandler.cpp index 3f16496b5be..7e555220489 100644 --- a/dbms/programs/server/MySQLHandler.cpp +++ b/dbms/programs/server/MySQLHandler.cpp @@ -293,7 +293,8 @@ void MySQLHandler::comQuery(ReadBuffer & payload) should_replace = true; } - executeQuery(should_replace ? empty_select : payload, *out, true, connection_context, set_content_type, nullptr); + Context query_context = connection_context; + executeQuery(should_replace ? empty_select : payload, *out, true, query_context, set_content_type, nullptr); if (!with_output) packet_sender->sendPacket(OK_Packet(0x00, client_capability_flags, 0, 0, 0), true); From 5e106fe1a91981fbf6e6a231b3ef8b5478d192db Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 14 Sep 2019 01:43:49 +0300 Subject: [PATCH 0013/2007] Auto version update to [19.14.4.9] [54425] --- dbms/cmake/version.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 15f6373ba3f..aa4add51b2c 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -3,9 +3,9 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) set(VERSION_PATCH 4) -set(VERSION_GITHASH 94788f7702248692d796aaf08b38376707aae74f) -set(VERSION_DESCRIBE v19.14.4.1-stable) -set(VERSION_STRING 19.14.4.1) +set(VERSION_GITHASH 60cc8fb29f15a58604eec0168ef2dd5bb531f9aa) +set(VERSION_DESCRIBE v19.14.4.9-prestable) +set(VERSION_STRING 19.14.4.9) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") From c6464f2813c7d5cb939af11759d05705ba0f15ee Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 14 Sep 2019 01:44:07 +0300 Subject: [PATCH 0014/2007] Auto version update to [19.14.5.1] [54425] --- dbms/cmake/version.cmake | 6 +++--- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index aa4add51b2c..af3c12512a6 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) -set(VERSION_PATCH 4) +set(VERSION_PATCH 5) set(VERSION_GITHASH 60cc8fb29f15a58604eec0168ef2dd5bb531f9aa) -set(VERSION_DESCRIBE v19.14.4.9-prestable) -set(VERSION_STRING 19.14.4.9) +set(VERSION_DESCRIBE v19.14.5.1-prestable) +set(VERSION_STRING 19.14.5.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index 55716635681..f2fcc1240d4 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.14.4.1) unstable; urgency=low +clickhouse (19.14.5.1) unstable; urgency=low * Modified source code - -- clickhouse-release Tue, 10 Sep 2019 20:49:34 +0300 + -- clickhouse-release Sat, 14 Sep 2019 01:44:03 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 04904134763..872ba169315 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.4.* +ARG version=19.14.5.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 62eb5f41e8e..6ca1b69bb5e 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.4.* +ARG version=19.14.5.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 49799749fbd..cabad4e1c1f 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.4.* +ARG version=19.14.5.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From c5a328cbfa1ea124e1a59c3e76f8fc6f85a1fcbe Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 18 Sep 2019 17:53:35 +0300 Subject: [PATCH 0015/2007] Auto version update to [19.14.5.10] [54425] --- dbms/cmake/version.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index af3c12512a6..1663310e563 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -3,9 +3,9 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) set(VERSION_PATCH 5) -set(VERSION_GITHASH 60cc8fb29f15a58604eec0168ef2dd5bb531f9aa) -set(VERSION_DESCRIBE v19.14.5.1-prestable) -set(VERSION_STRING 19.14.5.1) +set(VERSION_GITHASH c6464f2813c7d5cb939af11759d05705ba0f15ee) +set(VERSION_DESCRIBE v19.14.5.10-prestable) +set(VERSION_STRING 19.14.5.10) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") From 33c50a7328456b7d1a1d8e262e5ce34f14fce960 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 18 Sep 2019 17:53:53 +0300 Subject: [PATCH 0016/2007] Auto version update to [19.14.6.1] [54425] --- dbms/cmake/version.cmake | 6 +++--- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 1663310e563..565f2157390 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) -set(VERSION_PATCH 5) +set(VERSION_PATCH 6) set(VERSION_GITHASH c6464f2813c7d5cb939af11759d05705ba0f15ee) -set(VERSION_DESCRIBE v19.14.5.10-prestable) -set(VERSION_STRING 19.14.5.10) +set(VERSION_DESCRIBE v19.14.6.1-prestable) +set(VERSION_STRING 19.14.6.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index f2fcc1240d4..4d6d8be0b76 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.14.5.1) unstable; urgency=low +clickhouse (19.14.6.1) unstable; urgency=low * Modified source code - -- clickhouse-release Sat, 14 Sep 2019 01:44:03 +0300 + -- clickhouse-release Wed, 18 Sep 2019 17:53:49 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 872ba169315..771e6405414 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.5.* +ARG version=19.14.6.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 6ca1b69bb5e..0337faa86ff 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.5.* +ARG version=19.14.6.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index cabad4e1c1f..62cc9c0b958 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.5.* +ARG version=19.14.6.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From 6beb8a716aa1db08f5992a7b5924a2d8834bb60d Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 19 Sep 2019 17:41:49 +0300 Subject: [PATCH 0017/2007] Fix system contributors generating script --- dbms/src/Storages/System/StorageSystemContributors.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/System/StorageSystemContributors.sh b/dbms/src/Storages/System/StorageSystemContributors.sh index aea122df0dc..c4c4eb5ad30 100755 --- a/dbms/src/Storages/System/StorageSystemContributors.sh +++ b/dbms/src/Storages/System/StorageSystemContributors.sh @@ -2,11 +2,15 @@ set -x +# doesn't actually cd to directory, but return absolute path CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# cd to directory +cd $CUR_DIR CONTRIBUTORS_FILE=${CONTRIBUTORS_FILE=$CUR_DIR/StorageSystemContributors.generated.cpp} -git shortlog --summary | perl -lnE 's/^\s+\d+\s+(.+)/ "$1",/; next unless $1; say $_' > $CONTRIBUTORS_FILE.tmp +# if you don't specify HEAD here, without terminal `git shortlog` would expect input from stdin +git shortlog HEAD --summary | perl -lnE 's/^\s+\d+\s+(.+)/ "$1",/; next unless $1; say $_' > $CONTRIBUTORS_FILE.tmp # If git history not available - dont make target file if [ ! -s $CONTRIBUTORS_FILE.tmp ]; then From b946d2770e48261a39a2c7dfa2971093cc38f929 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 19 Sep 2019 01:54:27 +0300 Subject: [PATCH 0018/2007] Merge pull request #6977 from Akazz/minor_fixes_in_tests-2 Reworked flapping test - 00715_fetch_merged_or_mutated_part_zookeeper (cherry picked from commit d7681d0239edadfbd50973220ede08ec217d3d44) --- ..._fetch_merged_or_mutated_part_zookeeper.sh | 52 +++++++++++++++++++ ...fetch_merged_or_mutated_part_zookeeper.sql | 42 --------------- 2 files changed, 52 insertions(+), 42 deletions(-) create mode 100755 dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh delete mode 100644 dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sql diff --git a/dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh b/dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh new file mode 100755 index 00000000000..a7cb79908ae --- /dev/null +++ b/dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh +. $CURDIR/mergetree_mutations.lib + + +${CLICKHOUSE_CLIENT} -n --query=" + DROP TABLE IF EXISTS fetches_r1; + DROP TABLE IF EXISTS fetches_r2" + +${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/fetches', 'r1') ORDER BY x" +${CLICKHOUSE_CLIENT} --query="CREATE TABLE fetches_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/fetches', 'r2') ORDER BY x \ + SETTINGS prefer_fetch_merged_part_time_threshold=0, \ + prefer_fetch_merged_part_size_threshold=0" + +${CLICKHOUSE_CLIENT} -n --query=" + INSERT INTO fetches_r1 VALUES (1); + INSERT INTO fetches_r1 VALUES (2); + INSERT INTO fetches_r1 VALUES (3)" + +${CLICKHOUSE_CLIENT} --query="SYSTEM SYNC REPLICA fetches_r2" +${CLICKHOUSE_CLIENT} --query="DETACH TABLE fetches_r2" + +${CLICKHOUSE_CLIENT} --query="OPTIMIZE TABLE fetches_r1 PARTITION tuple() FINAL" --replication_alter_partitions_sync=0 +${CLICKHOUSE_CLIENT} --query="SYSTEM SYNC REPLICA fetches_r1" + +# After attach replica r2 should fetch the merged part from r1. +${CLICKHOUSE_CLIENT} --query="ATTACH TABLE fetches_r2" +${CLICKHOUSE_CLIENT} --query="SYSTEM SYNC REPLICA fetches_r2" + +${CLICKHOUSE_CLIENT} --query="SELECT '*** Check data after fetch of merged part ***'" +${CLICKHOUSE_CLIENT} --query="SELECT _part, * FROM fetches_r2 ORDER BY x" + +${CLICKHOUSE_CLIENT} --query="DETACH TABLE fetches_r2" + +# Add mutation that doesn't change data. +${CLICKHOUSE_CLIENT} --query="ALTER TABLE fetches_r1 DELETE WHERE x = 0" --replication_alter_partitions_sync=0 + +wait_for_mutation "fetches_r1" "0000000000" +${CLICKHOUSE_CLIENT} --query="SYSTEM SYNC REPLICA fetches_r1" + +# After attach replica r2 should compare checksums for mutated part and clone the local part. +${CLICKHOUSE_CLIENT} --query="ATTACH TABLE fetches_r2" +${CLICKHOUSE_CLIENT} --query="SYSTEM SYNC REPLICA fetches_r2" + +${CLICKHOUSE_CLIENT} --query="SELECT '*** Check data after fetch/clone of mutated part ***'" +${CLICKHOUSE_CLIENT} --query="SELECT _part, * FROM fetches_r2 ORDER BY x" + +${CLICKHOUSE_CLIENT} -n --query=" + DROP TABLE fetches_r1; + DROP TABLE fetches_r2" diff --git a/dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sql b/dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sql deleted file mode 100644 index 9a1c1b77cae..00000000000 --- a/dbms/tests/queries/0_stateless/00715_fetch_merged_or_mutated_part_zookeeper.sql +++ /dev/null @@ -1,42 +0,0 @@ -DROP TABLE IF EXISTS fetches_r1; -DROP TABLE IF EXISTS fetches_r2; - -CREATE TABLE fetches_r1(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/fetches', 'r1') ORDER BY x; -CREATE TABLE fetches_r2(x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/test/fetches', 'r2') ORDER BY x - SETTINGS prefer_fetch_merged_part_time_threshold=0, - prefer_fetch_merged_part_size_threshold=0; - -INSERT INTO fetches_r1 VALUES (1); -INSERT INTO fetches_r1 VALUES (2); -INSERT INTO fetches_r1 VALUES (3); - -SYSTEM SYNC REPLICA fetches_r2; - -DETACH TABLE fetches_r2; - -SET replication_alter_partitions_sync=0; -OPTIMIZE TABLE fetches_r1 PARTITION tuple() FINAL; -SYSTEM SYNC REPLICA fetches_r1; - --- After attach replica r2 should fetch the merged part from r1. -ATTACH TABLE fetches_r2; -SYSTEM SYNC REPLICA fetches_r2; - -SELECT '*** Check data after fetch of merged part ***'; -SELECT _part, * FROM fetches_r2 ORDER BY x; - -DETACH TABLE fetches_r2; - --- Add mutation that doesn't change data. -ALTER TABLE fetches_r1 DELETE WHERE x = 0; -SYSTEM SYNC REPLICA fetches_r1; - --- After attach replica r2 should compare checksums for mutated part and clone the local part. -ATTACH TABLE fetches_r2; -SYSTEM SYNC REPLICA fetches_r2; - -SELECT '*** Check data after fetch/clone of mutated part ***'; -SELECT _part, * FROM fetches_r2 ORDER BY x; - -DROP TABLE fetches_r1; -DROP TABLE fetches_r2; From 1fd0ec6578c336cbc598be40f489b4e781afff12 Mon Sep 17 00:00:00 2001 From: akuzm <36882414+akuzm@users.noreply.github.com> Date: Tue, 17 Sep 2019 14:57:38 +0300 Subject: [PATCH 0019/2007] Merge pull request #6937 from amosbird/ubmemfix In hash tables, properly initialize zero key cell. (cherry picked from commit 75dc7d331e3c4db4ea6490fadb996bc02e175bd9) --- dbms/src/Common/HashTable/HashTable.h | 14 ++++++++++++-- .../01011_group_uniq_array_memsan.reference | 1 + .../0_stateless/01011_group_uniq_array_memsan.sql | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.reference create mode 100644 dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.sql diff --git a/dbms/src/Common/HashTable/HashTable.h b/dbms/src/Common/HashTable/HashTable.h index d29459a90d5..5c389af9828 100644 --- a/dbms/src/Common/HashTable/HashTable.h +++ b/dbms/src/Common/HashTable/HashTable.h @@ -224,8 +224,18 @@ private: public: bool hasZero() const { return has_zero; } - void setHasZero() { has_zero = true; } - void clearHasZero() { has_zero = false; } + + void setHasZero() + { + has_zero = true; + new (zeroValue()) Cell(); + } + + void clearHasZero() + { + has_zero = false; + zeroValue()->~Cell(); + } Cell * zeroValue() { return reinterpret_cast(&zero_value_storage); } const Cell * zeroValue() const { return reinterpret_cast(&zero_value_storage); } diff --git a/dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.reference b/dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.reference new file mode 100644 index 00000000000..b7c55c59479 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.reference @@ -0,0 +1 @@ +[[],[2]] diff --git a/dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.sql b/dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.sql new file mode 100644 index 00000000000..b8c16e48c42 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01011_group_uniq_array_memsan.sql @@ -0,0 +1 @@ +select groupUniqArray(v) from values('id int, v Array(int)', (1, [2]), (1, [])) group by id; From 1712ef0d5d79214e44afcaf1aa7a3fcdcec222c0 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 15 Sep 2019 09:46:10 +0300 Subject: [PATCH 0020/2007] Merge pull request #6934 from abyss7/CLICKHOUSE-4643 Name temporary external table with existing subquery alias (cherry picked from commit 831eebe4dc1721dea4dd1ba034c34727774d36e1) --- .../Interpreters/GlobalSubqueriesVisitor.h | 12 ++++++++---- .../01009_global_array_join_names.reference | 0 .../01009_global_array_join_names.sql | 19 +++++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01009_global_array_join_names.reference create mode 100644 dbms/tests/queries/0_stateless/01009_global_array_join_names.sql diff --git a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h index 926e6afd1c2..6c380162af4 100644 --- a/dbms/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/dbms/src/Interpreters/GlobalSubqueriesVisitor.h @@ -78,12 +78,16 @@ public: return; } - /// Generate the name for the external table. - String external_table_name = "_data" + toString(external_table_id); - while (external_tables.count(external_table_name)) + String external_table_name = subquery_or_table_name->tryGetAlias(); + if (external_table_name.empty()) { - ++external_table_id; + /// Generate the name for the external table. external_table_name = "_data" + toString(external_table_id); + while (external_tables.count(external_table_name)) + { + ++external_table_id; + external_table_name = "_data" + toString(external_table_id); + } } auto interpreter = interpretSubquery(subquery_or_table_name, context, subquery_depth, {}); diff --git a/dbms/tests/queries/0_stateless/01009_global_array_join_names.reference b/dbms/tests/queries/0_stateless/01009_global_array_join_names.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01009_global_array_join_names.sql b/dbms/tests/queries/0_stateless/01009_global_array_join_names.sql new file mode 100644 index 00000000000..f3cbd0cf44d --- /dev/null +++ b/dbms/tests/queries/0_stateless/01009_global_array_join_names.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS test1; +DROP TABLE IF EXISTS test2; + +CREATE TABLE test1 (a UInt8, b Array(DateTime)) ENGINE Memory; +CREATE TABLE test2 as test1 ENGINE Distributed(test_shard_localhost, currentDatabase(), test1); + +INSERT INTO test1 VALUES (1, [1, 2, 3]); + +SELECT 1 +FROM test2 AS test2 +ARRAY JOIN arrayFilter(t -> (t GLOBAL IN + ( + SELECT DISTINCT now() AS `ym:a` + WHERE 1 + )), test2.b) AS test2_b +WHERE 1; + +DROP TABLE test1; +DROP TABLE test2; From 1989b50166760e2aba201c1aa50f7aebce780cb6 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 13 Sep 2019 08:57:30 +0300 Subject: [PATCH 0021/2007] Merge pull request #6928 from proller/fix27 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLICKHOUSE-4652 Another fix for АrrayEnumerateUniqRanked with empty arrays (cherry picked from commit ba40858094e06358d5d76f008059940ef9808823) --- dbms/src/Functions/array/arrayEnumerateRanked.h | 5 +---- .../queries/0_stateless/00909_arrayEnumerateUniq.reference | 3 +++ dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql | 3 +++ 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dbms/src/Functions/array/arrayEnumerateRanked.h b/dbms/src/Functions/array/arrayEnumerateRanked.h index a1019ba83bf..de7656b70da 100644 --- a/dbms/src/Functions/array/arrayEnumerateRanked.h +++ b/dbms/src/Functions/array/arrayEnumerateRanked.h @@ -336,10 +336,6 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( /// Skipping offsets if no data in this array if (prev_off == off) { - - if (depth_to_look > 2) - want_clear = true; - if (depth_to_look >= 2) { /// Advance to the next element of the parent array. @@ -357,6 +353,7 @@ void FunctionArrayEnumerateRankedExtended::executeMethodImpl( { last_offset_by_depth[depth] = (*offsets_by_depth[depth])[current_offset_n_by_depth[depth]]; ++current_offset_n_by_depth[depth]; + want_clear = true; } else { diff --git a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference index 595dcdf3803..6f9a0bb9ea3 100644 --- a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference +++ b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.reference @@ -284,3 +284,6 @@ a1,a2 12 [1,2] [[1],[],[2],[],[3],[],[4],[],[5],[],[6],[],[7],[],[8],[],[9]] [[],[1],[],[2],[],[3],[],[4],[],[5],[],[6],[],[7],[],[8]] [[1],[2],[],[3]] +-- empty corner +[[],[1],[]] [[],[1],[]] +[[1]] [[1]] diff --git a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql index 9cf82a368d6..33097c99272 100644 --- a/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql +++ b/dbms/tests/queries/0_stateless/00909_arrayEnumerateUniq.sql @@ -313,3 +313,6 @@ SELECT arrayEnumerateUniqRanked([[1], [], [1]]); SELECT arrayEnumerateUniqRanked([[1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1]]); SELECT arrayEnumerateUniqRanked([[], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1], [], [1]]); SELECT arrayEnumerateUniqRanked([[1], [1], [], [1]]); + +select '-- empty corner'; +SELECT a, arrayEnumerateUniqRanked(a) FROM ( SELECT * FROM ( SELECT [[],[1],[]] AS a UNION ALL SELECT [[1]] AS a ) ORDER BY a ASC ); From 497dffbf8bccdc5f0aceb7562e8f325c2352b2b2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 19 Sep 2019 22:21:38 +0300 Subject: [PATCH 0022/2007] Auto version update to [19.14.6.12] [54425] --- dbms/cmake/version.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 565f2157390..787f84ca33f 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -3,9 +3,9 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) set(VERSION_PATCH 6) -set(VERSION_GITHASH c6464f2813c7d5cb939af11759d05705ba0f15ee) -set(VERSION_DESCRIBE v19.14.6.1-prestable) -set(VERSION_STRING 19.14.6.1) +set(VERSION_GITHASH 1989b50166760e2aba201c1aa50f7aebce780cb6) +set(VERSION_DESCRIBE v19.14.6.12-stable) +set(VERSION_STRING 19.14.6.12) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") From 7e157f15180268be33bd0334e78a4d36e69fb82b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 19 Sep 2019 22:21:55 +0300 Subject: [PATCH 0023/2007] Auto version update to [19.14.7.1] [54425] --- dbms/cmake/version.cmake | 6 +++--- .../StorageSystemContributors.generated.cpp | 18 ++++++++++++++++++ debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 6 files changed, 26 insertions(+), 8 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 787f84ca33f..06ee301be79 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) -set(VERSION_PATCH 6) +set(VERSION_PATCH 7) set(VERSION_GITHASH 1989b50166760e2aba201c1aa50f7aebce780cb6) -set(VERSION_DESCRIBE v19.14.6.12-stable) -set(VERSION_STRING 19.14.6.12) +set(VERSION_DESCRIBE v19.14.7.1-stable) +set(VERSION_STRING 19.14.7.1) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/dbms/src/Storages/System/StorageSystemContributors.generated.cpp b/dbms/src/Storages/System/StorageSystemContributors.generated.cpp index debd1fe2dc6..3b6ccfc4db2 100644 --- a/dbms/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/dbms/src/Storages/System/StorageSystemContributors.generated.cpp @@ -6,6 +6,7 @@ const char * auto_contributors[] { "Aleksandra (Ася)", "Alex Bocharov", "Alex Krash", + "Alex Ryndin", "Alex Zatelepin", "Alexander Avdonkin", "Alexander Ermolaev", @@ -21,6 +22,7 @@ const char * auto_contributors[] { "Alexander Millin", "Alexander Mochalin", "Alexander Prudaev", + "Alexander Rodin", "Alexander Sapin", "Alexander Tokmakov", "Alexander Tretiakov", @@ -69,6 +71,7 @@ const char * auto_contributors[] { "Bakhtiyor Ruziev", "BanyRule", "BayoNet", + "Big Elephant", "BlahGeek", "Bogdan", "Bogdan Voronin", @@ -96,6 +99,7 @@ const char * auto_contributors[] { "Dmitry Petukhov", "Dmitry Rubashkin", "Dmitry S..ky / skype: dvska-at-skype", + "Doge", "Elghazal Ahmed", "Emmanuel Donin de Rosière", "Eric", @@ -108,6 +112,7 @@ const char * auto_contributors[] { "Fadi Hadzh", "FeehanG", "Flowyi", + "Francisco Barón", "Fruit of Eden", "Gary Dotzler", "George", @@ -235,6 +240,8 @@ const char * auto_contributors[] { "Pawel Rog", "Persiyanov Dmitriy Andreevich", "Quid37", + "Rafael David Tinoco", + "Ramazan Polat", "Ravengg", "Reto Kromer", "Roman Lipovsky", @@ -271,6 +278,7 @@ const char * auto_contributors[] { "The-Alchemist", "Tobias Adamson", "Tsarkova Anastasia", + "VDimir", "Vadim", "Vadim Plakhtinskiy", "Vadim Skipin", @@ -284,6 +292,7 @@ const char * auto_contributors[] { "Victor Tarnavsky", "Vitaliy Karnienko", "Vitaliy Lyudvichenko", + "Vitaliy Zakaznikov", "Vitaly Baranov", "Vitaly Samigullin", "Vivien Maisonneuve", @@ -296,6 +305,7 @@ const char * auto_contributors[] { "Vladislav Smirnov", "Vojtech Splichal", "Vsevolod Orlov", + "Vxider", "Vyacheslav Alipov", "Weiqing Xu", "William Shallum", @@ -312,9 +322,11 @@ const char * auto_contributors[] { "abdrakhmanov", "abyss7", "achulkov2", + "akazz", "akonyaev", "akuzm", "alesapin", + "alex-zaitsev", "alexander kozhikhov", "alexey-milovidov", "andrewsg", @@ -339,6 +351,8 @@ const char * auto_contributors[] { "davydovska", "decaseal", "dimarub2000", + "dmitrii", + "dmitriiut", "dmitry kuzmin", "eejoin", "egatov", @@ -363,6 +377,7 @@ const char * auto_contributors[] { "javi", "javi santana", "kmeaw", + "kreuzerkrieg", "ks1322", "kshvakov", "leozhang", @@ -389,6 +404,7 @@ const char * auto_contributors[] { "olegkv", "orantius", "peshkurov", + "philip.han", "proller", "pyos", "qianlixiang", @@ -399,6 +415,7 @@ const char * auto_contributors[] { "santaux", "sdk2", "serebrserg", + "sev7e0", "shangshujie", "shedx", "simon-says", @@ -408,6 +425,7 @@ const char * auto_contributors[] { "sundyli", "svladykin", "tai", + "tavplubix", "topvisor", "unknown", "urgordeadbeef", diff --git a/debian/changelog b/debian/changelog index 4d6d8be0b76..ac8f5586b00 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.14.6.1) unstable; urgency=low +clickhouse (19.14.7.1) unstable; urgency=low * Modified source code - -- clickhouse-release Wed, 18 Sep 2019 17:53:49 +0300 + -- clickhouse-release Thu, 19 Sep 2019 22:21:52 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 771e6405414..a3ea01d3a17 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.6.* +ARG version=19.14.7.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 0337faa86ff..dfc0e98a7ca 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.6.* +ARG version=19.14.7.* ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 62cc9c0b958..cc4f7b66320 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.14.6.* +ARG version=19.14.7.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From 2c930c6d803d1ffa99b3b5704e88a2471599656e Mon Sep 17 00:00:00 2001 From: proller Date: Tue, 10 Sep 2019 00:40:40 +0300 Subject: [PATCH 0024/2007] Build fixes (Orc, ...) (#6835) * Fix build * cmake: fix cpuinfo * Fix includes after processors merge Conflicts: dbms/src/Processors/Formats/Impl/CapnProtoRowInputFormat.cpp dbms/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp dbms/src/Processors/Formats/Impl/ProtobufRowInputFormat.cpp dbms/src/Processors/Formats/Impl/ProtobufRowOutputFormat.cpp * Fix build in gcc8 * fix test link * fix test link * Fix test link * link fix * Fix includes after processors merge 2 Conflicts: dbms/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp * Fix includes after processors merge 3 * link fix * Fix likely/unlikely conflict with cython * Fix conflict with protobuf/stubs/atomicops.h * remove unlikely.h * Fix macos build (do not use timer_t) * wip * Fix build (orc, ...) * Missing files * Try fix * fix hdfs * Fix llvm 7.1 find --- CMakeLists.txt | 2 +- cmake/find_hdfs3.cmake | 35 +-- cmake/find_llvm.cmake | 22 +- cmake/find_orc.cmake | 40 ++- cmake/find_parquet.cmake | 1 + contrib/CMakeLists.txt | 17 +- contrib/arrow-cmake/CMakeLists.txt | 24 +- contrib/arrow-cmake/orc_check.cmake | 126 ++++++++++ contrib/libhdfs3-cmake/CMakeLists.txt | 10 +- contrib/orc-cmake/CMakeLists.txt | 229 ++++++++++++++++++ .../Formats/Impl/ArrowColumnToCHColumn.cpp | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.h | 2 +- 12 files changed, 443 insertions(+), 67 deletions(-) create mode 100644 contrib/arrow-cmake/orc_check.cmake create mode 100644 contrib/orc-cmake/CMakeLists.txt diff --git a/CMakeLists.txt b/CMakeLists.txt index 5330c8daeb5..578e25b8e16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -343,7 +343,7 @@ include (cmake/find_hyperscan.cmake) include (cmake/find_simdjson.cmake) include (cmake/find_rapidjson.cmake) include (cmake/find_fastops.cmake) -include (cmake/find_orc.cmake) +#include (cmake/find_orc.cmake) find_contrib_lib(cityhash) find_contrib_lib(farmhash) diff --git a/cmake/find_hdfs3.cmake b/cmake/find_hdfs3.cmake index 4c29047fc75..9c593d3266a 100644 --- a/cmake/find_hdfs3.cmake +++ b/cmake/find_hdfs3.cmake @@ -1,24 +1,29 @@ -if (NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND USE_PROTOBUF) - option (ENABLE_HDFS "Enable HDFS" ${NOT_UNBUNDLED}) -endif () +if(NOT ARCH_ARM AND NOT OS_FREEBSD AND NOT APPLE AND USE_PROTOBUF) + option(ENABLE_HDFS "Enable HDFS" 1) +endif() -if (ENABLE_HDFS AND NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") - message (WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive") - set (ENABLE_HDFS 0) -endif () +if(ENABLE_HDFS) +option(USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ${NOT_UNBUNDLED}) -if (ENABLE_HDFS) -option (USE_INTERNAL_HDFS3_LIBRARY "Set to FALSE to use system HDFS3 instead of bundled" ON) +if(NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/hdfs.h") + if(USE_INTERNAL_HDFS3_LIBRARY) + message(WARNING "submodule contrib/libhdfs3 is missing. to fix try run: \n git submodule update --init --recursive") + endif() + set(MISSING_INTERNAL_HDFS3_LIBRARY 1) + set(USE_INTERNAL_HDFS3_LIBRARY 0) +endif() -if (NOT USE_INTERNAL_HDFS3_LIBRARY) - find_package(hdfs3) -endif () +if(NOT USE_INTERNAL_HDFS3_LIBRARY) + find_library(HDFS3_LIBRARY hdfs3) + find_path(HDFS3_INCLUDE_DIR NAMES hdfs/hdfs.h PATHS ${HDFS3_INCLUDE_PATHS}) +endif() -if (HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) +if(HDFS3_LIBRARY AND HDFS3_INCLUDE_DIR) set(USE_HDFS 1) -elseif (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) +elseif(NOT MISSING_INTERNAL_HDFS3_LIBRARY AND LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) set(HDFS3_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include") set(HDFS3_LIBRARY hdfs3) + set(USE_INTERNAL_HDFS3_LIBRARY 1) set(USE_HDFS 1) else() set(USE_INTERNAL_HDFS3_LIBRARY 0) @@ -26,4 +31,4 @@ endif() endif() -message (STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") +message(STATUS "Using hdfs3=${USE_HDFS}: ${HDFS3_INCLUDE_DIR} : ${HDFS3_LIBRARY}") diff --git a/cmake/find_llvm.cmake b/cmake/find_llvm.cmake index 3692a98b979..c668416c0c0 100644 --- a/cmake/find_llvm.cmake +++ b/cmake/find_llvm.cmake @@ -18,22 +18,12 @@ if (ENABLE_EMBEDDED_COMPILER) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") find_package(LLVM ${CMAKE_CXX_COMPILER_VERSION} CONFIG PATHS ${LLVM_PATHS}) else () - #TODO: - #if(NOT LLVM_FOUND) - # find_package(LLVM 9 CONFIG PATHS ${LLVM_PATHS}) - #endif() - #if(NOT LLVM_FOUND) - # find_package(LLVM 8 CONFIG PATHS ${LLVM_PATHS}) - #endif() - if (NOT LLVM_FOUND) - find_package (LLVM 7 CONFIG PATHS ${LLVM_PATHS}) - endif () - if (NOT LLVM_FOUND) - find_package (LLVM 6 CONFIG PATHS ${LLVM_PATHS}) - endif () - if (NOT LLVM_FOUND) - find_package (LLVM 5 CONFIG PATHS ${LLVM_PATHS}) - endif () + # TODO: 9 8 + foreach(llvm_v 7.1 7 6 5) + if (NOT LLVM_FOUND) + find_package (LLVM ${llvm_v} CONFIG PATHS ${LLVM_PATHS}) + endif () + endforeach () endif () if (LLVM_FOUND) diff --git a/cmake/find_orc.cmake b/cmake/find_orc.cmake index 3676bec1b6b..50e563b04b4 100644 --- a/cmake/find_orc.cmake +++ b/cmake/find_orc.cmake @@ -1,8 +1,38 @@ -##TODO replace hardcode to find procedure +option (ENABLE_ORC "Enable ORC" 1) -set(USE_ORC 0) -set(USE_INTERNAL_ORC_LIBRARY ON) +if(ENABLE_ORC) +option (USE_INTERNAL_ORC_LIBRARY "Set to FALSE to use system ORC instead of bundled" ${NOT_UNBUNDLED}) -if (ARROW_LIBRARY) +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include/orc/OrcFile.hh") + if(USE_INTERNAL_ORC_LIBRARY) + message(WARNING "submodule contrib/orc is missing. to fix try run: \n git submodule update --init --recursive") + set(USE_INTERNAL_ORC_LIBRARY 0) + endif() + set(MISSING_INTERNAL_ORC_LIBRARY 1) +endif () + +if (NOT USE_INTERNAL_ORC_LIBRARY) + find_package(orc) +endif () + +#if (USE_INTERNAL_ORC_LIBRARY) +#find_path(CYRUS_SASL_INCLUDE_DIR sasl/sasl.h) +#find_library(CYRUS_SASL_SHARED_LIB sasl2) +#if (NOT CYRUS_SASL_INCLUDE_DIR OR NOT CYRUS_SASL_SHARED_LIB) +# set(USE_ORC 0) +#endif() +#endif() + +if (ORC_LIBRARY AND ORC_INCLUDE_DIR) set(USE_ORC 1) -endif() \ No newline at end of file +elseif(NOT MISSING_INTERNAL_ORC_LIBRARY AND ARROW_LIBRARY) # (LIBGSASL_LIBRARY AND LIBXML2_LIBRARY) + set(ORC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include") + set(ORC_LIBRARY orc) + set(USE_ORC 1) +else() + set(USE_INTERNAL_ORC_LIBRARY 0) +endif() + +endif() + +message (STATUS "Using internal=${USE_INTERNAL_ORC_LIBRARY} orc=${USE_ORC}: ${ORC_INCLUDE_DIR} : ${ORC_LIBRARY}") diff --git a/cmake/find_parquet.cmake b/cmake/find_parquet.cmake index 63f589a9ea5..5c5bc664113 100644 --- a/cmake/find_parquet.cmake +++ b/cmake/find_parquet.cmake @@ -62,6 +62,7 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD) endif() set(USE_PARQUET 1) + set(USE_ORC 1) endif() endif() diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 96462de0190..54fdc4d69e0 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -10,19 +10,6 @@ endif () set_property(DIRECTORY PROPERTY EXCLUDE_FROM_ALL 1) -if (USE_INTERNAL_ORC_LIBRARY) - set(BUILD_JAVA OFF) - set (ANALYZE_JAVA OFF) - set (BUILD_CPP_TESTS OFF) - set (BUILD_TOOLS OFF) - option(BUILD_JAVA OFF) - option (ANALYZE_JAVA OFF) - option (BUILD_CPP_TESTS OFF) - option (BUILD_TOOLS OFF) - set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/contrib/orc/cmake_modules") - add_subdirectory(orc) -endif() - if (USE_INTERNAL_BOOST_LIBRARY) add_subdirectory (boost-cmake) endif () @@ -327,3 +314,7 @@ endif() if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() + +#if (USE_INTERNAL_ORC_LIBRARY) +# add_subdirectory(orc-cmake) +#endif () diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index ba1ddc2414a..cfd57f2b296 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -56,11 +56,11 @@ set(ORC_SOURCE_WRAP_DIR ${ORC_SOURCE_DIR}/wrap) set(ORC_BUILD_SRC_DIR ${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/src) set(ORC_BUILD_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/include) -set(GOOGLE_PROTOBUF_DIR ${ClickHouse_SOURCE_DIR}/contrib/protobuf/src/) +set(GOOGLE_PROTOBUF_DIR ${Protobuf_INCLUDE_DIR}/) set(ORC_ADDITION_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(ARROW_SRC_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src) -set(PROTOBUF_EXECUTABLE ${CMAKE_CURRENT_BINARY_DIR}/../protobuf/cmake/protoc) +set(PROTOBUF_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) set(PROTO_DIR ${ORC_SOURCE_DIR}/../proto) @@ -70,14 +70,10 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" "${PROTO_DIR}/orc_proto.proto") -include_directories(SYSTEM ${ORC_INCLUDE_DIR}) -include_directories(SYSTEM ${ORC_SOURCE_SRC_DIR}) -include_directories(SYSTEM ${ORC_SOURCE_WRAP_DIR}) -include_directories(SYSTEM ${GOOGLE_PROTOBUF_DIR}) -include_directories(SYSTEM ${ORC_BUILD_SRC_DIR}) -include_directories(SYSTEM ${ORC_BUILD_INCLUDE_DIR}) -include_directories(SYSTEM ${ORC_ADDITION_SOURCE_DIR}) -include_directories(SYSTEM ${ARROW_SRC_DIR}) +include(${ClickHouse_SOURCE_DIR}/contrib/orc/cmake_modules/CheckSourceCompiles.cmake) +include(orc_check.cmake) +configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DIR}/orc/orc-config.hh") +configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/Adaptor.hh") set(ORC_SRCS @@ -232,6 +228,14 @@ if (ARROW_WITH_ZSTD) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZSTD_LIBRARY}) endif() +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_INCLUDE_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_SOURCE_SRC_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_SOURCE_WRAP_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${GOOGLE_PROTOBUF_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_SRC_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_INCLUDE_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_ADDITION_SOURCE_DIR}) +target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_SRC_DIR}) # === parquet diff --git a/contrib/arrow-cmake/orc_check.cmake b/contrib/arrow-cmake/orc_check.cmake new file mode 100644 index 00000000000..ec1e53cc649 --- /dev/null +++ b/contrib/arrow-cmake/orc_check.cmake @@ -0,0 +1,126 @@ +# Not changed part of contrib/orc/c++/src/CMakeLists.txt + +INCLUDE(CheckCXXSourceCompiles) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int f = open(\"/x/y\", O_RDONLY); + char buf[100]; + return pread(f, buf, 100, 1000) == 0; + }" + HAS_PREAD +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char*[]){ + struct tm time2020; + return !strptime(\"2020-02-02 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2020); + }" + HAS_STRPTIME +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char* argv[]){ + return static_cast(std::stoll(argv[0])); + }" + HAS_STOLL +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int64_t x = 1; printf(\"%lld\",x); + }" + INT64_IS_LL +) + +CHECK_CXX_SOURCE_COMPILES(" + #ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored \"-Wdeprecated\" + #pragma clang diagnostic pop + #elif defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored \"-Wdeprecated\" + #pragma GCC diagnostic pop + #elif defined(_MSC_VER) + #pragma warning( push ) + #pragma warning( disable : 4996 ) + #pragma warning( pop ) + #else + unknownCompiler! + #endif + int main(int, char *[]) {}" + HAS_DIAGNOSTIC_PUSH +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + return std::isnan(1.0f); + }" + HAS_STD_ISNAN +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + std::mutex test_mutex; + std::lock_guard lock_mutex(test_mutex); + }" + HAS_STD_MUTEX +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + std::string func() { + std::string var = \"test\"; + return std::move(var); + } + int main(int, char *[]) {}" + NEEDS_REDUNDANT_MOVE +) + +INCLUDE(CheckCXXSourceRuns) + +CHECK_CXX_SOURCE_RUNS(" + #include + int main(int, char *[]) { + time_t t = -14210715; // 1969-07-20 12:34:45 + struct tm *ptm = gmtime(&t); + return !(ptm && ptm->tm_year == 69); + }" + HAS_PRE_1970 +) + +CHECK_CXX_SOURCE_RUNS(" + #include + #include + int main(int, char *[]) { + setenv(\"TZ\", \"America/Los_Angeles\", 1); + tzset(); + struct tm time2037; + struct tm time2038; + strptime(\"2037-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2037); + strptime(\"2038-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2038); + return mktime(&time2038) - mktime(&time2037) != 31536000; + }" + HAS_POST_2038 +) + +set(CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIR}) +set(CMAKE_REQUIRED_LIBRARIES zlib) +CHECK_CXX_SOURCE_COMPILES(" + #define Z_PREFIX + #include + z_stream strm; + int main(int, char *[]) { + deflateReset(&strm); + }" + NEEDS_Z_PREFIX +) diff --git a/contrib/libhdfs3-cmake/CMakeLists.txt b/contrib/libhdfs3-cmake/CMakeLists.txt index 8ec14f897b9..e1ba7225b0f 100644 --- a/contrib/libhdfs3-cmake/CMakeLists.txt +++ b/contrib/libhdfs3-cmake/CMakeLists.txt @@ -199,17 +199,17 @@ if (WITH_KERBEROS) endif() target_include_directories(hdfs3 PRIVATE ${LIBXML2_INCLUDE_DIR}) -target_link_libraries(hdfs3 ${LIBGSASL_LIBRARY}) +target_link_libraries(hdfs3 PRIVATE ${LIBGSASL_LIBRARY}) if (WITH_KERBEROS) - target_link_libraries(hdfs3 ${KERBEROS_LIBRARIES}) + target_link_libraries(hdfs3 PRIVATE ${KERBEROS_LIBRARIES}) endif() -target_link_libraries(hdfs3 ${LIBXML2_LIBRARY}) +target_link_libraries(hdfs3 PRIVATE ${LIBXML2_LIBRARY}) # inherit from parent cmake target_include_directories(hdfs3 PRIVATE ${Boost_INCLUDE_DIRS}) target_include_directories(hdfs3 PRIVATE ${Protobuf_INCLUDE_DIR}) -target_link_libraries(hdfs3 ${Protobuf_LIBRARY}) +target_link_libraries(hdfs3 PRIVATE ${Protobuf_LIBRARY}) if(OPENSSL_INCLUDE_DIR AND OPENSSL_LIBRARIES) target_include_directories(hdfs3 PRIVATE ${OPENSSL_INCLUDE_DIR}) - target_link_libraries(hdfs3 ${OPENSSL_LIBRARIES}) + target_link_libraries(hdfs3 PRIVATE ${OPENSSL_LIBRARIES}) endif() diff --git a/contrib/orc-cmake/CMakeLists.txt b/contrib/orc-cmake/CMakeLists.txt new file mode 100644 index 00000000000..066ba00aede --- /dev/null +++ b/contrib/orc-cmake/CMakeLists.txt @@ -0,0 +1,229 @@ +# modifyed copy of contrib/orc/c++/src/CMakeLists.txt +set(LIBRARY_INCLUDE ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/include) +set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/orc/c++/src) + +set(PROTOBUF_INCLUDE_DIR ${Protobuf_INCLUDE_DIR}) +set(PROTOBUF_EXECUTABLE ${Protobuf_PROTOC_EXECUTABLE}) + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX11_FLAGS} ${WARN_FLAGS}") + +INCLUDE(CheckCXXSourceCompiles) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int f = open(\"/x/y\", O_RDONLY); + char buf[100]; + return pread(f, buf, 100, 1000) == 0; + }" + HAS_PREAD +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char*[]){ + struct tm time2020; + return !strptime(\"2020-02-02 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2020); + }" + HAS_STRPTIME +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int,char* argv[]){ + return static_cast(std::stoll(argv[0])); + }" + HAS_STOLL +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + #include + int main(int,char*[]){ + int64_t x = 1; printf(\"%lld\",x); + }" + INT64_IS_LL +) + +CHECK_CXX_SOURCE_COMPILES(" + #ifdef __clang__ + #pragma clang diagnostic push + #pragma clang diagnostic ignored \"-Wdeprecated\" + #pragma clang diagnostic pop + #elif defined(__GNUC__) + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored \"-Wdeprecated\" + #pragma GCC diagnostic pop + #elif defined(_MSC_VER) + #pragma warning( push ) + #pragma warning( disable : 4996 ) + #pragma warning( pop ) + #else + unknownCompiler! + #endif + int main(int, char *[]) {}" + HAS_DIAGNOSTIC_PUSH +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + return std::isnan(1.0f); + }" + HAS_STD_ISNAN +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + int main(int, char *[]) { + std::mutex test_mutex; + std::lock_guard lock_mutex(test_mutex); + }" + HAS_STD_MUTEX +) + +CHECK_CXX_SOURCE_COMPILES(" + #include + std::string func() { + std::string var = \"test\"; + return std::move(var); + } + int main(int, char *[]) {}" + NEEDS_REDUNDANT_MOVE +) + +INCLUDE(CheckCXXSourceRuns) + +CHECK_CXX_SOURCE_RUNS(" + #include + int main(int, char *[]) { + time_t t = -14210715; // 1969-07-20 12:34:45 + struct tm *ptm = gmtime(&t); + return !(ptm && ptm->tm_year == 69); + }" + HAS_PRE_1970 +) + +CHECK_CXX_SOURCE_RUNS(" + #include + #include + int main(int, char *[]) { + setenv(\"TZ\", \"America/Los_Angeles\", 1); + tzset(); + struct tm time2037; + struct tm time2038; + strptime(\"2037-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2037); + strptime(\"2038-05-05 12:34:56\", \"%Y-%m-%d %H:%M:%S\", &time2038); + return mktime(&time2038) - mktime(&time2037) != 31536000; + }" + HAS_POST_2038 +) + +set(CMAKE_REQUIRED_INCLUDES ${ZLIB_INCLUDE_DIR}) +set(CMAKE_REQUIRED_LIBRARIES zlib) +CHECK_CXX_SOURCE_COMPILES(" + #define Z_PREFIX + #include + z_stream strm; + int main(int, char *[]) { + deflateReset(&strm); + }" + NEEDS_Z_PREFIX +) + +configure_file ( + "${LIBRARY_DIR}/Adaptor.hh.in" + "${CMAKE_CURRENT_BINARY_DIR}/Adaptor.hh" + ) + + +add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.cc + COMMAND ${PROTOBUF_EXECUTABLE} + -I${ClickHouse_SOURCE_DIR}/contrib/orc/proto + --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" + "${ClickHouse_SOURCE_DIR}/contrib/orc/proto/orc_proto.proto" +) + +set(SOURCE_FILES + "${CMAKE_CURRENT_BINARY_DIR}/Adaptor.hh" + ${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h + ${LIBRARY_DIR}/io/InputStream.cc + ${LIBRARY_DIR}/io/OutputStream.cc + ${LIBRARY_DIR}/wrap/orc-proto-wrapper.cc + ${LIBRARY_DIR}/Adaptor.cc + ${LIBRARY_DIR}/ByteRLE.cc + ${LIBRARY_DIR}/ColumnPrinter.cc + ${LIBRARY_DIR}/ColumnReader.cc + ${LIBRARY_DIR}/ColumnWriter.cc + ${LIBRARY_DIR}/Common.cc + ${LIBRARY_DIR}/Compression.cc + ${LIBRARY_DIR}/Exceptions.cc + ${LIBRARY_DIR}/Int128.cc + ${LIBRARY_DIR}/LzoDecompressor.cc + ${LIBRARY_DIR}/MemoryPool.cc + ${LIBRARY_DIR}/OrcFile.cc + ${LIBRARY_DIR}/Reader.cc + ${LIBRARY_DIR}/RLEv1.cc + ${LIBRARY_DIR}/RLEv2.cc + ${LIBRARY_DIR}/RLE.cc + ${LIBRARY_DIR}/Statistics.cc + ${LIBRARY_DIR}/StripeStream.cc + ${LIBRARY_DIR}/Timezone.cc + ${LIBRARY_DIR}/TypeImpl.cc + ${LIBRARY_DIR}/Vector.cc + ${LIBRARY_DIR}/Writer.cc + ) + +if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) + set(SOURCE_FILES ${SOURCE_FILES} ${LIBRARY_DIR}/OrcHdfsFile.cc) +endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) + +#list(TRANSFORM SOURCE_FILES PREPEND ${LIBRARY_DIR}/) + +configure_file ( + "${LIBRARY_INCLUDE}/orc/orc-config.hh.in" + "${CMAKE_CURRENT_BINARY_DIR}/orc/orc-config.hh" + ) + +add_library (orc ${SOURCE_FILES}) + +target_include_directories (orc + PRIVATE + ${LIBRARY_INCLUDE} + ${LIBRARY_DIR} + #PUBLIC + ${CMAKE_CURRENT_BINARY_DIR} + PRIVATE + ${PROTOBUF_INCLUDE_DIR} + ${ZLIB_INCLUDE_DIR} + ${SNAPPY_INCLUDE_DIR} + ${LZ4_INCLUDE_DIR} + ${LIBHDFSPP_INCLUDE_DIR} + ) + +target_link_libraries (orc PRIVATE + ${Protobuf_LIBRARY} + ${ZLIB_LIBRARIES} + ${SNAPPY_LIBRARY} + ${LZ4_LIBRARY} + ${LIBHDFSPP_LIBRARIES} + ) + +#install(TARGETS orc DESTINATION lib) + +if(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) + add_definitions(-DBUILD_LIBHDFSPP) +endif(ORC_CXX_HAS_THREAD_LOCAL AND BUILD_LIBHDFSPP) diff --git a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index 0cd5ffb03e0..edb8d5c15f4 100644 --- a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1,7 +1,7 @@ #include "config_formats.h" #include "ArrowColumnToCHColumn.h" -#if USE_ORC or USE_PARQUET +#if USE_ORC || USE_PARQUET #include #include #include diff --git a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index b5f4732d107..34b58a80091 100644 --- a/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/dbms/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -1,6 +1,6 @@ #include "config_formats.h" -#if USE_ORC or USE_PARQUET +#if USE_ORC || USE_PARQUET #include #include From 9690fbec093e84113214c39f18ac6ffe14c73983 Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 1 Oct 2019 11:44:52 +0300 Subject: [PATCH 0025/2007] Empty commit From 599e63425abe31c42cdf57b176f881db6f1f9e49 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 20 Sep 2019 20:32:51 +0300 Subject: [PATCH 0026/2007] Merge pull request #6990 from Akazz/timeout_for_sync_replica_cmd Fixed timeout mechanism for SYNC REPLICA command + simplified related code (cherry picked from commit 123b8cb43ccb3342805db79764dc03766289d904) --- .../Interpreters/InterpreterSystemQuery.cpp | 13 +++++++- .../Storages/StorageReplicatedMergeTree.cpp | 27 ++++++----------- ...3_sync_replica_timeout_zookeeper.reference | 1 + .../01013_sync_replica_timeout_zookeeper.sh | 30 +++++++++++++++++++ 4 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.reference create mode 100755 dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 6e434189c66..b6ed4dcb2e6 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -38,6 +38,7 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int CANNOT_KILL; extern const int NOT_IMPLEMENTED; + extern const int TIMEOUT_EXCEEDED; } @@ -331,7 +332,17 @@ void InterpreterSystemQuery::syncReplica(ASTSystemQuery & query) StoragePtr table = context.getTable(database_name, table_name); if (auto storage_replicated = dynamic_cast(table.get())) - storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.value.milliseconds()); + { + LOG_TRACE(log, "Synchronizing entries in replica's queue with table's log and waiting for it to become empty"); + if (!storage_replicated->waitForShrinkingQueueSize(0, context.getSettingsRef().receive_timeout.totalMilliseconds())) + { + LOG_ERROR(log, "SYNC REPLICA " + database_name + "." + table_name + ": Timed out!"); + throw Exception( + "SYNC REPLICA " + database_name + "." + table_name + ": command timed out! " + "See the 'receive_timeout' setting", ErrorCodes::TIMEOUT_EXCEEDED); + } + LOG_TRACE(log, "SYNC REPLICA " + database_name + "." + table_name + ": OK"); + } else throw Exception("Table " + database_name + "." + table_name + " is not replicated", ErrorCodes::BAD_ARGUMENTS); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index e5821c1bcaf..64e011df99c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -5126,38 +5126,29 @@ ActionLock StorageReplicatedMergeTree::getActionLock(StorageActionBlockType acti bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UInt64 max_wait_milliseconds) { + Stopwatch watch; + /// Let's fetch new log entries firstly queue.pullLogsToQueue(getZooKeeper()); - Stopwatch watch; - Poco::Event event; - std::atomic cond_reached{false}; - - auto callback = [&event, &cond_reached, queue_size] (size_t new_queue_size) + Poco::Event target_size_event; + auto callback = [&target_size_event, queue_size] (size_t new_queue_size) { if (new_queue_size <= queue_size) - cond_reached.store(true, std::memory_order_relaxed); - - event.set(); + target_size_event.set(); }; + const auto handler = queue.addSubscriber(std::move(callback)); - auto handler = queue.addSubscriber(std::move(callback)); - - while (true) + while (!target_size_event.tryWait(50)) { - event.tryWait(50); - if (max_wait_milliseconds && watch.elapsedMilliseconds() > max_wait_milliseconds) - break; - - if (cond_reached) - break; + return false; if (partial_shutdown_called) throw Exception("Shutdown is called for table", ErrorCodes::ABORTED); } - return cond_reached.load(std::memory_order_relaxed); + return true; } diff --git a/dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.reference b/dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.reference @@ -0,0 +1 @@ +OK diff --git a/dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh b/dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh new file mode 100755 index 00000000000..9e846b42591 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01013_sync_replica_timeout_zookeeper.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + + +R1=table_1013_1 +R2=table_1013_2 + +${CLICKHOUSE_CLIENT} -n -q " + DROP TABLE IF EXISTS $R1; + DROP TABLE IF EXISTS $R2; + + CREATE TABLE $R1 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1013', 'r1') ORDER BY x; + CREATE TABLE $R2 (x UInt32) ENGINE ReplicatedMergeTree('/clickhouse/tables/${CLICKHOUSE_DATABASE}.table_1013', 'r2') ORDER BY x; + + SYSTEM STOP FETCHES $R2; + INSERT INTO $R1 VALUES (1) +" + +timeout 10s ${CLICKHOUSE_CLIENT} -n -q " + SET receive_timeout=1; + SYSTEM SYNC REPLICA $R2 +" 2>&1 | fgrep -q "DB::Exception: SYNC REPLICA ${CLICKHOUSE_DATABASE}.$R2: command timed out!" && echo 'OK' || echo 'Failed!' + +# By dropping tables all related SYNC REPLICA queries would be terminated as well +${CLICKHOUSE_CLIENT} -n -q " + DROP TABLE IF EXISTS $R2; + DROP TABLE IF EXISTS $R1; +" From d83de5e61f05da5848698768497dd551496e6f88 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 20 Sep 2019 23:45:51 +0300 Subject: [PATCH 0027/2007] Merge pull request #7022 from 4ertus2/bugs Fix "Unknown identifier" in ORDER BY and GROUP BY with Multiple Joins (cherry picked from commit 2432a68009bc726f9efdfa94b85f348096b1ab6c) --- .../JoinToSubqueryTransformVisitor.cpp | 21 ++++++----- .../00847_multiple_join_same_column.sql | 28 +++++++++++---- .../00882_multiple_join_no_alias.reference | 8 +++++ .../00882_multiple_join_no_alias.sql | 35 +++++++++++++++++++ 4 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.reference create mode 100644 dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.sql diff --git a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index c6e72b4d252..b60e6533921 100644 --- a/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/dbms/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -194,14 +194,14 @@ struct ColumnAliasesMatcher } }; - static bool needChildVisit(ASTPtr & node, const ASTPtr &) + static bool needChildVisit(const ASTPtr & node, const ASTPtr &) { if (node->as()) return false; return true; } - static void visit(ASTPtr & ast, Data & data) + static void visit(const ASTPtr & ast, Data & data) { if (auto * t = ast->as()) visit(*t, ast, data); @@ -210,8 +210,9 @@ struct ColumnAliasesMatcher throw Exception("Multiple JOIN do not support asterisks for complex queries yet", ErrorCodes::NOT_IMPLEMENTED); } - static void visit(ASTIdentifier & node, ASTPtr &, Data & data) + static void visit(const ASTIdentifier & const_node, const ASTPtr &, Data & data) { + ASTIdentifier & node = const_cast(const_node); /// we know it's not const if (node.isShort()) return; @@ -375,7 +376,7 @@ using RewriteVisitor = InDepthNodeVisitor; using SetSubqueryAliasMatcher = OneTypeMatcher; using SetSubqueryAliasVisitor = InDepthNodeVisitor; using ExtractAsterisksVisitor = ExtractAsterisksMatcher::Visitor; -using ColumnAliasesVisitor = InDepthNodeVisitor; +using ColumnAliasesVisitor = ConstInDepthNodeVisitor; using AppendSemanticMatcher = OneTypeMatcher; using AppendSemanticVisitor = InDepthNodeVisitor; @@ -403,15 +404,19 @@ void JoinToSubqueryTransformMatcher::visit(ASTSelectQuery & select, ASTPtr & ast if (select.select()) { aliases_data.public_names = true; - ColumnAliasesVisitor(aliases_data).visit(select.refSelect()); + ColumnAliasesVisitor(aliases_data).visit(select.select()); aliases_data.public_names = false; } if (select.where()) - ColumnAliasesVisitor(aliases_data).visit(select.refWhere()); + ColumnAliasesVisitor(aliases_data).visit(select.where()); if (select.prewhere()) - ColumnAliasesVisitor(aliases_data).visit(select.refPrewhere()); + ColumnAliasesVisitor(aliases_data).visit(select.prewhere()); + if (select.orderBy()) + ColumnAliasesVisitor(aliases_data).visit(select.orderBy()); + if (select.groupBy()) + ColumnAliasesVisitor(aliases_data).visit(select.groupBy()); if (select.having()) - ColumnAliasesVisitor(aliases_data).visit(select.refHaving()); + ColumnAliasesVisitor(aliases_data).visit(select.having()); /// JOIN sections for (auto & child : select.tables()->children) diff --git a/dbms/tests/queries/0_stateless/00847_multiple_join_same_column.sql b/dbms/tests/queries/0_stateless/00847_multiple_join_same_column.sql index d444655a6ce..44b3fe202d3 100644 --- a/dbms/tests/queries/0_stateless/00847_multiple_join_same_column.sql +++ b/dbms/tests/queries/0_stateless/00847_multiple_join_same_column.sql @@ -16,30 +16,44 @@ left join y on (y.a = s.a and y.b = s.b) format Vertical; select t.a, s.b, s.a, s.b, y.a, y.b from t left join s on (t.a = s.a and s.b = t.b) -left join y on (y.a = s.a and y.b = s.b) format PrettyCompactNoEscapes; +left join y on (y.a = s.a and y.b = s.b) +order by t.a +format PrettyCompactNoEscapes; select t.a as t_a from t -left join s on s.a = t_a format PrettyCompactNoEscapes; +left join s on s.a = t_a +order by t.a +format PrettyCompactNoEscapes; select t.a, s.a as s_a from t left join s on s.a = t.a -left join y on y.b = s.b format PrettyCompactNoEscapes; +left join y on y.b = s.b +order by t.a +format PrettyCompactNoEscapes; select t.a, t.a, t.b as t_b from t left join s on t.a = s.a -left join y on y.b = s.b format PrettyCompactNoEscapes; +left join y on y.b = s.b +order by t.a +format PrettyCompactNoEscapes; select s.a, s.a, s.b as s_b, s.b from t left join s on s.a = t.a -left join y on s.b = y.b format PrettyCompactNoEscapes; +left join y on s.b = y.b +order by t.a +format PrettyCompactNoEscapes; select y.a, y.a, y.b as y_b, y.b from t left join s on s.a = t.a -left join y on y.b = s.b format PrettyCompactNoEscapes; +left join y on y.b = s.b +order by t.a +format PrettyCompactNoEscapes; select t.a, t.a as t_a, s.a, s.a as s_a, y.a, y.a as y_a from t left join s on t.a = s.a -left join y on y.b = s.b format PrettyCompactNoEscapes; +left join y on y.b = s.b +order by t.a +format PrettyCompactNoEscapes; drop table t; drop table s; diff --git a/dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.reference b/dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.reference new file mode 100644 index 00000000000..a3723bc9976 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.reference @@ -0,0 +1,8 @@ +1 1 1 1 +0 0 0 0 +0 +1 +1 1 1 1 1 1 +2 2 0 0 0 0 +2 2 0 +1 1 1 diff --git a/dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.sql b/dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.sql new file mode 100644 index 00000000000..bd3a2a19913 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00882_multiple_join_no_alias.sql @@ -0,0 +1,35 @@ +drop table if exists t; +drop table if exists s; +drop table if exists y; + +create table t(a Int64, b Int64) engine = Memory; +create table s(a Int64, b Int64) engine = Memory; +create table y(a Int64, b Int64) engine = Memory; + +insert into t values (1,1), (2,2); +insert into s values (1,1); +insert into y values (1,1); + +select s.a, s.a, s.b as s_b, s.b from t +left join s on s.a = t.a +left join y on s.b = y.b +order by t.a; + +select max(s.a) from t +left join s on s.a = t.a +left join y on s.b = y.b +group by t.a; + +select t.a, t.a as t_a, s.a, s.a as s_a, y.a, y.a as y_a from t +left join s on t.a = s.a +left join y on y.b = s.b +order by t.a; + +select t.a, t.a as t_a, max(s.a) from t +left join s on t.a = s.a +left join y on y.b = s.b +group by t.a; + +drop table t; +drop table s; +drop table y; From 9533ff12ddd5d2fab161381d27ed3c964f4f1c9d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 22 Sep 2019 05:12:55 +0300 Subject: [PATCH 0028/2007] Merge pull request #7032 from azat-archive/distributed-directory-monitor-SIGSEGV Avoid SIGSEGV on batch send failure (file with index XX is absent) (cherry picked from commit 5970aafd9aa5f1c885365297dfdb3d10fec8fdb8) --- dbms/src/Storages/Distributed/DirectoryMonitor.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 13c9cf3050a..6d5bffe3491 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -390,7 +390,8 @@ struct StorageDistributedDirectoryMonitor::Batch remote->writePrepared(in); } - remote->writeSuffix(); + if (remote) + remote->writeSuffix(); } catch (const Exception & e) { From af5099b869f9656389bd7bc10f8d6b77b692201d Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 24 Sep 2019 02:11:15 +0300 Subject: [PATCH 0029/2007] Merge pull request #7062 from ClickHouse/fix-msan-for-low-cardinality Fix msan for LowCardinality (cherry picked from commit a314a36feb18b19217d7f544bd8ef15c2fcd1c53) --- dbms/src/Core/Block.cpp | 5 +++- dbms/src/Core/Block.h | 2 +- dbms/src/Functions/IFunction.cpp | 50 +++++++++++++++++++------------- 3 files changed, 35 insertions(+), 22 deletions(-) diff --git a/dbms/src/Core/Block.cpp b/dbms/src/Core/Block.cpp index b045b9ec1ff..c64cf387a3b 100644 --- a/dbms/src/Core/Block.cpp +++ b/dbms/src/Core/Block.cpp @@ -219,11 +219,14 @@ size_t Block::getPositionByName(const std::string & name) const } -void Block::checkNumberOfRows() const +void Block::checkNumberOfRows(bool allow_null_columns) const { ssize_t rows = -1; for (const auto & elem : data) { + if (!elem.column && allow_null_columns) + continue; + if (!elem.column) throw Exception("Column " + elem.name + " in block is nullptr, in method checkNumberOfRows." , ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); diff --git a/dbms/src/Core/Block.h b/dbms/src/Core/Block.h index 4a93e5ed803..ae8b07718dd 100644 --- a/dbms/src/Core/Block.h +++ b/dbms/src/Core/Block.h @@ -90,7 +90,7 @@ public: size_t columns() const { return data.size(); } /// Checks that every column in block is not nullptr and has same number of elements. - void checkNumberOfRows() const; + void checkNumberOfRows(bool allow_null_columns = false) const; /// Approximate number of bytes in memory - for profiling and limits. size_t bytes() const; diff --git a/dbms/src/Functions/IFunction.cpp b/dbms/src/Functions/IFunction.cpp index a86ea724f7a..9a3633a9790 100644 --- a/dbms/src/Functions/IFunction.cpp +++ b/dbms/src/Functions/IFunction.cpp @@ -337,19 +337,43 @@ static ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes( size_t num_rows = input_rows_count; ColumnPtr indexes; + /// Find first LowCardinality column and replace it to nested dictionary. for (auto arg : args) { ColumnWithTypeAndName & column = block.getByPosition(arg); if (auto * low_cardinality_column = checkAndGetColumn(column.column.get())) { + /// Single LowCardinality column is supported now. if (indexes) throw Exception("Expected single dictionary argument for function.", ErrorCodes::LOGICAL_ERROR); - indexes = low_cardinality_column->getIndexesPtr(); - num_rows = low_cardinality_column->getDictionary().size(); + auto * low_cardinality_type = checkAndGetDataType(column.type.get()); + + if (!low_cardinality_type) + throw Exception("Incompatible type for low cardinality column: " + column.type->getName(), + ErrorCodes::LOGICAL_ERROR); + + if (can_be_executed_on_default_arguments) + { + /// Normal case, when function can be executed on values's default. + column.column = low_cardinality_column->getDictionary().getNestedColumn(); + indexes = low_cardinality_column->getIndexesPtr(); + } + else + { + /// Special case when default value can't be used. Example: 1 % LowCardinality(Int). + /// LowCardinality always contains default, so 1 % 0 will throw exception in normal case. + auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size()); + column.column = dict_encoded.dictionary; + indexes = dict_encoded.indexes; + } + + num_rows = column.column->size(); + column.type = low_cardinality_type->getDictionaryType(); } } + /// Change size of constants. for (auto arg : args) { ColumnWithTypeAndName & column = block.getByPosition(arg); @@ -358,26 +382,12 @@ static ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes( column.column = column_const->removeLowCardinality()->cloneResized(num_rows); column.type = removeLowCardinality(column.type); } - else if (auto * low_cardinality_column = checkAndGetColumn(column.column.get())) - { - auto * low_cardinality_type = checkAndGetDataType(column.type.get()); - - if (!low_cardinality_type) - throw Exception("Incompatible type for low cardinality column: " + column.type->getName(), - ErrorCodes::LOGICAL_ERROR); - - if (can_be_executed_on_default_arguments) - column.column = low_cardinality_column->getDictionary().getNestedColumn(); - else - { - auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size()); - column.column = dict_encoded.dictionary; - indexes = dict_encoded.indexes; - } - column.type = low_cardinality_type->getDictionaryType(); - } } +#ifndef NDEBUG + block.checkNumberOfRows(true); +#endif + return indexes; } From 8e4dd482496062fd5fa74da4ce8d40e5866f4ad8 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 24 Sep 2019 02:11:33 +0300 Subject: [PATCH 0030/2007] Merge pull request #7069 from ClickHouse/compatibility-settings-19.14 Fixed compatibility for distributed queries between 19.14 and earlier versions (cherry picked from commit 575ddefa6c32c692f2db7ee8688ca586f3b1d622) --- dbms/src/Interpreters/ClusterProxy/executeQuery.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index 989595b3647..9a0494cca45 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -18,8 +18,6 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin { Settings new_settings = settings; new_settings.queue_max_wait_ms = Cluster::saturate(new_settings.queue_max_wait_ms, settings.max_execution_time); - new_settings.connection_pool_max_wait_ms = Cluster::saturate(new_settings.connection_pool_max_wait_ms, settings.max_execution_time); - new_settings.replace_running_query_max_wait_ms = Cluster::saturate(new_settings.replace_running_query_max_wait_ms, settings.max_execution_time); /// Does not matter on remote servers, because queries are sent under different user. new_settings.max_concurrent_queries_for_user = 0; @@ -39,8 +37,8 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin } BlockInputStreams executeQuery( - IStreamFactory & stream_factory, const ClusterPtr & cluster, - const ASTPtr & query_ast, const Context & context, const Settings & settings) + IStreamFactory & stream_factory, const ClusterPtr & cluster, + const ASTPtr & query_ast, const Context & context, const Settings & settings) { BlockInputStreams res; From 79b96de31af46895726fc71072fded6263c7f5da Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 24 Sep 2019 05:18:15 +0300 Subject: [PATCH 0031/2007] Merge pull request #7075 from ClickHouse/revert-index-analysis-slowdown Revert "CHYT-142: extend KeyCondition interface so that it returns BoolMask" (cherry picked from commit a62866918fc333338bc7ac624db1681761fc5029) --- dbms/src/Interpreters/Set.cpp | 2 +- dbms/src/Interpreters/Set.h | 4 +- dbms/src/Storages/MergeTree/KeyCondition.cpp | 39 ++++++++++--------- dbms/src/Storages/MergeTree/KeyCondition.h | 14 +++---- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 12 +++--- .../MergeTree/MergeTreeIndexFullText.cpp | 4 +- .../MergeTree/MergeTreeIndexMinMax.cpp | 2 +- 7 files changed, 39 insertions(+), 38 deletions(-) diff --git a/dbms/src/Interpreters/Set.cpp b/dbms/src/Interpreters/Set.cpp index 4313decd36d..68c219c3a91 100644 --- a/dbms/src/Interpreters/Set.cpp +++ b/dbms/src/Interpreters/Set.cpp @@ -470,7 +470,7 @@ MergeTreeSetIndex::MergeTreeSetIndex(const Columns & set_elements, std::vector & key_ranges, const DataTypes & data_types) +BoolMask MergeTreeSetIndex::mayBeTrueInRange(const std::vector & key_ranges, const DataTypes & data_types) { size_t tuple_size = indexes_mapping.size(); diff --git a/dbms/src/Interpreters/Set.h b/dbms/src/Interpreters/Set.h index a252f1ebc1e..987252e37ba 100644 --- a/dbms/src/Interpreters/Set.h +++ b/dbms/src/Interpreters/Set.h @@ -170,7 +170,7 @@ using Sets = std::vector; class IFunction; using FunctionPtr = std::shared_ptr; -/// Class for checkInRange function. +/// Class for mayBeTrueInRange function. class MergeTreeSetIndex { public: @@ -188,7 +188,7 @@ public: size_t size() const { return ordered_set.at(0)->size(); } - BoolMask checkInRange(const std::vector & key_ranges, const DataTypes & data_types); + BoolMask mayBeTrueInRange(const std::vector & key_ranges, const DataTypes & data_types); private: Columns ordered_set; diff --git a/dbms/src/Storages/MergeTree/KeyCondition.cpp b/dbms/src/Storages/MergeTree/KeyCondition.cpp index 0ebe8f79aba..b3e4c776605 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.cpp +++ b/dbms/src/Storages/MergeTree/KeyCondition.cpp @@ -886,7 +886,7 @@ String KeyCondition::toString() const */ template -static BoolMask forAnyParallelogram( +static bool forAnyParallelogram( size_t key_size, const Field * key_left, const Field * key_right, @@ -942,15 +942,16 @@ static BoolMask forAnyParallelogram( for (size_t i = prefix_size + 1; i < key_size; ++i) parallelogram[i] = Range(); - BoolMask result(false, false); - result = result | callback(parallelogram); + if (callback(parallelogram)) + return true; /// [x1] x [y1 .. +inf) if (left_bounded) { parallelogram[prefix_size] = Range(key_left[prefix_size]); - result = result | forAnyParallelogram(key_size, key_left, key_right, true, false, parallelogram, prefix_size + 1, callback); + if (forAnyParallelogram(key_size, key_left, key_right, true, false, parallelogram, prefix_size + 1, callback)) + return true; } /// [x2] x (-inf .. y2] @@ -958,14 +959,15 @@ static BoolMask forAnyParallelogram( if (right_bounded) { parallelogram[prefix_size] = Range(key_right[prefix_size]); - result = result | forAnyParallelogram(key_size, key_left, key_right, false, true, parallelogram, prefix_size + 1, callback); + if (forAnyParallelogram(key_size, key_left, key_right, false, true, parallelogram, prefix_size + 1, callback)) + return true; } - return result; + return false; } -BoolMask KeyCondition::checkInRange( +bool KeyCondition::mayBeTrueInRange( size_t used_key_size, const Field * left_key, const Field * right_key, @@ -991,7 +993,7 @@ BoolMask KeyCondition::checkInRange( return forAnyParallelogram(used_key_size, left_key, right_key, true, right_bounded, key_ranges, 0, [&] (const std::vector & key_ranges_parallelogram) { - auto res = checkInParallelogram(key_ranges_parallelogram, data_types); + auto res = mayBeTrueInParallelogram(key_ranges_parallelogram, data_types); /* std::cerr << "Parallelogram: "; for (size_t i = 0, size = key_ranges.size(); i != size; ++i) @@ -1002,11 +1004,11 @@ BoolMask KeyCondition::checkInRange( }); } - std::optional KeyCondition::applyMonotonicFunctionsChainToRange( Range key_range, MonotonicFunctionsChain & functions, - DataTypePtr current_type) + DataTypePtr current_type +) { for (auto & func : functions) { @@ -1039,7 +1041,7 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( return key_range; } -BoolMask KeyCondition::checkInParallelogram(const std::vector & parallelogram, const DataTypes & data_types) const +bool KeyCondition::mayBeTrueInParallelogram(const std::vector & parallelogram, const DataTypes & data_types) const { std::vector rpn_stack; for (size_t i = 0; i < rpn.size(); ++i) @@ -1087,7 +1089,7 @@ BoolMask KeyCondition::checkInParallelogram(const std::vector & parallelo if (!element.set_index) throw Exception("Set for IN is not created yet", ErrorCodes::LOGICAL_ERROR); - rpn_stack.emplace_back(element.set_index->checkInRange(parallelogram, data_types)); + rpn_stack.emplace_back(element.set_index->mayBeTrueInRange(parallelogram, data_types)); if (element.function == RPNElement::FUNCTION_NOT_IN_SET) rpn_stack.back() = !rpn_stack.back(); } @@ -1122,23 +1124,22 @@ BoolMask KeyCondition::checkInParallelogram(const std::vector & parallelo } if (rpn_stack.size() != 1) - throw Exception("Unexpected stack size in KeyCondition::checkInRange", ErrorCodes::LOGICAL_ERROR); + throw Exception("Unexpected stack size in KeyCondition::mayBeTrueInRange", ErrorCodes::LOGICAL_ERROR); - return rpn_stack[0]; + return rpn_stack[0].can_be_true; } -BoolMask KeyCondition::checkInRange( +bool KeyCondition::mayBeTrueInRange( size_t used_key_size, const Field * left_key, const Field * right_key, const DataTypes & data_types) const { - return checkInRange(used_key_size, left_key, right_key, data_types, true); + return mayBeTrueInRange(used_key_size, left_key, right_key, data_types, true); } - -BoolMask KeyCondition::getMaskAfter( +bool KeyCondition::mayBeTrueAfter( size_t used_key_size, const Field * left_key, const DataTypes & data_types) const { - return checkInRange(used_key_size, left_key, nullptr, data_types, false); + return mayBeTrueInRange(used_key_size, left_key, nullptr, data_types, false); } diff --git a/dbms/src/Storages/MergeTree/KeyCondition.h b/dbms/src/Storages/MergeTree/KeyCondition.h index 2a5c520b243..61989d1b2d9 100644 --- a/dbms/src/Storages/MergeTree/KeyCondition.h +++ b/dbms/src/Storages/MergeTree/KeyCondition.h @@ -235,17 +235,17 @@ public: const Names & key_column_names, const ExpressionActionsPtr & key_expr); - /// Whether the condition and its negation are (independently) feasible in the key range. + /// Whether the condition is feasible in the key range. /// left_key and right_key must contain all fields in the sort_descr in the appropriate order. /// data_types - the types of the key columns. - BoolMask checkInRange(size_t used_key_size, const Field * left_key, const Field * right_key, const DataTypes & data_types) const; + bool mayBeTrueInRange(size_t used_key_size, const Field * left_key, const Field * right_key, const DataTypes & data_types) const; - /// Whether the condition and its negation are feasible in the direct product of single column ranges specified by `parallelogram`. - BoolMask checkInParallelogram(const std::vector & parallelogram, const DataTypes & data_types) const; + /// Whether the condition is feasible in the direct product of single column ranges specified by `parallelogram`. + bool mayBeTrueInParallelogram(const std::vector & parallelogram, const DataTypes & data_types) const; - /// Are the condition and its negation valid in a semi-infinite (not limited to the right) key range. + /// Is the condition valid in a semi-infinite (not limited to the right) key range. /// left_key must contain all the fields in the sort_descr in the appropriate order. - BoolMask getMaskAfter(size_t used_key_size, const Field * left_key, const DataTypes & data_types) const; + bool mayBeTrueAfter(size_t used_key_size, const Field * left_key, const DataTypes & data_types) const; /// Checks that the index can not be used. bool alwaysUnknownOrTrue() const; @@ -330,7 +330,7 @@ public: static const AtomMap atom_map; private: - BoolMask checkInRange( + bool mayBeTrueInRange( size_t used_key_size, const Field * left_key, const Field * right_key, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 40dc0bf6b52..99b4a49d111 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -269,8 +269,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( if (part->isEmpty()) continue; - if (minmax_idx_condition && !minmax_idx_condition->checkInParallelogram( - part->minmax_idx.parallelogram, data.minmax_idx_column_types).can_be_true) + if (minmax_idx_condition && !minmax_idx_condition->mayBeTrueInParallelogram( + part->minmax_idx.parallelogram, data.minmax_idx_column_types)) continue; if (max_block_numbers_to_read) @@ -1200,8 +1200,8 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( for (size_t i = 0; i < used_key_size; ++i) index[i]->get(range.begin, index_left[i]); - may_be_true = key_condition.getMaskAfter( - used_key_size, index_left.data(), data.primary_key_data_types).can_be_true; + may_be_true = key_condition.mayBeTrueAfter( + used_key_size, index_left.data(), data.primary_key_data_types); } else { @@ -1214,8 +1214,8 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange( index[i]->get(range.end, index_right[i]); } - may_be_true = key_condition.checkInRange( - used_key_size, index_left.data(), index_right.data(), data.primary_key_data_types).can_be_true; + may_be_true = key_condition.mayBeTrueInRange( + used_key_size, index_left.data(), index_right.data(), data.primary_key_data_types); } if (!may_be_true) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp index 246ad6784b2..264c91cd890 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexFullText.cpp @@ -378,11 +378,11 @@ bool MergeTreeConditionFullText::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx rpn_stack.emplace_back(true, false); } else - throw Exception("Unexpected function type in BloomFilterCondition::RPNElement", ErrorCodes::LOGICAL_ERROR); + throw Exception("Unexpected function type in KeyCondition::RPNElement", ErrorCodes::LOGICAL_ERROR); } if (rpn_stack.size() != 1) - throw Exception("Unexpected stack size in BloomFilterCondition::mayBeTrueOnGranule", ErrorCodes::LOGICAL_ERROR); + throw Exception("Unexpected stack size in KeyCondition::mayBeTrueInRange", ErrorCodes::LOGICAL_ERROR); return rpn_stack[0].can_be_true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp index 360e69eacc6..37c094db215 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp @@ -143,7 +143,7 @@ bool MergeTreeIndexConditionMinMax::mayBeTrueOnGranule(MergeTreeIndexGranulePtr for (const auto & range : granule->parallelogram) if (range.left.isNull() || range.right.isNull()) return true; - return condition.checkInParallelogram(granule->parallelogram, index.data_types).can_be_true; + return condition.mayBeTrueInParallelogram(granule->parallelogram, index.data_types); } From 269dcad274685a4bda2104036565381801162ffe Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 2 Oct 2019 20:29:01 +0300 Subject: [PATCH 0032/2007] Auto version update to [19.14.7.15] [54425] --- dbms/cmake/version.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index 06ee301be79..05acccfe7c3 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -3,9 +3,9 @@ set(VERSION_REVISION 54425) set(VERSION_MAJOR 19) set(VERSION_MINOR 14) set(VERSION_PATCH 7) -set(VERSION_GITHASH 1989b50166760e2aba201c1aa50f7aebce780cb6) -set(VERSION_DESCRIBE v19.14.7.1-stable) -set(VERSION_STRING 19.14.7.1) +set(VERSION_GITHASH 79b96de31af46895726fc71072fded6263c7f5da) +set(VERSION_DESCRIBE v19.14.7.15-stable) +set(VERSION_STRING 19.14.7.15) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") From 314144630231b7ad65344d66618646015a47550a Mon Sep 17 00:00:00 2001 From: Clement Rodriguez Date: Tue, 8 Oct 2019 19:27:00 +0200 Subject: [PATCH 0033/2007] Implemented MySQL connection mutualization for external MySQL dictionaries --- .../Dictionaries/MySQLDictionarySource.cpp | 4 +- .../src/Interpreters/ExternalDictionaries.cpp | 20 +++ dbms/src/Interpreters/ExternalDictionaries.h | 9 +- libs/libmysqlxx/CMakeLists.txt | 2 + libs/libmysqlxx/include/mysqlxx/PoolFactory.h | 51 +++++++ .../include/mysqlxx/PoolWithFailover.h | 4 + libs/libmysqlxx/src/PoolFactory.cpp | 136 ++++++++++++++++++ libs/libmysqlxx/src/PoolWithFailover.cpp | 21 ++- 8 files changed, 237 insertions(+), 10 deletions(-) create mode 100644 libs/libmysqlxx/include/mysqlxx/PoolFactory.h create mode 100644 libs/libmysqlxx/src/PoolFactory.cpp diff --git a/dbms/src/Dictionaries/MySQLDictionarySource.cpp b/dbms/src/Dictionaries/MySQLDictionarySource.cpp index 497448bf64c..38ddd6df921 100644 --- a/dbms/src/Dictionaries/MySQLDictionarySource.cpp +++ b/dbms/src/Dictionaries/MySQLDictionarySource.cpp @@ -6,7 +6,6 @@ #include "DictionaryStructure.h" - namespace DB { namespace ErrorCodes @@ -47,6 +46,7 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) # include # include # include "readInvalidateQuery.h" +# include namespace DB { @@ -67,7 +67,7 @@ MySQLDictionarySource::MySQLDictionarySource( , update_field{config.getString(config_prefix + ".update_field", "")} , dont_check_update_time{config.getBool(config_prefix + ".dont_check_update_time", false)} , sample_block{sample_block_} - , pool{config, config_prefix} + , pool{mysqlxx::PoolFactory::instance().Get(config, config_prefix)} , query_builder{dict_struct, db, table, where, IdentifierQuotingStyle::Backticks} , load_all_query{query_builder.composeLoadAllQuery()} , invalidate_query{config.getString(config_prefix + ".invalidate_query", "")} diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp index e1cbd377978..1281ebd49af 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ b/dbms/src/Interpreters/ExternalDictionaries.cpp @@ -2,6 +2,10 @@ #include #include +#if USE_MYSQL +# include +#endif + namespace DB { @@ -27,4 +31,20 @@ ExternalLoader::LoadablePtr ExternalDictionaries::create( return DictionaryFactory::instance().create(name, config, key_in_config, context); } +void ExternalDictionaries::reload(const String & name, bool load_never_loading) +{ + #if USE_MYSQL + mysqlxx::PoolFactory::instance().reset(); + #endif + ExternalLoader::reload(name, load_never_loading); +} + +void ExternalDictionaries::reload(bool load_never_loading) +{ + #if USE_MYSQL + mysqlxx::PoolFactory::instance().reset(); + #endif + ExternalLoader::reload(load_never_loading); +} + } diff --git a/dbms/src/Interpreters/ExternalDictionaries.h b/dbms/src/Interpreters/ExternalDictionaries.h index c071349cc97..5265c83379b 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.h +++ b/dbms/src/Interpreters/ExternalDictionaries.h @@ -5,7 +5,6 @@ #include #include - namespace DB { @@ -33,6 +32,14 @@ public: return std::static_pointer_cast(tryGetLoadable(name)); } + /// Override ExternalLoader::reload to reset mysqlxx::PoolFactory.h + /// since connection parameters might have changed. Inherited method is called afterward + void reload(const String & name, bool load_never_loading = false); + + /// Override ExternalLoader::reload to reset mysqlxx::PoolFactory.h + /// since connection parameters might have changed. Inherited method is called afterward + void reload(bool load_never_loading = false); + protected: LoadablePtr create(const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & key_in_config) const override; diff --git a/libs/libmysqlxx/CMakeLists.txt b/libs/libmysqlxx/CMakeLists.txt index 263a031d7b0..4315c71fab3 100644 --- a/libs/libmysqlxx/CMakeLists.txt +++ b/libs/libmysqlxx/CMakeLists.txt @@ -8,6 +8,7 @@ add_library (mysqlxx src/Row.cpp src/Value.cpp src/Pool.cpp + src/PoolFactory.cpp src/PoolWithFailover.cpp include/mysqlxx/Connection.h @@ -15,6 +16,7 @@ add_library (mysqlxx include/mysqlxx/mysqlxx.h include/mysqlxx/Null.h include/mysqlxx/Pool.h + include/mysqlxx/PoolFactory.h include/mysqlxx/PoolWithFailover.h include/mysqlxx/Query.h include/mysqlxx/ResultBase.h diff --git a/libs/libmysqlxx/include/mysqlxx/PoolFactory.h b/libs/libmysqlxx/include/mysqlxx/PoolFactory.h new file mode 100644 index 00000000000..3c553b8b6da --- /dev/null +++ b/libs/libmysqlxx/include/mysqlxx/PoolFactory.h @@ -0,0 +1,51 @@ +#pragma once + +#include +#include +#include +#include "PoolWithFailover.h" + +#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS 1 +#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS 16 +#define MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES 3 + +namespace mysqlxx +{ +/* + * PoolFactory.h + * This class is a helper singleton to mutualize connections to MySQL. + */ +class PoolFactory final : private boost::noncopyable +{ +public: + static PoolFactory & instance(); + + PoolFactory(const PoolFactory &) = delete; + + /** Allocates a PoolWithFailover to connect to MySQL. */ + PoolWithFailover Get(const std::string & config_name, + unsigned default_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS, + unsigned max_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS, + size_t max_tries = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); + + /** Allocates a PoolWithFailover to connect to MySQL. */ + PoolWithFailover Get(const Poco::Util::AbstractConfiguration & config, + const std::string & config_name, + unsigned default_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_START_CONNECTIONS, + unsigned max_connections = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_CONNECTIONS, + size_t max_tries = MYSQLXX_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES); + + void reset(); + + + ~PoolFactory() = default; + PoolFactory& operator=(const PoolFactory &) = delete; + +private: + PoolFactory(); + + struct Impl; + std::unique_ptr impl; +}; + +} diff --git a/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h b/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h index 21b27ebd4fe..af59b705a61 100644 --- a/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h +++ b/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h @@ -77,6 +77,10 @@ namespace mysqlxx size_t max_tries; /// Mutex for set of replicas. std::mutex mutex; + std::string config_name; + + /// Can the Pool be shared + bool shareable; public: using Entry = Pool::Entry; diff --git a/libs/libmysqlxx/src/PoolFactory.cpp b/libs/libmysqlxx/src/PoolFactory.cpp new file mode 100644 index 00000000000..0e6244c333e --- /dev/null +++ b/libs/libmysqlxx/src/PoolFactory.cpp @@ -0,0 +1,136 @@ +#include +#include +#include + +namespace mysqlxx +{ + +struct PoolFactory::Impl +{ + // Cache of already affected pools identified by their config name + std::map> pools; + + // Cache of Pool ID (host + port + user +...) cibling already established shareable pool + std::map pools_by_ids; + + /// Protect pools and pools_by_ids caches + std::mutex mutex; +}; + +PoolWithFailover PoolFactory::Get(const std::string & config_name, unsigned default_connections, + unsigned max_connections, size_t max_tries) +{ + return Get(Poco::Util::Application::instance().config(), config_name, default_connections, max_connections, max_tries); +} + +/// Duplicate of code from StringUtils.h. Copied here for less dependencies. +static bool startsWith(const std::string & s, const char * prefix) +{ + return s.size() >= strlen(prefix) && 0 == memcmp(s.data(), prefix, strlen(prefix)); +} + +static std::string getPoolEntryName(const Poco::Util::AbstractConfiguration & config, + const std::string & config_name) +{ + bool shared = config.getBool(config_name + ".share_connection", false); + + // Not shared no need to generate a name the pool won't be stored + if (!shared) + return ""; + + std::string entry_name = ""; + std::string host = config.getString(config_name + ".host", ""); + std::string port = config.getString(config_name + ".port", ""); + std::string user = config.getString(config_name + ".user", ""); + std::string db = config.getString(config_name + ".db", ""); + std::string table = config.getString(config_name + ".table", ""); + + Poco::Util::AbstractConfiguration::Keys keys; + config.keys(config_name, keys); + + if (config.has(config_name + ".replica")) + { + Poco::Util::AbstractConfiguration::Keys replica_keys; + config.keys(config_name, replica_keys); + for (const auto & replica_config_key : replica_keys) + { + /// There could be another elements in the same level in configuration file, like "user", "port"... + if (startsWith(replica_config_key, "replica")) + { + std::string replica_name = config_name + "." + replica_config_key; + std::string tmp_host = config.getString(replica_name + ".host", host); + std::string tmp_port = config.getString(replica_name + ".port", port); + std::string tmp_user = config.getString(replica_name + ".user", user); + entry_name += (entry_name.empty() ? "" : "|") + tmp_user + "@" + tmp_host + ":" + tmp_port + "/" + db; + } + } + } + else + { + entry_name = user + "@" + host + ":" + port + "/" + db; + } + return entry_name; +} + +PoolWithFailover PoolFactory::Get(const Poco::Util::AbstractConfiguration & config, + const std::string & config_name, unsigned default_connections, unsigned max_connections, size_t max_tries) +{ + + std::lock_guard lock(impl->mutex); + Poco::Util::Application & app = Poco::Util::Application::instance(); + app.logger().warning("Config name=" + config_name); + if (auto entry = impl->pools.find(config_name); entry != impl->pools.end()) + { + app.logger().warning("Entry found=" + config_name); + return *(entry->second.get()); + } + else + { + app.logger().warning("Searching confg=" + config_name); + std::string entry_name = getPoolEntryName(config, config_name); + app.logger().warning("Entry name created=" + entry_name); + if (auto id = impl->pools_by_ids.find(entry_name); id != impl->pools_by_ids.end()) + { + app.logger().warning("found"); + entry = impl->pools.find(id->second); + std::shared_ptr pool = entry->second; + impl->pools.insert_or_assign(config_name, pool); + app.logger().warning("found OK"); + return *pool; + } + + app.logger().warning("make pool"); + auto pool = std::make_shared(config, config_name, default_connections, max_connections, max_tries); + app.logger().warning("make pool OK"); + // Check the pool will be shared + if (!entry_name.empty()) + { + // Store shared pool + app.logger().warning("store"); + impl->pools.insert_or_assign(config_name, pool); + impl->pools_by_ids.insert_or_assign(entry_name, config_name); + app.logger().warning("store OK"); + } + app.logger().warning("a2"); + auto a2 = *(pool.get()); + app.logger().warning("a2 OK"); + return *(pool.get()); + } +} + +void PoolFactory::reset() +{ + std::lock_guard lock(impl->mutex); + impl->pools.clear(); + impl->pools_by_ids.clear(); +} + +PoolFactory::PoolFactory() : impl(std::make_unique()) {} + +PoolFactory & PoolFactory::instance() +{ + static PoolFactory ret; + return ret; +} + +} diff --git a/libs/libmysqlxx/src/PoolWithFailover.cpp b/libs/libmysqlxx/src/PoolWithFailover.cpp index dd89f1596d3..bcdbcb3df72 100644 --- a/libs/libmysqlxx/src/PoolWithFailover.cpp +++ b/libs/libmysqlxx/src/PoolWithFailover.cpp @@ -48,15 +48,22 @@ PoolWithFailover::PoolWithFailover(const std::string & config_name, const unsign {} PoolWithFailover::PoolWithFailover(const PoolWithFailover & other) - : max_tries{other.max_tries} + : max_tries{other.max_tries}, config_name{other.config_name} { - for (const auto & priority_replicas : other.replicas_by_priority) + if (shareable) { - Replicas replicas; - replicas.reserve(priority_replicas.second.size()); - for (const auto & pool : priority_replicas.second) - replicas.emplace_back(std::make_shared(*pool)); - replicas_by_priority.emplace(priority_replicas.first, std::move(replicas)); + replicas_by_priority = other.replicas_by_priority; + } + else + { + for (const auto & priority_replicas : other.replicas_by_priority) + { + Replicas replicas; + replicas.reserve(priority_replicas.second.size()); + for (const auto & pool : priority_replicas.second) + replicas.emplace_back(std::make_shared(*pool)); + replicas_by_priority.emplace(priority_replicas.first, std::move(replicas)); + } } } From 1a9b7b97afe363ef492f0084d649d6691ae20127 Mon Sep 17 00:00:00 2001 From: Clement Rodriguez Date: Wed, 9 Oct 2019 15:47:09 +0200 Subject: [PATCH 0034/2007] Fixed PoolWithFailOver --- libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h b/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h index af59b705a61..d3cf5ae661a 100644 --- a/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h +++ b/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h @@ -104,7 +104,7 @@ namespace mysqlxx PoolWithFailover(const PoolWithFailover & other); - PoolWithFailover & operator=(const PoolWithFailover &) = delete; +// PoolWithFailover & operator=(const PoolWithFailover &) = delete; /** Allocates a connection to use. */ Entry Get(); From 72427b06836e2ff07a82016f8a8d69ce8f34f6bf Mon Sep 17 00:00:00 2001 From: Clement Rodriguez Date: Thu, 10 Oct 2019 18:10:46 +0200 Subject: [PATCH 0035/2007] Mutualization of MySQL connection + integration tests --- .../ExternalDictionariesLoader.cpp | 4 +- .../test_dictionaries_mysql/__init__.py | 0 .../configs/config.xml | 30 +++++ .../configs/dictionaries/mysql_dict1.xml | 39 ++++++ .../configs/dictionaries/mysql_dict2.xml | 113 ++++++++++++++++++ .../test_dictionaries_mysql/configs/users.xml | 23 ++++ .../test_dictionaries_mysql/test.py | 89 ++++++++++++++ .../include/mysqlxx/PoolWithFailover.h | 2 - libs/libmysqlxx/src/PoolFactory.cpp | 13 -- 9 files changed, 296 insertions(+), 17 deletions(-) create mode 100644 dbms/tests/integration/test_dictionaries_mysql/__init__.py create mode 100644 dbms/tests/integration/test_dictionaries_mysql/configs/config.xml create mode 100644 dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict1.xml create mode 100644 dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml create mode 100644 dbms/tests/integration/test_dictionaries_mysql/configs/users.xml create mode 100644 dbms/tests/integration/test_dictionaries_mysql/test.py diff --git a/dbms/src/Interpreters/ExternalDictionariesLoader.cpp b/dbms/src/Interpreters/ExternalDictionariesLoader.cpp index eb387b8842e..6bbe7c0d999 100644 --- a/dbms/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/dbms/src/Interpreters/ExternalDictionariesLoader.cpp @@ -27,7 +27,7 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( return DictionaryFactory::instance().create(name, config, key_in_config, context); } -void ExternalDictionaries::reload(const String & name, bool load_never_loading) +void ExternalDictionariesLoader::reload(const String & name, bool load_never_loading) { #if USE_MYSQL mysqlxx::PoolFactory::instance().reset(); @@ -35,7 +35,7 @@ void ExternalDictionaries::reload(const String & name, bool load_never_loading) ExternalLoader::reload(name, load_never_loading); } -void ExternalDictionaries::reload(bool load_never_loading) +void ExternalDictionariesLoader::reload(bool load_never_loading) { #if USE_MYSQL mysqlxx::PoolFactory::instance().reset(); diff --git a/dbms/tests/integration/test_dictionaries_mysql/__init__.py b/dbms/tests/integration/test_dictionaries_mysql/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/integration/test_dictionaries_mysql/configs/config.xml b/dbms/tests/integration/test_dictionaries_mysql/configs/config.xml new file mode 100644 index 00000000000..b60daf72dcf --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_mysql/configs/config.xml @@ -0,0 +1,30 @@ + + + + trace + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 1000M + 10 + + + 9000 + 127.0.0.1 + + + + true + none + + AcceptCertificateHandler + + + + + 500 + 5368709120 + ./clickhouse/ + users.xml + + /etc/clickhouse-server/config.d/*.xml + diff --git a/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict1.xml b/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict1.xml new file mode 100644 index 00000000000..0a3a613dfdc --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict1.xml @@ -0,0 +1,39 @@ + + + + dict1 + + + test + mysql1 + 3306 + root + clickhouse + test1
+ true + true + + + + + + + + id + UInt32 + CAST(id AS UNSIGNED) + + + id + Int32 + + + + value + String + (UNDEFINED) + + + 0 + + diff --git a/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml b/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml new file mode 100644 index 00000000000..024d13b14b2 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml @@ -0,0 +1,113 @@ + + + + dict2 + + + test + mysql1 + 3306 + root + clickhouse + test2
+ true + true + + + + + + + + id + UInt32 + CAST(id AS UNSIGNED) + + + id + Int32 + + + + value + String + (UNDEFINED) + + + 0 + + + + dict3 + + + test + mysql1 + 3306 + root + clickhouse + test2
+ true + true + + + + + + + + id + UInt32 + CAST(id AS UNSIGNED) + + + id + Int32 + + + + value + String + (UNDEFINED) + + + 0 + + + dict4 + + + test + mysql1 + 3306 + root + clickhouse + test2
+ true + true + + + + + + + + id + UInt32 + CAST(id AS UNSIGNED) + + + id + Int32 + + + + value + String + (UNDEFINED) + + + 0 + + + diff --git a/dbms/tests/integration/test_dictionaries_mysql/configs/users.xml b/dbms/tests/integration/test_dictionaries_mysql/configs/users.xml new file mode 100644 index 00000000000..6061af8e33d --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_mysql/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + ::/0 + + default + default + + + + + + + + diff --git a/dbms/tests/integration/test_dictionaries_mysql/test.py b/dbms/tests/integration/test_dictionaries_mysql/test.py new file mode 100644 index 00000000000..52f82cc9b39 --- /dev/null +++ b/dbms/tests/integration/test_dictionaries_mysql/test.py @@ -0,0 +1,89 @@ +import pytest +import os +import time + +## sudo -H pip install PyMySQL +import pymysql.cursors + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import assert_eq_with_retry + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +DICTIONARY_FILES = ['configs/dictionaries/mysql_dict1.xml', 'configs/dictionaries/mysql_dict2.xml'] + +cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) +instance = cluster.add_instance('instance', main_configs=DICTIONARY_FILES) + +create_table_mysql_template = """ + CREATE TABLE `test`.`{}` ( + `id` int(11) NOT NULL, + `value` varchar(50) NOT NULL, + PRIMARY KEY (`id`) + ) ENGINE=InnoDB; + """ + +create_clickhouse_dictionary_table_template = """ + CREATE TABLE `test`.`{}` (`id` Int32, `value` String) ENGINE = Dictionary({}) + ORDER BY `id` DESC SETTINGS index_granularity = 8192 + """ + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + instance.query("CREATE DATABASE IF NOT EXISTS test") + + # Create a MySQL database + create_mysql_db(get_mysql_conn(), 'test') + instance.query("CREATE DATABASE clickhouse_mysql ENGINE = MySQL('mysql1:3306', 'test', 'root', 'clickhouse')") + + yield cluster + + finally: + cluster.shutdown() + + +def test_load_mysql_dictionaries(started_cluster): + # Load dictionaries + query = instance.query + query("SYSTEM RELOAD DICTIONARIES") + + for n in range(0, 5): + # Create MySQL tables and fills them + prepare_mysql_table('test' + n) + + #Create Dictionary tables based on MySQL tables + query(create_clickhouse_dictionary_table_template.format('test' + n), 'dict' + n) + + # Check dictionaries are loaded and have correct number of elements + for n in range(0, 100): + if (n % 10) == 0: + # Force reload of dictionaries + query("SYSTEM RELOAD DICTIONARIES") + assert query("SELECT count() FROM `test`.{}".format('test' + (n % 5))).rstrip() == '10000' + +def create_mysql_db(mysql_connection, name): + with mysql_connection.cursor() as cursor: + cursor.execute( + "CREATE DATABASE {} DEFAULT CHARACTER SET 'utf8'".format(name)) + +def prepare_mysql_table(table_name): + mysql_connection = get_mysql_conn() + + # Create table + create_mysql_table(mysql_connection, table_name) + + # Insert rows using CH + query = instance.query + query("INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(table_name, table_name)) + assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name)).rstrip() == '10000' + mysql_connection.close() + +def get_mysql_conn(): + conn = pymysql.connect(user='root', password='clickhouse', host='mysql1', port=3308) + return conn + +def create_mysql_table(conn, table_name): + with conn.cursor() as cursor: + cursor.execute(create_table_mysql_template.format(table_name)) + diff --git a/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h b/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h index d3cf5ae661a..fe151240fa5 100644 --- a/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h +++ b/libs/libmysqlxx/include/mysqlxx/PoolWithFailover.h @@ -104,8 +104,6 @@ namespace mysqlxx PoolWithFailover(const PoolWithFailover & other); -// PoolWithFailover & operator=(const PoolWithFailover &) = delete; - /** Allocates a connection to use. */ Entry Get(); }; diff --git a/libs/libmysqlxx/src/PoolFactory.cpp b/libs/libmysqlxx/src/PoolFactory.cpp index 0e6244c333e..e721fe5027d 100644 --- a/libs/libmysqlxx/src/PoolFactory.cpp +++ b/libs/libmysqlxx/src/PoolFactory.cpp @@ -77,43 +77,30 @@ PoolWithFailover PoolFactory::Get(const Poco::Util::AbstractConfiguration & conf { std::lock_guard lock(impl->mutex); - Poco::Util::Application & app = Poco::Util::Application::instance(); - app.logger().warning("Config name=" + config_name); if (auto entry = impl->pools.find(config_name); entry != impl->pools.end()) { - app.logger().warning("Entry found=" + config_name); return *(entry->second.get()); } else { - app.logger().warning("Searching confg=" + config_name); std::string entry_name = getPoolEntryName(config, config_name); - app.logger().warning("Entry name created=" + entry_name); if (auto id = impl->pools_by_ids.find(entry_name); id != impl->pools_by_ids.end()) { - app.logger().warning("found"); entry = impl->pools.find(id->second); std::shared_ptr pool = entry->second; impl->pools.insert_or_assign(config_name, pool); - app.logger().warning("found OK"); return *pool; } - app.logger().warning("make pool"); auto pool = std::make_shared(config, config_name, default_connections, max_connections, max_tries); - app.logger().warning("make pool OK"); // Check the pool will be shared if (!entry_name.empty()) { // Store shared pool - app.logger().warning("store"); impl->pools.insert_or_assign(config_name, pool); impl->pools_by_ids.insert_or_assign(entry_name, config_name); - app.logger().warning("store OK"); } - app.logger().warning("a2"); auto a2 = *(pool.get()); - app.logger().warning("a2 OK"); return *(pool.get()); } } From b433add65cb84b6c30a0bad559b71ad3c557e1e4 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 10 Oct 2019 19:30:30 +0300 Subject: [PATCH 0036/2007] polymorphic parts (development) --- dbms/src/DataStreams/TTLBlockInputStream.cpp | 4 +- dbms/src/DataStreams/TTLBlockInputStream.h | 6 +- dbms/src/Interpreters/PartLog.cpp | 2 +- dbms/src/Interpreters/PartLog.h | 4 +- .../Storages/MergeTree/DataPartsExchange.cpp | 4 +- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 461 +++++++++++++++++ ...rgeTreeDataPart.h => IMergeTreeDataPart.h} | 237 +++++---- .../Storages/MergeTree/IMergeTreeReader.cpp | 191 +++++++ .../{MergeTreeReader.h => IMergeTreeReader.h} | 45 +- .../MergeTreeBaseSelectBlockInputStream.cpp | 11 +- .../MergeTreeBaseSelectBlockInputStream.h | 12 +- .../MergeTree/MergeTreeBlockReadUtils.h | 1 + dbms/src/Storages/MergeTree/MergeTreeData.cpp | 36 +- dbms/src/Storages/MergeTree/MergeTreeData.h | 13 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 21 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 0 .../MergeTree/MergeTreeDataPartCompact.h | 114 +++++ .../MergeTree/MergeTreeDataPartFactory.cpp | 16 + .../MergeTree/MergeTreeDataPartFactory.h | 9 + .../MergeTree/MergeTreeDataPartOnDisk.cpp | 0 .../MergeTree/MergeTreeDataPartOnDisk.h | 33 ++ ...DataPart.cpp => MergeTreeDataPartWide.cpp} | 482 +++--------------- .../MergeTree/MergeTreeDataPartWide.h | 116 +++++ .../MergeTree/MergeTreeDataSelectExecutor.cpp | 41 +- .../MergeTree/MergeTreeDataSelectExecutor.h | 9 +- .../MergeTree/MergeTreeDataWriter.cpp | 8 +- .../MergeTree/MergeTreeIndexGranularity.h | 2 +- .../MergeTree/MergeTreeIndexReader.cpp | 5 +- .../Storages/MergeTree/MergeTreePartition.cpp | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 9 +- .../Storages/MergeTree/MergeTreePartsMover.h | 14 +- .../MergeTree/MergeTreeRangeReader.cpp | 8 +- .../Storages/MergeTree/MergeTreeRangeReader.h | 14 +- .../Storages/MergeTree/MergeTreeReadPool.cpp | 4 +- .../Storages/MergeTree/MergeTreeReadPool.h | 2 +- .../MergeTree/MergeTreeReaderCompact.cpp | 0 .../MergeTree/MergeTreeReaderCompact.h | 42 ++ .../MergeTree/MergeTreeReaderSettings.h | 13 + .../MergeTree/MergeTreeReaderStream.cpp | 16 +- .../MergeTree/MergeTreeReaderStream.h | 7 +- ...TreeReader.cpp => MergeTreeReaderWide.cpp} | 179 +------ .../Storages/MergeTree/MergeTreeReaderWide.h | 49 ++ ...MergeTreeReverseSelectBlockInputStream.cpp | 23 +- .../MergeTreeReverseSelectBlockInputStream.h | 4 +- .../MergeTreeSelectBlockInputStream.cpp | 22 +- .../MergeTreeSelectBlockInputStream.h | 4 +- .../MergeTreeSequentialBlockInputStream.cpp | 16 +- .../MergeTreeSequentialBlockInputStream.h | 4 +- .../MergeTreeThreadSelectBlockInputStream.cpp | 34 +- .../MergeTreeThreadSelectBlockInputStream.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeWriter.h | 13 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 2 +- .../ReplicatedMergeTreeQuorumAddedParts.h | 2 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 10 +- .../Storages/StorageReplicatedMergeTree.cpp | 6 +- .../src/Storages/StorageReplicatedMergeTree.h | 2 +- .../Storages/System/StorageSystemParts.cpp | 4 +- .../System/StorageSystemPartsColumns.cpp | 2 +- 58 files changed, 1498 insertions(+), 894 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp rename dbms/src/Storages/MergeTree/{MergeTreeDataPart.h => IMergeTreeDataPart.h} (70%) create mode 100644 dbms/src/Storages/MergeTree/IMergeTreeReader.cpp rename dbms/src/Storages/MergeTree/{MergeTreeReader.h => IMergeTreeReader.h} (67%) create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h rename dbms/src/Storages/MergeTree/{MergeTreeDataPart.cpp => MergeTreeDataPartWide.cpp} (60%) create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h rename dbms/src/Storages/MergeTree/{MergeTreeReader.cpp => MergeTreeReaderWide.cpp} (54%) create mode 100644 dbms/src/Storages/MergeTree/MergeTreeReaderWide.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeWriter.h diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index e98ce4eb1b7..3e92555f890 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -36,7 +36,7 @@ TTLBlockInputStream::TTLBlockInputStream( { if (force || isTTLExpired(ttl_info.min)) { - new_ttl_infos.columns_ttl.emplace(name, MergeTreeDataPart::TTLInfo{}); + new_ttl_infos.columns_ttl.emplace(name,IMergeTreeDataPart::TTLInfo{}); empty_columns.emplace(name); auto it = column_defaults.find(name); @@ -96,7 +96,7 @@ void TTLBlockInputStream::readSuffixImpl() new_ttl_infos.updatePartMinMaxTTL(new_ttl_infos.table_ttl.min, new_ttl_infos.table_ttl.max); data_part->ttl_infos = std::move(new_ttl_infos); - data_part->empty_columns = std::move(empty_columns); + data_part->expired_columns = std::move(empty_columns); if (rows_removed) LOG_INFO(log, "Removed " << rows_removed << " rows with expired TTL from part " << data_part->name); diff --git a/dbms/src/DataStreams/TTLBlockInputStream.h b/dbms/src/DataStreams/TTLBlockInputStream.h index 5ed6aa9e520..067118279cb 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.h +++ b/dbms/src/DataStreams/TTLBlockInputStream.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include +#include #include #include @@ -39,8 +39,8 @@ private: time_t current_time; bool force; - MergeTreeDataPart::TTLInfos old_ttl_infos; - MergeTreeDataPart::TTLInfos new_ttl_infos; + IMergeTreeDataPart::TTLInfos old_ttl_infos; + IMergeTreeDataPart::TTLInfos new_ttl_infos; NameSet empty_columns; size_t rows_removed = 0; diff --git a/dbms/src/Interpreters/PartLog.cpp b/dbms/src/Interpreters/PartLog.cpp index c860f3212c7..0f8904bab2e 100644 --- a/dbms/src/Interpreters/PartLog.cpp +++ b/dbms/src/Interpreters/PartLog.cpp @@ -7,7 +7,7 @@ #include #include #include -#include +#include #include #include diff --git a/dbms/src/Interpreters/PartLog.h b/dbms/src/Interpreters/PartLog.h index bb11338f411..4c4930ccefa 100644 --- a/dbms/src/Interpreters/PartLog.h +++ b/dbms/src/Interpreters/PartLog.h @@ -51,7 +51,7 @@ struct PartLogElement void appendToBlock(Block & block) const; }; -struct MergeTreeDataPart; +class IMergeTreeDataPart; /// Instead of typedef - to allow forward declaration. @@ -59,7 +59,7 @@ class PartLog : public SystemLog { using SystemLog::SystemLog; - using MutableDataPartPtr = std::shared_ptr; + using MutableDataPartPtr = std::shared_ptr; using MutableDataPartsVector = std::vector; public: diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 073f33441ac..6cff1dfe9f5 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -268,8 +269,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( part_file.createDirectory(); - MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared(data, reservation->getDisk(), part_name); - new_data_part->relative_path = relative_part_path; + MergeTreeData::MutableDataPartPtr new_data_part = createPart(data, reservation->getDisk(), part_name, relative_part_path); new_data_part->is_temp = true; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp new file mode 100644 index 00000000000..ea6182c2fae --- /dev/null +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -0,0 +1,461 @@ +#include "IMergeTreeDataPart.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int NO_FILE_IN_DATA_PART; + extern const int EXPECTED_END_OF_FILE; + extern const int CORRUPTED_DATA; + extern const int NOT_FOUND_EXPECTED_DATA_PART; + extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; + extern const int BAD_TTL_FILE; + extern const int CANNOT_UNLINK; +} + + +static ReadBufferFromFile openForReading(const String & path) +{ + return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); +} + +void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const String & part_path) +{ + size_t minmax_idx_size = data.minmax_idx_column_types.size(); + parallelogram.reserve(minmax_idx_size); + for (size_t i = 0; i < minmax_idx_size; ++i) + { + String file_name = part_path + "minmax_" + escapeForFileName(data.minmax_idx_columns[i]) + ".idx"; + ReadBufferFromFile file = openForReading(file_name); + const DataTypePtr & type = data.minmax_idx_column_types[i]; + + Field min_val; + type->deserializeBinary(min_val, file); + Field max_val; + type->deserializeBinary(max_val, file); + + parallelogram.emplace_back(min_val, true, max_val, true); + } + initialized = true; +} + +void IMergeTreeDataPart::MinMaxIndex::store(const MergeTreeData & data, const String & part_path, Checksums & out_checksums) const +{ + store(data.minmax_idx_columns, data.minmax_idx_column_types, part_path, out_checksums); +} + +void IMergeTreeDataPart::MinMaxIndex::store(const Names & column_names, const DataTypes & data_types, const String & part_path, Checksums & out_checksums) const +{ + if (!initialized) + throw Exception("Attempt to store uninitialized MinMax index for part " + part_path + ". This is a bug.", + ErrorCodes::LOGICAL_ERROR); + + for (size_t i = 0; i < column_names.size(); ++i) + { + String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; + const DataTypePtr & type = data_types.at(i); + + WriteBufferFromFile out(part_path + file_name); + HashingWriteBuffer out_hashing(out); + type->serializeBinary(parallelogram[i].left, out_hashing); + type->serializeBinary(parallelogram[i].right, out_hashing); + out_hashing.next(); + out_checksums.files[file_name].file_size = out_hashing.count(); + out_checksums.files[file_name].file_hash = out_hashing.getHash(); + } +} + +void IMergeTreeDataPart::MinMaxIndex::update(const Block & block, const Names & column_names) +{ + if (!initialized) + parallelogram.reserve(column_names.size()); + + for (size_t i = 0; i < column_names.size(); ++i) + { + Field min_value; + Field max_value; + const ColumnWithTypeAndName & column = block.getByName(column_names[i]); + column.column->getExtremes(min_value, max_value); + + if (!initialized) + parallelogram.emplace_back(min_value, true, max_value, true); + else + { + parallelogram[i].left = std::min(parallelogram[i].left, min_value); + parallelogram[i].right = std::max(parallelogram[i].right, max_value); + } + } + + initialized = true; +} + +void IMergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other) +{ + if (!other.initialized) + return; + + if (!initialized) + { + parallelogram = other.parallelogram; + initialized = true; + } + else + { + for (size_t i = 0; i < parallelogram.size(); ++i) + { + parallelogram[i].left = std::min(parallelogram[i].left, other.parallelogram[i].left); + parallelogram[i].right = std::max(parallelogram[i].right, other.parallelogram[i].right); + } + } +} + + +IMergeTreeDataPart::IMergeTreeDataPart( + MergeTreeData & storage_, + const String & name_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_) + : storage(storage_) + , name(name_) + , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) + , disk(disk_) + , relative_path(relative_path_.value_or(name_)) + , index_granularity_info(storage) {} + +IMergeTreeDataPart::IMergeTreeDataPart( + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_) + : storage(storage_) + , name(name_) + , info(info_) + , disk(disk_) + , relative_path(relative_path_.value_or(name_)) + , index_granularity_info(storage) {} + + +ColumnSize IMergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const +{ + return getColumnSizeImpl(column_name, type, nullptr); +} + +ColumnSize IMergeTreeDataPart::getTotalColumnsSize() const +{ + ColumnSize totals; + std::unordered_set processed_substreams; + for (const NameAndTypePair & column : columns) + { + ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams); + totals.add(size); + } + return totals; +} + + +String IMergeTreeDataPart::getNewName(const MergeTreePartInfo & new_part_info) const +{ + if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + /// NOTE: getting min and max dates from the part name (instead of part data) because we want + /// the merged part name be determined only by source part names. + /// It is simpler this way when the real min and max dates for the block range can change + /// (e.g. after an ALTER DELETE command). + DayNum min_date; + DayNum max_date; + MergeTreePartInfo::parseMinMaxDatesFromPartName(name, min_date, max_date); + return new_part_info.getPartNameV0(min_date, max_date); + } + else + return new_part_info.getPartName(); +} + +DayNum IMergeTreeDataPart::getMinDate() const +{ + if (storage.minmax_idx_date_column_pos != -1 && minmax_idx.initialized) + return DayNum(minmax_idx.parallelogram[storage.minmax_idx_date_column_pos].left.get()); + else + return DayNum(); +} + + +DayNum IMergeTreeDataPart::getMaxDate() const +{ + if (storage.minmax_idx_date_column_pos != -1 && minmax_idx.initialized) + return DayNum(minmax_idx.parallelogram[storage.minmax_idx_date_column_pos].right.get()); + else + return DayNum(); +} + +time_t IMergeTreeDataPart::getMinTime() const +{ + if (storage.minmax_idx_time_column_pos != -1 && minmax_idx.initialized) + return minmax_idx.parallelogram[storage.minmax_idx_time_column_pos].left.get(); + else + return 0; +} + + +time_t IMergeTreeDataPart::getMaxTime() const +{ + if (storage.minmax_idx_time_column_pos != -1 && minmax_idx.initialized) + return minmax_idx.parallelogram[storage.minmax_idx_time_column_pos].right.get(); + else + return 0; +} + +IMergeTreeDataPart::~IMergeTreeDataPart() +{ + // if (on_disk && (state == State::DeleteOnDestroy || is_temp)) + // { + // try + // { + // std::string path = on_disk->getFullPath(); + + // Poco::File dir(path); + // if (!dir.exists()) + // return; + + // if (is_temp) + // { + // if (!startsWith(on_disk->getNameWithPrefix(), "tmp")) + // { + // LOG_ERROR(storage.log, "~DataPart() should remove part " << path + // << " but its name doesn't start with tmp. Too suspicious, keeping the part."); + // return; + // } + // } + + // dir.remove(true); + // } + // catch (...) + // { + // tryLogCurrentException(__PRETTY_FUNCTION__); + // } + // } +} + + +UInt64 IMergeTreeDataPart::getIndexSizeInBytes() const +{ + UInt64 res = 0; + for (const ColumnPtr & column : index) + res += column->byteSize(); + return res; +} + +UInt64 IMergeTreeDataPart::getIndexSizeInAllocatedBytes() const +{ + UInt64 res = 0; + for (const ColumnPtr & column : index) + res += column->allocatedBytes(); + return res; +} + +String IMergeTreeDataPart::stateToString(IMergeTreeDataPart::State state) +{ + switch (state) + { + case State::Temporary: + return "Temporary"; + case State::PreCommitted: + return "PreCommitted"; + case State::Committed: + return "Committed"; + case State::Outdated: + return "Outdated"; + case State::Deleting: + return "Deleting"; + case State::DeleteOnDestroy: + return "DeleteOnDestroy"; + } + + __builtin_unreachable(); +} + +String IMergeTreeDataPart::stateString() const +{ + return stateToString(state); +} + + + +void IMergeTreeDataPart::assertState(const std::initializer_list & affordable_states) const +{ + if (!checkState(affordable_states)) + { + String states_str; + for (auto affordable_state : affordable_states) + states_str += stateToString(affordable_state) + " "; + + throw Exception("Unexpected state of part " + getNameWithState() + ". Expected: " + states_str, ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART); + } +} + +void IMergeTreeDataPart::assertOnDisk() const +{ + if (!isStoredOnDisk()) + throw Exception("Data part '" + name + "is not stored on disk", ErrorCodes::LOGICAL_ERROR); +} + + +UInt64 IMergeTreeDataPart::getMarksCount() const +{ + return index_granularity.getMarksCount(); +} + +size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const +{ + auto checksum = checksums.files.find(file_name); + if (checksum == checksums.files.end()) + return 0; + return checksum->second.file_size; +} + +String IMergeTreeDataPart::getFullPath() const +{ + assertOnDisk(); + + if (relative_path.empty()) + throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); + + return storage.getFullPathOnDisk(disk) + relative_path + "/"; +} + + +UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) +{ + Poco::File cur(from); + if (cur.isFile()) + return cur.getSize(); + std::vector files; + cur.list(files); + UInt64 res = 0; + for (const auto & file : files) + res += calculateTotalSizeOnDisk(from + file); + return res; +} + + +void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_new_dir_if_exists) const +{ + String from = getFullPath(); + String to = storage.getFullPathOnDisk(disk) + new_relative_path + "/"; + + Poco::File from_file(from); + if (!from_file.exists()) + throw Exception("Part directory " + from + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST); + + Poco::File to_file(to); + if (to_file.exists()) + { + if (remove_new_dir_if_exists) + { + Names files; + Poco::File(from).list(files); + + LOG_WARNING(storage.log, "Part directory " << to << " already exists" + << " and contains " << files.size() << " files. Removing it."); + + to_file.remove(true); + } + else + { + throw Exception("Part directory " + to + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + } + } + + from_file.setLastModified(Poco::Timestamp::fromEpochTime(time(nullptr))); + from_file.renameTo(to); + relative_path = new_relative_path; +} + + +String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) const +{ + /// Do not allow underscores in the prefix because they are used as separators. + + assert(prefix.find_first_of('_') == String::npos); + String res; + + /** If you need to detach a part, and directory into which we want to rename it already exists, + * we will rename to the directory with the name to which the suffix is added in the form of "_tryN". + * This is done only in the case of `to_detached`, because it is assumed that in this case the exact name does not matter. + * No more than 10 attempts are made so that there are not too many junk directories left. + */ + for (int try_no = 0; try_no < 10; try_no++) + { + res = "detached/" + (prefix.empty() ? "" : prefix + "_") + + name + (try_no ? "_try" + DB::toString(try_no) : ""); + + if (!Poco::File(storage.getFullPathOnDisk(disk) + res).exists()) + return res; + + LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." + " Will detach to directory with '_tryN' suffix."); + } + + return res; +} + +void IMergeTreeDataPart::renameToDetached(const String & prefix) const +{ + assertOnDisk(); + renameTo(getRelativePathForDetachedPart(prefix)); +} + +void IMergeTreeDataPart::makeCloneInDetached(const String & prefix) const +{ + assertOnDisk(); + LOG_INFO(storage.log, "Detaching " << relative_path); + + Poco::Path src(getFullPath()); + Poco::Path dst(storage.getFullPathOnDisk(disk) + getRelativePathForDetachedPart(prefix)); + /// Backup is not recursive (max_level is 0), so do not copy inner directories + localBackup(src, dst, 0); +} + +void IMergeTreeDataPart::makeCloneOnDiskDetached(const DiskSpace::ReservationPtr & reservation) const +{ + assertOnDisk(); + + auto & reserved_disk = reservation->getDisk(); + if (reserved_disk->getName() == disk->getName()) + throw Exception("Can not clone data part " + name + " to same disk " + disk->getName(), ErrorCodes::LOGICAL_ERROR); + + String path_to_clone = storage.getFullPathOnDisk(reserved_disk) + "detached/"; + + if (Poco::File(path_to_clone + relative_path).exists()) + throw Exception("Path " + path_to_clone + relative_path + " already exists. Can not clone ", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + Poco::File(path_to_clone).createDirectory(); + + Poco::File cloning_directory(getFullPath()); + cloning_directory.copyTo(path_to_clone); +} + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h similarity index 70% rename from dbms/src/Storages/MergeTree/MergeTreeDataPart.h rename to dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index a953c400851..f30d72f45de 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -1,9 +1,12 @@ #pragma once +#include + #include #include #include #include +#include #include #include #include @@ -12,54 +15,146 @@ #include #include #include +// #include +// #include +// #include #include +#include + #include #include - namespace DB { -struct ColumnSize; -class MergeTreeData; + struct ColumnSize; + class MergeTreeData; + class IMergeTreeReader; - -/// Description of the data part. -struct MergeTreeDataPart +namespace ErrorCodes { + extern const int NOT_IMPLEMETED; +} + +// class MergeTreeDataPartOnDisk; + +class IMergeTreeDataPart : public std::enable_shared_from_this +{ +public: + using Checksums = MergeTreeDataPartChecksums; using Checksum = MergeTreeDataPartChecksums::Checksum; + using MergeTreeReaderPtr = std::unique_ptr; + using ValueSizeMap = std::map; - MergeTreeDataPart(const MergeTreeData & storage_, const DiskSpace::DiskPtr & disk_, const String & name_, const MergeTreePartInfo & info_); + // virtual BlockInputStreamPtr readAll() = 0; + // virtual BlockInputStreamPtr read() = 0; + // virtual BlockInputStreamPtr readWithThreadPool() = 0; + // virtual BlockInputStreamPtr readReverse() = 0; - MergeTreeDataPart(MergeTreeData & storage_, const DiskSpace::DiskPtr & disk_, const String & name_); + virtual MergeTreeReaderPtr getReader( + const NamesAndTypesList & columns_, + const MarkRanges & mark_ranges, + UncompressedCache * uncompressed_cache, + MarkCache * mark_cache, + const ReaderSettings & reader_settings_, + const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}) const = 0; + + // virtual MergeTreeWriterPtr getWriter() const = 0; - /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). - /// If no checksums are present returns the name of the first physically existing column. - String getColumnNameWithMinumumCompressedSize() const; + virtual bool isStoredOnDisk() const = 0; + + virtual void remove() const = 0; + + virtual bool supportsVerticalMerge() const { return false; } /// NOTE: Returns zeros if column files are not found in checksums. /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. /// (either by locking columns_lock, or by locking table structure). - ColumnSize getColumnSize(const String & name, const IDataType & type) const; + virtual ColumnSize getColumnSize(const String & name, const IDataType & type) const = 0; + + /// Initialize columns (from columns.txt if exists, or create from column files if not). + /// Load checksums from checksums.txt if exists. Load index if required. + virtual void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) = 0; + + /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). + /// If no checksums are present returns the name of the first physically existing column. + virtual String getColumnNameWithMinumumCompressedSize() const = 0; + + // virtual void detach() = 0; + + // virtual Checksums check( + // bool require_checksums, + // const DataTypes & primary_key_data_types, /// Check the primary key. If it is not necessary, pass an empty array. + // const MergeTreeIndices & indices = {}, /// Check skip indices + // std::function is_cancelled = []{ return false; }) + // { + // return {}; + // } + + using ColumnToSize = std::map; + + + // void accumulateColumnSizes(ColumnToSize & column_to_size) const + // { + // throw Exception("Method 'accumulateColumnSizes' is not supported for data part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMETED); + // } + + enum class Type + { + WIDE, + STRIPED, + IN_MEMORY, + }; + + virtual Type getType() const = 0; + + // virtual void renameTo() = 0; + + static String typeToString(Type type) + { + switch(type) + { + case Type::WIDE: + return "Wide"; + case Type::STRIPED: + return "Striped"; + case Type::IN_MEMORY: + return "InMemory"; + } + + __builtin_unreachable(); + } + + String getTypeName() { return typeToString(getType()); } + + virtual ~IMergeTreeDataPart(); + + IMergeTreeDataPart( + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const DiskSpace::DiskPtr & disk = {}, + const std::optional & relative_path = {}); + + IMergeTreeDataPart( + MergeTreeData & storage_, + const String & name_, + const DiskSpace::DiskPtr & disk = {}, + const std::optional & relative_path = {}); + + void assertOnDisk() const; ColumnSize getTotalColumnsSize() const; - size_t getFileSizeOrZero(const String & file_name) const; - - /// Returns full path to part dir - String getFullPath() const; - - /// Returns part->name with prefixes like 'tmp_' - String getNameWithPrefix() const; - /// Generate the new name for this part according to `new_part_info` and min/max dates from the old name. /// This is useful when you want to change e.g. block numbers or the mutation version of the part. String getNewName(const MergeTreePartInfo & new_part_info) const; - bool contains(const MergeTreeDataPart & other) const { return info.contains(other.info); } + bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } /// If the partition key includes date column (a common case), these functions will return min and max values for this column. DayNum getMinDate() const; @@ -73,18 +168,19 @@ struct MergeTreeDataPart const MergeTreeData & storage; - DiskSpace::DiskPtr disk; String name; MergeTreePartInfo info; - /// A directory path (relative to storage's path) where part data is actually stored - /// Examples: 'detached/tmp_fetch_', 'tmp_', '' + DiskSpace::DiskPtr disk; + mutable String relative_path; size_t rows_count = 0; + std::atomic bytes_on_disk {0}; /// 0 - if not counted; /// Is used from several threads without locks (it is changed with ALTER). /// May not contain size of checksums.txt and columns.txt + time_t modification_time = 0; /// When the part is removed from the working set. Changes once. mutable std::atomic remove_time { std::numeric_limits::max() }; @@ -152,24 +248,6 @@ struct MergeTreeDataPart /// Throws an exception if state of the part is not in affordable_states void assertState(const std::initializer_list & affordable_states) const; - /// In comparison with lambdas, it is move assignable and could has several overloaded operator() - struct StatesFilter - { - std::initializer_list affordable_states; - StatesFilter(const std::initializer_list & affordable_states_) : affordable_states(affordable_states_) {} - - bool operator() (const std::shared_ptr & part) const - { - return part->checkState(affordable_states); - } - }; - - /// Returns a lambda that returns true only for part with states from specified list - static inline StatesFilter getStatesFilter(const std::initializer_list & affordable_states) - { - return StatesFilter(affordable_states); - } - /// Primary key (correspond to primary.idx file). /// Always loaded in RAM. Contains each index_granularity-th value of primary key tuple. /// Note that marks (also correspond to primary key) is not always in RAM, but cached. See MarkCache.h. @@ -218,9 +296,7 @@ struct MergeTreeDataPart NamesAndTypesList columns; /// Columns with values, that all have been zeroed by expired ttl - NameSet empty_columns; - - using ColumnToSize = std::map; + NameSet expired_columns; /** It is blocked for writing when changing columns, checksums or any part files. * Locked to read when reading columns, checksums or any part files. @@ -239,72 +315,31 @@ struct MergeTreeDataPart MergeTreeIndexGranularityInfo index_granularity_info; - ~MergeTreeDataPart(); - - /// Calculate the total size of the entire directory with all the files - static UInt64 calculateTotalSizeOnDisk(const String & from); - - void remove() const; - - /// Makes checks and move part to new directory - /// Changes only relative_dir_name, you need to update other metadata (name, is_temp) explicitly - void renameTo(const String & new_relative_path, bool remove_new_dir_if_exists = true) const; - - /// Generate unique path to detach part - String getRelativePathForDetachedPart(const String & prefix) const; - - /// Moves a part to detached/ directory and adds prefix to its name - void renameToDetached(const String & prefix) const; - - /// Makes clone of a part in detached/ directory via hard links - void makeCloneInDetached(const String & prefix) const; - - /// Makes full clone of part in detached/ on another disk - void makeCloneOnDiskDetached(const DiskSpace::ReservationPtr & reservation) const; - - /// Populates columns_to_size map (compressed size). - void accumulateColumnSizes(ColumnToSize & column_to_size) const; - - /// Initialize columns (from columns.txt if exists, or create from column files if not). - /// Load checksums from checksums.txt if exists. Load index if required. - void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency); - - /// Checks that .bin and .mrk files exist - bool hasColumnFiles(const String & column, const IDataType & type) const; - /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; UInt64 getMarksCount() const; + size_t getFileSizeOrZero(const String & file_name) const; + String getFullPath() const; + void renameTo(const String & new_relative_path, bool remove_new_dir_if_exists = false) const; + void renameToDetached(const String & prefix) const; + void makeCloneInDetached(const String & prefix) const; + void makeCloneOnDiskDetached(const DiskSpace::ReservationPtr & reservation) const; + + /// Checks that .bin and .mrk files exist + bool hasColumnFiles(const String & /* column */, const IDataType & /* type */ ) const { return true; } + + static UInt64 calculateTotalSizeOnDisk(const String & from); + private: - /// Reads columns names and types from columns.txt - void loadColumns(bool require); - - /// If checksums.txt exists, reads files' checksums (and sizes) from it - void loadChecksums(bool require); - - /// Loads marks index granularity into memory - void loadIndexGranularity(); - - /// Loads index file. - void loadIndex(); - - /// Load rows count for this part from disk (for the newer storage format version). - /// For the older format version calculates rows count from the size of a column with a fixed size. - void loadRowsCount(); - - /// Loads ttl infos in json format from file ttl.txt. If file doesn`t exists assigns ttl infos with all zeros - void loadTTLInfos(); - - void loadPartitionAndMinMaxIndex(); + String getRelativePathForDetachedPart(const String & prefix) const; void checkConsistency(bool require_part_metadata); - ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const; + virtual ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const = 0; }; - -using MergeTreeDataPartState = MergeTreeDataPart::State; +using MergeTreeDataPartState = IMergeTreeDataPart::State; } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp new file mode 100644 index 00000000000..75b01a22081 --- /dev/null +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -0,0 +1,191 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + using OffsetColumns = std::map; +} + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NOT_FOUND_EXPECTED_DATA_PART; + extern const int MEMORY_LIMIT_EXCEEDED; + extern const int ARGUMENT_OUT_OF_BOUND; +} + + +IMergeTreeReader::IMergeTreeReader(const MergeTreeData::DataPartPtr & data_part_, + const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, + const MarkRanges & all_mark_ranges_, const ReaderSettings & settings_, + const ValueSizeMap & avg_value_size_hints_) + : data_part(data_part_), avg_value_size_hints(avg_value_size_hints_), path(data_part_->getFullPath()) + , columns(columns_), uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_) + , settings(settings_), storage(data_part_->storage) + , all_mark_ranges(all_mark_ranges_) +{ +} + +IMergeTreeReader::~IMergeTreeReader() = default; + + +const IMergeTreeReader::ValueSizeMap & IMergeTreeReader::getAvgValueSizeHints() const +{ + return avg_value_size_hints; +} + + +static bool arrayHasNoElementsRead(const IColumn & column) +{ + const ColumnArray * column_array = typeid_cast(&column); + + if (!column_array) + return false; + + size_t size = column_array->size(); + if (!size) + return false; + + size_t data_size = column_array->getData().size(); + if (data_size) + return false; + + size_t last_offset = column_array->getOffsets()[size - 1]; + return last_offset != 0; +} + + +void IMergeTreeReader::fillMissingColumns(Block & res, bool & should_reorder, bool & should_evaluate_missing_defaults, size_t num_rows) +{ + try + { + /// For a missing column of a nested data structure we must create not a column of empty + /// arrays, but a column of arrays of correct length. + + /// First, collect offset columns for all arrays in the block. + OffsetColumns offset_columns; + for (size_t i = 0; i < res.columns(); ++i) + { + const ColumnWithTypeAndName & column = res.safeGetByPosition(i); + + if (const ColumnArray * array = typeid_cast(column.column.get())) + { + String offsets_name = Nested::extractTableName(column.name); + auto & offsets_column = offset_columns[offsets_name]; + + /// If for some reason multiple offsets columns are present for the same nested data structure, + /// choose the one that is not empty. + if (!offsets_column || offsets_column->empty()) + offsets_column = array->getOffsetsPtr(); + } + } + + should_evaluate_missing_defaults = false; + should_reorder = false; + + /// insert default values only for columns without default expressions + for (const auto & requested_column : columns) + { + bool has_column = res.has(requested_column.name); + if (has_column) + { + const auto & col = *res.getByName(requested_column.name).column; + if (arrayHasNoElementsRead(col)) + { + res.erase(requested_column.name); + has_column = false; + } + } + + if (!has_column) + { + should_reorder = true; + if (storage.getColumns().hasDefault(requested_column.name)) + { + should_evaluate_missing_defaults = true; + continue; + } + + ColumnWithTypeAndName column_to_add; + column_to_add.name = requested_column.name; + column_to_add.type = requested_column.type; + + String offsets_name = Nested::extractTableName(column_to_add.name); + if (offset_columns.count(offsets_name)) + { + ColumnPtr offsets_column = offset_columns[offsets_name]; + DataTypePtr nested_type = typeid_cast(*column_to_add.type).getNestedType(); + size_t nested_rows = typeid_cast(*offsets_column).getData().back(); + + ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(nested_rows)->convertToFullColumnIfConst(); + + column_to_add.column = ColumnArray::create(nested_column, offsets_column); + } + else + { + /// We must turn a constant column into a full column because the interpreter could infer that it is constant everywhere + /// but in some blocks (from other parts) it can be a full column. + column_to_add.column = column_to_add.type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst(); + } + + res.insert(std::move(column_to_add)); + } + } + } + catch (Exception & e) + { + /// Better diagnostics. + e.addMessage("(while reading from part " + path + ")"); + throw; + } +} + +void IMergeTreeReader::reorderColumns(Block & res, const Names & ordered_names, const String * filter_name) +{ + try + { + Block ordered_block; + + for (const auto & name : ordered_names) + if (res.has(name)) + ordered_block.insert(res.getByName(name)); + + if (filter_name && !ordered_block.has(*filter_name) && res.has(*filter_name)) + ordered_block.insert(res.getByName(*filter_name)); + + std::swap(res, ordered_block); + } + catch (Exception & e) + { + /// Better diagnostics. + e.addMessage("(while reading from part " + path + ")"); + throw; + } +} + +void IMergeTreeReader::evaluateMissingDefaults(Block & res) +{ + try + { + DB::evaluateMissingDefaults(res, columns, storage.getColumns().getDefaults(), storage.global_context); + } + catch (Exception & e) + { + /// Better diagnostics. + e.addMessage("(while reading from part " + path + ")"); + throw; + } +} + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h similarity index 67% rename from dbms/src/Storages/MergeTree/MergeTreeReader.h rename to dbms/src/Storages/MergeTree/IMergeTreeReader.h index 25f4c9ddd32..8ff463639e8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -2,7 +2,7 @@ #include #include -#include +#include namespace DB @@ -13,24 +13,25 @@ class IDataType; /// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. /// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. /// Avoids loading the marks file if it is not needed (e.g. when reading the whole part). -class MergeTreeReader : private boost::noncopyable +class IMergeTreeReader : private boost::noncopyable { public: using ValueSizeMap = std::map; using DeserializeBinaryBulkStateMap = std::map; - MergeTreeReader(const String & path_, /// Path to the directory containing the part - const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, + IMergeTreeReader(const MergeTreeData::DataPartPtr & data_part_, + const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, - bool save_marks_in_cache_, - const MergeTreeData & storage_, const MarkRanges & all_mark_ranges_, - size_t aio_threshold_, size_t max_read_buffer_size_, - const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}, - clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); + const MarkRanges & all_mark_ranges_, + const ReaderSettings & settings_, + const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}); - ~MergeTreeReader(); + /// Return the number of rows has been read or zero if there is no columns to read. + /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark + virtual size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) = 0; + + virtual ~IMergeTreeReader(); const ValueSizeMap & getAvgValueSizeHints() const; @@ -47,19 +48,14 @@ public: const NamesAndTypesList & getColumns() const { return columns; } - /// Return the number of rows has been read or zero if there is no columns to read. - /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark - size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res); - MergeTreeData::DataPartPtr data_part; size_t getFirstMarkToRead() const { return all_mark_ranges.back().begin; } -private: - using FileStreams = std::map>; +protected: /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk @@ -67,28 +63,17 @@ private: /// Path to the directory containing the part String path; - FileStreams streams; - /// Columns that are read. NamesAndTypesList columns; UncompressedCache * uncompressed_cache; MarkCache * mark_cache; /// If save_marks_in_cache is false, then, if marks are not in cache, we will load them but won't save in the cache, to avoid evicting other data. - bool save_marks_in_cache; + + ReaderSettings settings; const MergeTreeData & storage; MarkRanges all_mark_ranges; - size_t aio_threshold; - size_t max_read_buffer_size; - - void addStreams(const String & name, const IDataType & type, - const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); - - void readData( - const String & name, const IDataType & type, IColumn & column, - size_t from_mark, bool continue_reading, size_t max_rows_to_read, - bool read_offsets = true); friend class MergeTreeRangeReader::DelayedStream; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp index 0489182fe55..be3f7496439 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp @@ -1,6 +1,7 @@ #include #include -#include +#include +#include #include #include #include @@ -25,10 +26,8 @@ MergeTreeBaseSelectBlockInputStream::MergeTreeBaseSelectBlockInputStream( UInt64 max_block_size_rows_, UInt64 preferred_block_size_bytes_, UInt64 preferred_max_column_in_block_size_bytes_, - UInt64 min_bytes_to_use_direct_io_, - UInt64 max_read_buffer_size_, + const ReaderSettings & reader_settings_, bool use_uncompressed_cache_, - bool save_marks_in_cache_, const Names & virt_column_names_) : storage(storage_), @@ -36,10 +35,8 @@ MergeTreeBaseSelectBlockInputStream::MergeTreeBaseSelectBlockInputStream( max_block_size_rows(max_block_size_rows_), preferred_block_size_bytes(preferred_block_size_bytes_), preferred_max_column_in_block_size_bytes(preferred_max_column_in_block_size_bytes_), - min_bytes_to_use_direct_io(min_bytes_to_use_direct_io_), - max_read_buffer_size(max_read_buffer_size_), + reader_settings(reader_settings_), use_uncompressed_cache(use_uncompressed_cache_), - save_marks_in_cache(save_marks_in_cache_), virt_column_names(virt_column_names_) { } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h index 640f73652e4..842f663c2f9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h @@ -8,7 +8,7 @@ namespace DB { -class MergeTreeReader; +class IMergeTreeReader; class UncompressedCache; class MarkCache; @@ -23,10 +23,8 @@ public: UInt64 max_block_size_rows_, UInt64 preferred_block_size_bytes_, UInt64 preferred_max_column_in_block_size_bytes_, - UInt64 min_bytes_to_use_direct_io_, - UInt64 max_read_buffer_size_, + const ReaderSettings & reader_settings_, bool use_uncompressed_cache_, - bool save_marks_in_cache_ = true, const Names & virt_column_names_ = {}); ~MergeTreeBaseSelectBlockInputStream() override; @@ -61,11 +59,9 @@ protected: UInt64 preferred_block_size_bytes; UInt64 preferred_max_column_in_block_size_bytes; - UInt64 min_bytes_to_use_direct_io; - UInt64 max_read_buffer_size; + ReaderSettings reader_settings; bool use_uncompressed_cache; - bool save_marks_in_cache; Names virt_column_names; @@ -74,7 +70,7 @@ protected: std::shared_ptr owned_uncompressed_cache; std::shared_ptr owned_mark_cache; - using MergeTreeReaderPtr = std::unique_ptr; + using MergeTreeReaderPtr = std::unique_ptr; MergeTreeReaderPtr reader; MergeTreeReaderPtr pre_reader; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index a031255b3ab..fc3c466f467 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index d68d96675af..67215efd082 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -810,8 +811,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return; - MutableDataPartPtr part = std::make_shared(*this, part_disk_ptr, part_name, part_info); - part->relative_path = part_name; + MutableDataPartPtr part = createPart(*this, part_disk_ptr, part_name, part_info, part_name); bool broken = false; try @@ -1738,7 +1738,7 @@ void MergeTreeData::checkSettingCanBeChanged(const String & setting_name) const void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr & data_part) { - auto & empty_columns = data_part->empty_columns; + auto & empty_columns = data_part->expired_columns; if (empty_columns.empty()) return; @@ -2126,14 +2126,14 @@ void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVect for (const DataPartPtr & part : remove) { - if (part->state == MergeTreeDataPart::State::Committed) + if (part->state ==IMergeTreeDataPart::State::Committed) removePartContributionToColumnSizes(part); - if (part->state == MergeTreeDataPart::State::Committed || clear_without_timeout) + if (part->state ==IMergeTreeDataPart::State::Committed || clear_without_timeout) part->remove_time.store(remove_time, std::memory_order_relaxed); - if (part->state != MergeTreeDataPart::State::Outdated) - modifyPartState(part, MergeTreeDataPart::State::Outdated); + if (part->state !=IMergeTreeDataPart::State::Outdated) + modifyPartState(part,IMergeTreeDataPart::State::Outdated); } } @@ -2583,8 +2583,7 @@ MergeTreeData::DataPartPtr MergeTreeData::getPartIfExists(const String & part_na MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const DiskSpace::DiskPtr & disk, const String & relative_path) { - MutableDataPartPtr part = std::make_shared(*this, disk, Poco::Path(relative_path).getFileName()); - part->relative_path = relative_path; + MutableDataPartPtr part = createPart(*this, disk, Poco::Path(relative_path).getFileName(), relative_path); loadPartAndFixMetadata(part); return part; } @@ -3022,8 +3021,8 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const for (const auto & part_names : renamed_parts.old_and_new_names) { LOG_DEBUG(log, "Checking part " << part_names.second); - MutableDataPartPtr part = std::make_shared(*this, name_to_disk[part_names.first], part_names.first); - part->relative_path = source_dir + part_names.second; + MutableDataPartPtr part = createPart( + *this, name_to_disk[part_names.first], part_names.first, source_dir + part_names.second); loadPartAndFixMetadata(part); loaded_parts.push_back(part); } @@ -3235,10 +3234,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPart(const Merg LOG_DEBUG(log, "Cloning part " << src_part_absolute_path.toString() << " to " << dst_part_absolute_path.toString()); localBackup(src_part_absolute_path, dst_part_absolute_path); - MergeTreeData::MutableDataPartPtr dst_data_part = std::make_shared( - *this, reservation->getDisk(), dst_part_name, dst_part_info); + MergeTreeData::MutableDataPartPtr dst_data_part = createPart( + *this, reservation->getDisk(), dst_part_name, dst_part_info, tmp_dst_part_name); - dst_data_part->relative_path = tmp_dst_part_name; dst_data_part->is_temp = true; dst_data_part->loadColumnsChecksumsIndexes(require_part_metadata, true); @@ -3395,7 +3393,7 @@ MergeTreeData::CurrentlyMovingPartsTagger::CurrentlyMovingPartsTagger(MergeTreeM : parts_to_move(std::move(moving_parts_)), data(data_) { for (const auto & moving_part : parts_to_move) - if (!data.currently_moving_parts.emplace(moving_part.part).second) + if (!data.currently_moving_parts.emplace(moving_part.part->name).second) throw Exception("Cannot move part '" + moving_part.part->name + "'. It's already moving.", ErrorCodes::LOGICAL_ERROR); } @@ -3405,9 +3403,9 @@ MergeTreeData::CurrentlyMovingPartsTagger::~CurrentlyMovingPartsTagger() for (const auto & moving_part : parts_to_move) { /// Something went completely wrong - if (!data.currently_moving_parts.count(moving_part.part)) + if (!data.currently_moving_parts.count(moving_part.part->name)) std::terminate(); - data.currently_moving_parts.erase(moving_part.part); + data.currently_moving_parts.erase(moving_part.part->name); } } @@ -3446,7 +3444,7 @@ MergeTreeData::CurrentlyMovingPartsTagger MergeTreeData::selectPartsForMove() *reason = "part already assigned to background operation."; return false; } - if (currently_moving_parts.count(part)) + if (currently_moving_parts.count(part->name)) { *reason = "part is already moving."; return false; @@ -3480,7 +3478,7 @@ MergeTreeData::CurrentlyMovingPartsTagger MergeTreeData::checkPartsForMove(const "Move is not possible: " + path_to_clone + part->name + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - if (currently_moving_parts.count(part) || partIsAssignedToBackgroundOperation(part)) + if (currently_moving_parts.count(part->name) || partIsAssignedToBackgroundOperation(part)) throw Exception( "Cannot move part '" + part->name + "' because it's participating in background process", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index a8bd661fafa..9e537c03616 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -15,7 +15,7 @@ #include #include #include -#include +#include #include #include #include @@ -78,7 +78,7 @@ namespace ErrorCodes /// The same files as for month-partitioned tables, plus /// count.txt - contains total number of rows in this part. /// partition.dat - contains the value of the partitioning expression. -/// minmax_[Column].idx - MinMax indexes (see MergeTreeDataPart::MinMaxIndex class) for the columns required by the partitioning expression. +/// minmax_[Column].idx - MinMax indexes (seeIMergeTreeDataPart::MinMaxIndex class) for the columns required by the partitioning expression. /// /// Several modes are implemented. Modes determine additional actions during merge: /// - Ordinary - don't do anything special @@ -101,14 +101,14 @@ class MergeTreeData : public IStorage public: /// Function to call if the part is suspected to contain corrupt data. using BrokenPartCallback = std::function; - using DataPart = MergeTreeDataPart; + using DataPart = IMergeTreeDataPart; using MutableDataPartPtr = std::shared_ptr; using MutableDataPartsVector = std::vector; /// After the DataPart is added to the working set, it cannot be changed. using DataPartPtr = std::shared_ptr; - using DataPartState = MergeTreeDataPart::State; + using DataPartState = IMergeTreeDataPart::State; using DataPartStates = std::initializer_list; using DataPartStateVector = std::vector; @@ -737,14 +737,15 @@ public: /// if we decide to move some part to another disk, than we /// assuredly will choose this disk for containing part, which will appear /// as result of merge or mutation. - DataParts currently_moving_parts; + NameSet currently_moving_parts; /// Mutex for currently_moving_parts mutable std::mutex moving_parts_mutex; protected: - friend struct MergeTreeDataPart; + friend class IMergeTreeDataPart; + friend class MergeTreeDataPartWide; friend class MergeTreeDataMergerMutator; friend class ReplicatedMergeTreeAlterThread; friend struct ReplicatedMergeTreeTableMetadata; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index df3720359d3..892fc50f0a8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -552,8 +553,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor throw Exception("Directory " + new_part_tmp_path + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); MergeTreeData::DataPart::ColumnToSize merged_column_to_size; - for (const MergeTreeData::DataPartPtr & part : parts) - part->accumulateColumnSizes(merged_column_to_size); + // for (const MergeTreeData::DataPartPtr & part : parts) + // part->accumulateColumnSizes(merged_column_to_size); Names all_column_names = data.getColumns().getNamesOfPhysical(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); @@ -566,10 +567,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor all_columns, data.sorting_key_expr, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); - MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared( - data, space_reservation->getDisk(), future_part.name, future_part.part_info); + MergeTreeData::MutableDataPartPtr new_data_part = createPart( + data, space_reservation->getDisk(), future_part.name, future_part.part_info, TMP_PREFIX + future_part.name); + new_data_part->partition.assign(future_part.getPartition()); - new_data_part->relative_path = TMP_PREFIX + future_part.name; new_data_part->is_temp = true; size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; @@ -922,7 +923,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor CurrentMetrics::Increment num_mutations{CurrentMetrics::PartMutation}; const auto & source_part = future_part.parts[0]; - auto storage_from_source_part = StorageFromMergeTreeDataPart::create(source_part); + auto storage_from_source_part = StorageFroIMergeTreeDataPart::create(source_part); auto context_for_reading = context; context_for_reading.getSettingsRef().merge_tree_uniform_read_distribution = 0; @@ -949,9 +950,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor else LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation); - MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared( - data, space_reservation->getDisk(), future_part.name, future_part.part_info); - new_data_part->relative_path = "tmp_mut_" + future_part.name; + MergeTreeData::MutableDataPartPtr new_data_part = createPart( + data, space_reservation->getDisk(), future_part.name, future_part.part_info, "tmp_mut_" + future_part.name); + new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; new_data_part->index_granularity_info = source_part->index_granularity_info; @@ -988,7 +989,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor in = std::make_shared( std::make_shared(in, data.primary_key_and_skip_indices_expr)); - MergeTreeDataPart::MinMaxIndex minmax_idx; + IMergeTreeDataPart::MinMaxIndex minmax_idx; MergedBlockOutputStream out(data, new_part_tmp_path, all_columns, compression_codec); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h new file mode 100644 index 00000000000..a6a2bf48dcf --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -0,0 +1,114 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + + +namespace DB +{ + +struct ColumnSize; +class MergeTreeData; + + +/// Description of the data part. +class MergeTreeDataPartCompact : public IMergeTreeDataPart +{ +public: + using Checksums = MergeTreeDataPartChecksums; + using Checksum = MergeTreeDataPartChecksums::Checksum; + + MergeTreeDataPartCompact( + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const DiskSpace::DiskPtr & disk, + const std::optional & relative_path = {}); + + MergeTreeDataPartCompact( + MergeTreeData & storage_, + const String & name_, + const DiskSpace::DiskPtr & disk, + const std::optional & relative_path = {}); + + MergeTreeReaderPtr getReader( + const NamesAndTypesList & columns, + const MarkRanges & mark_ranges, + UncompressedCache * uncompressed_cache, + MarkCache * mark_cache, + const ReaderSettings & reader_settings_, + const ValueSizeMap & avg_value_size_hints = ValueSizeMap{}, + const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{}) const override; + + bool isStoredOnDisk() const override { return true; } + + void remove() const override; + + /// NOTE: Returns zeros if column files are not found in checksums. + /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. + /// (either by locking columns_lock, or by locking table structure). + ColumnSize getColumnSize(const String & name, const IDataType & type) const override; + + /// Initialize columns (from columns.txt if exists, or create from column files if not). + /// Load checksums from checksums.txt if exists. Load index if required. + void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) override; + + /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). + /// If no checksums are present returns the name of the first physically existing column. + String getColumnNameWithMinumumCompressedSize() const override; + + virtual Type getType() const override { return Type::WIDE; } + + ~MergeTreeDataPartCompact() override; + + /// Calculate the total size of the entire directory with all the files + static UInt64 calculateTotalSizeOnDisk(const String & from); + + +private: + /// Reads columns names and types from columns.txt + void loadColumns(bool require); + + /// If checksums.txt exists, reads files' checksums (and sizes) from it + void loadChecksums(bool require); + + /// Loads marks index granularity into memory + void loadIndexGranularity(); + + /// Loads index file. + void loadIndex(); + + /// Load rows count for this part from disk (for the newer storage format version). + /// For the older format version calculates rows count from the size of a column with a fixed size. + void loadRowsCount(); + + /// Loads ttl infos in json format from file ttl.txt. If file doesn`t exists assigns ttl infos with all zeros + void loadTTLInfos(); + + void loadPartitionAndMinMaxIndex(); + + ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const override; + + void checkConsistency(bool require_part_metadata); +}; + + +// using MergeTreeDataPartState =IMergeTreeDataPart::State; + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp new file mode 100644 index 00000000000..1abfbbe3708 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp @@ -0,0 +1,16 @@ +#include "MergeTreeDataPartFactory.h" + +namespace DB +{ + std::shared_ptr createPart(const MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, + const MergeTreePartInfo & info, const String & relative_path) + { + return std::make_shared(storage, name, info, disk, relative_path); + } + + std::shared_ptr createPart(MergeTreeData & storage, const DiskSpace::DiskPtr & disk, + const String & name, const String & relative_path) + { + return std::make_shared(storage, name, disk, relative_path); + } +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h new file mode 100644 index 00000000000..38091e5d38b --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h @@ -0,0 +1,9 @@ +#include +#include + +namespace DB +{ + std::shared_ptr createPart(const MergeTreeData & storage_, const DiskSpace::DiskPtr & disk_, const String & name_, const MergeTreePartInfo & info_, const String & relative_path); + + std::shared_ptr createPart(MergeTreeData & storage_, const DiskSpace::DiskPtr & disk_, const String & name_, const String & relative_path); +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h new file mode 100644 index 00000000000..70e799ba926 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +namespace DB +{ + +class MergeTreeDataPartOnDisk : IMergeTreeDataPart +{ + + +}; + +} \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp similarity index 60% rename from dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp rename to dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index b84380442b2..a187460e38c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -1,4 +1,4 @@ -#include "MergeTreeDataPart.h" +#include "MergeTreeDataPartWide.h" #include #include @@ -22,9 +22,17 @@ #include #include +#include +#include + + namespace DB { +// namespace +// { +// } + namespace ErrorCodes { extern const int FILE_DOESNT_EXIST; @@ -43,123 +51,42 @@ static ReadBufferFromFile openForReading(const String & path) return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); } -void MergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const String & part_path) -{ - size_t minmax_idx_size = data.minmax_idx_column_types.size(); - parallelogram.reserve(minmax_idx_size); - for (size_t i = 0; i < minmax_idx_size; ++i) - { - String file_name = part_path + "minmax_" + escapeForFileName(data.minmax_idx_columns[i]) + ".idx"; - ReadBufferFromFile file = openForReading(file_name); - const DataTypePtr & type = data.minmax_idx_column_types[i]; - - Field min_val; - type->deserializeBinary(min_val, file); - Field max_val; - type->deserializeBinary(max_val, file); - - parallelogram.emplace_back(min_val, true, max_val, true); - } - initialized = true; -} - -void MergeTreeDataPart::MinMaxIndex::store(const MergeTreeData & data, const String & part_path, Checksums & out_checksums) const -{ - store(data.minmax_idx_columns, data.minmax_idx_column_types, part_path, out_checksums); -} - -void MergeTreeDataPart::MinMaxIndex::store(const Names & column_names, const DataTypes & data_types, const String & part_path, Checksums & out_checksums) const -{ - if (!initialized) - throw Exception("Attempt to store uninitialized MinMax index for part " + part_path + ". This is a bug.", - ErrorCodes::LOGICAL_ERROR); - - for (size_t i = 0; i < column_names.size(); ++i) - { - String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; - const DataTypePtr & type = data_types.at(i); - - WriteBufferFromFile out(part_path + file_name); - HashingWriteBuffer out_hashing(out); - type->serializeBinary(parallelogram[i].left, out_hashing); - type->serializeBinary(parallelogram[i].right, out_hashing); - out_hashing.next(); - out_checksums.files[file_name].file_size = out_hashing.count(); - out_checksums.files[file_name].file_hash = out_hashing.getHash(); - } -} - -void MergeTreeDataPart::MinMaxIndex::update(const Block & block, const Names & column_names) -{ - if (!initialized) - parallelogram.reserve(column_names.size()); - - for (size_t i = 0; i < column_names.size(); ++i) - { - Field min_value; - Field max_value; - const ColumnWithTypeAndName & column = block.getByName(column_names[i]); - column.column->getExtremes(min_value, max_value); - - if (!initialized) - parallelogram.emplace_back(min_value, true, max_value, true); - else - { - parallelogram[i].left = std::min(parallelogram[i].left, min_value); - parallelogram[i].right = std::max(parallelogram[i].right, max_value); - } - } - - initialized = true; -} - -void MergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other) -{ - if (!other.initialized) - return; - - if (!initialized) - { - parallelogram = other.parallelogram; - initialized = true; - } - else - { - for (size_t i = 0; i < parallelogram.size(); ++i) - { - parallelogram[i].left = std::min(parallelogram[i].left, other.parallelogram[i].left); - parallelogram[i].right = std::max(parallelogram[i].right, other.parallelogram[i].right); - } - } -} - - -MergeTreeDataPart::MergeTreeDataPart(MergeTreeData & storage_, const DiskSpace::DiskPtr & disk_, const String & name_) - : storage(storage_) - , disk(disk_) - , name(name_) - , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) - , index_granularity_info(storage) +MergeTreeDataPartWide::MergeTreeDataPartWide( + MergeTreeData & storage_, + const String & name_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_) + : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) { } -MergeTreeDataPart::MergeTreeDataPart( - const MergeTreeData & storage_, - const DiskSpace::DiskPtr & disk_, - const String & name_, - const MergeTreePartInfo & info_) - : storage(storage_) - , disk(disk_) - , name(name_) - , info(info_) - , index_granularity_info(storage) +MergeTreeDataPartWide::MergeTreeDataPartWide( + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_) + : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) { } +IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader( + const NamesAndTypesList & columns_to_read, + const MarkRanges & mark_ranges, + UncompressedCache * uncompressed_cache, + MarkCache * mark_cache, + const ReaderSettings & reader_settings, + const ValueSizeMap & avg_value_size_hints, + const ReadBufferFromFileBase::ProfileCallback & profile_callback) const +{ + return std::make_unique(shared_from_this(), columns_to_read, uncompressed_cache, + mark_cache, mark_ranges, reader_settings, avg_value_size_hints, profile_callback); +} + /// Takes into account the fact that several columns can e.g. share their .size substreams. /// When calculating totals these should be counted only once. -ColumnSize MergeTreeDataPart::getColumnSizeImpl( +ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( const String & column_name, const IDataType & type, std::unordered_set * processed_substreams) const { ColumnSize size; @@ -188,36 +115,15 @@ ColumnSize MergeTreeDataPart::getColumnSizeImpl( return size; } -ColumnSize MergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const +ColumnSize MergeTreeDataPartWide::getColumnSize(const String & column_name, const IDataType & type) const { return getColumnSizeImpl(column_name, type, nullptr); } -ColumnSize MergeTreeDataPart::getTotalColumnsSize() const -{ - ColumnSize totals; - std::unordered_set processed_substreams; - for (const NameAndTypePair & column : columns) - { - ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams); - totals.add(size); - } - return totals; -} - - -size_t MergeTreeDataPart::getFileSizeOrZero(const String & file_name) const -{ - auto checksum = checksums.files.find(file_name); - if (checksum == checksums.files.end()) - return 0; - return checksum->second.file_size; -} - /** Returns the name of a column with minimum compressed size (as returned by getColumnSize()). * If no checksums are present returns the name of the first physically existing column. */ -String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const +String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const { const auto & storage_columns = storage.getColumns().getAllPhysical(); const std::string * minimum_size_column = nullptr; @@ -242,77 +148,7 @@ String MergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const return *minimum_size_column; } - -String MergeTreeDataPart::getFullPath() const -{ - if (relative_path.empty()) - throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); - - return storage.getFullPathOnDisk(disk) + relative_path + "/"; -} - -String MergeTreeDataPart::getNameWithPrefix() const -{ - String res = Poco::Path(relative_path).getFileName(); - - if (res.empty()) - throw Exception("relative_path " + relative_path + " of part " + name + " is invalid or not set", ErrorCodes::LOGICAL_ERROR); - - return res; -} - -String MergeTreeDataPart::getNewName(const MergeTreePartInfo & new_part_info) const -{ - if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - /// NOTE: getting min and max dates from the part name (instead of part data) because we want - /// the merged part name be determined only by source part names. - /// It is simpler this way when the real min and max dates for the block range can change - /// (e.g. after an ALTER DELETE command). - DayNum min_date; - DayNum max_date; - MergeTreePartInfo::parseMinMaxDatesFromPartName(name, min_date, max_date); - return new_part_info.getPartNameV0(min_date, max_date); - } - else - return new_part_info.getPartName(); -} - -DayNum MergeTreeDataPart::getMinDate() const -{ - if (storage.minmax_idx_date_column_pos != -1 && minmax_idx.initialized) - return DayNum(minmax_idx.parallelogram[storage.minmax_idx_date_column_pos].left.get()); - else - return DayNum(); -} - - -DayNum MergeTreeDataPart::getMaxDate() const -{ - if (storage.minmax_idx_date_column_pos != -1 && minmax_idx.initialized) - return DayNum(minmax_idx.parallelogram[storage.minmax_idx_date_column_pos].right.get()); - else - return DayNum(); -} - -time_t MergeTreeDataPart::getMinTime() const -{ - if (storage.minmax_idx_time_column_pos != -1 && minmax_idx.initialized) - return minmax_idx.parallelogram[storage.minmax_idx_time_column_pos].left.get(); - else - return 0; -} - - -time_t MergeTreeDataPart::getMaxTime() const -{ - if (storage.minmax_idx_time_column_pos != -1 && minmax_idx.initialized) - return minmax_idx.parallelogram[storage.minmax_idx_time_column_pos].right.get(); - else - return 0; -} - -MergeTreeDataPart::~MergeTreeDataPart() +MergeTreeDataPartWide::~MergeTreeDataPartWide() { if (state == State::DeleteOnDestroy || is_temp) { @@ -326,7 +162,12 @@ MergeTreeDataPart::~MergeTreeDataPart() if (is_temp) { - if (!startsWith(getNameWithPrefix(), "tmp")) + String file_name = Poco::Path(relative_path).getFileName(); + + if (file_name.empty()) + throw Exception("relative_path " + relative_path + " of part " + name + " is invalid or not set", ErrorCodes::LOGICAL_ERROR); + + if (!startsWith(file_name, "tmp")) { LOG_ERROR(storage.log, "~DataPart() should remove part " << path << " but its name doesn't start with tmp. Too suspicious, keeping the part."); @@ -343,7 +184,7 @@ MergeTreeDataPart::~MergeTreeDataPart() } } -UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) +UInt64 MergeTreeDataPartWide::calculateTotalSizeOnDisk(const String & from) { Poco::File cur(from); if (cur.isFile()) @@ -356,7 +197,7 @@ UInt64 MergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) return res; } -void MergeTreeDataPart::remove() const +void MergeTreeDataPartWide::remove() const { if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); @@ -453,104 +294,7 @@ void MergeTreeDataPart::remove() const } } - -void MergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_new_dir_if_exists) const -{ - String from = getFullPath(); - String to = storage.getFullPathOnDisk(disk) + new_relative_path + "/"; - - Poco::File from_file(from); - if (!from_file.exists()) - throw Exception("Part directory " + from + " doesn't exist. Most likely it is logical error.", ErrorCodes::FILE_DOESNT_EXIST); - - Poco::File to_file(to); - if (to_file.exists()) - { - if (remove_new_dir_if_exists) - { - Names files; - Poco::File(from).list(files); - - LOG_WARNING(storage.log, "Part directory " << to << " already exists" - << " and contains " << files.size() << " files. Removing it."); - - to_file.remove(true); - } - else - { - throw Exception("Part directory " + to + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - } - } - - from_file.setLastModified(Poco::Timestamp::fromEpochTime(time(nullptr))); - from_file.renameTo(to); - relative_path = new_relative_path; -} - - -String MergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) const -{ - /// Do not allow underscores in the prefix because they are used as separators. - assert(prefix.find_first_of('_') == String::npos); - - String res; - - /** If you need to detach a part, and directory into which we want to rename it already exists, - * we will rename to the directory with the name to which the suffix is added in the form of "_tryN". - * This is done only in the case of `to_detached`, because it is assumed that in this case the exact name does not matter. - * No more than 10 attempts are made so that there are not too many junk directories left. - */ - for (int try_no = 0; try_no < 10; try_no++) - { - res = "detached/" + (prefix.empty() ? "" : prefix + "_") - + name + (try_no ? "_try" + DB::toString(try_no) : ""); - - if (!Poco::File(storage.getFullPathOnDisk(disk) + res).exists()) - return res; - - LOG_WARNING(storage.log, "Directory " << res << " (to detach to) already exists." - " Will detach to directory with '_tryN' suffix."); - } - - return res; -} - -void MergeTreeDataPart::renameToDetached(const String & prefix) const -{ - renameTo(getRelativePathForDetachedPart(prefix)); -} - - -UInt64 MergeTreeDataPart::getMarksCount() const -{ - return index_granularity.getMarksCount(); -} - -void MergeTreeDataPart::makeCloneInDetached(const String & prefix) const -{ - Poco::Path src(getFullPath()); - Poco::Path dst(storage.getFullPathOnDisk(disk) + getRelativePathForDetachedPart(prefix)); - /// Backup is not recursive (max_level is 0), so do not copy inner directories - localBackup(src, dst, 0); -} - -void MergeTreeDataPart::makeCloneOnDiskDetached(const DiskSpace::ReservationPtr & reservation) const -{ - auto & reserved_disk = reservation->getDisk(); - if (reserved_disk->getName() == disk->getName()) - throw Exception("Can not clone data part " + name + " to same disk " + disk->getName(), ErrorCodes::LOGICAL_ERROR); - - String path_to_clone = storage.getFullPathOnDisk(reserved_disk) + "detached/"; - - if (Poco::File(path_to_clone + relative_path).exists()) - throw Exception("Path " + path_to_clone + relative_path + " already exists. Can not clone ", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - Poco::File(path_to_clone).createDirectory(); - - Poco::File cloning_directory(getFullPath()); - cloning_directory.copyTo(path_to_clone); -} - -void MergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) +void MergeTreeDataPartWide::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool /* check_consistency */) { /// Memory should not be limited during ATTACH TABLE query. /// This is already true at the server startup but must be also ensured for manual table ATTACH. @@ -564,13 +308,12 @@ void MergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksu loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. loadPartitionAndMinMaxIndex(); loadTTLInfos(); - if (check_consistency) - checkConsistency(require_columns_checksums); + // if (check_consistency) + // checkConsistency(require_columns_checksums); } -void MergeTreeDataPart::loadIndexGranularity() +void MergeTreeDataPartWide::loadIndexGranularity() { - String full_path = getFullPath(); index_granularity_info.changeGranularityIfRequired(full_path); @@ -606,7 +349,7 @@ void MergeTreeDataPart::loadIndexGranularity() index_granularity.setInitialized(); } -void MergeTreeDataPart::loadIndex() +void MergeTreeDataPartWide::loadIndex() { /// It can be empty in case of mutations if (!index_granularity.isInitialized()) @@ -648,7 +391,7 @@ void MergeTreeDataPart::loadIndex() } } -void MergeTreeDataPart::loadPartitionAndMinMaxIndex() +void MergeTreeDataPartWide::loadPartitionAndMinMaxIndex() { if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { @@ -676,7 +419,7 @@ void MergeTreeDataPart::loadPartitionAndMinMaxIndex() ErrorCodes::CORRUPTED_DATA); } -void MergeTreeDataPart::loadChecksums(bool require) +void MergeTreeDataPartWide::loadChecksums(bool require) { String path = getFullPath() + "checksums.txt"; Poco::File checksums_file(path); @@ -700,7 +443,7 @@ void MergeTreeDataPart::loadChecksums(bool require) } } -void MergeTreeDataPart::loadRowsCount() +void MergeTreeDataPartWide::loadRowsCount() { if (index_granularity.empty()) { @@ -754,7 +497,7 @@ void MergeTreeDataPart::loadRowsCount() } } -void MergeTreeDataPart::loadTTLInfos() +void MergeTreeDataPartWide::loadTTLInfos() { String path = getFullPath() + "ttl.txt"; if (Poco::File(path).exists()) @@ -781,23 +524,23 @@ void MergeTreeDataPart::loadTTLInfos() } } -void MergeTreeDataPart::accumulateColumnSizes(ColumnToSize & column_to_size) const -{ - std::shared_lock part_lock(columns_lock); +// void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) const +// { +// std::shared_lock part_lock(columns_lock); - for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical()) - { - IDataType::SubstreamPath path; - name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - Poco::File bin_file(getFullPath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"); - if (bin_file.exists()) - column_to_size[name_type.name] += bin_file.getSize(); - }, path); - } -} +// for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical()) +// { +// IDataType::SubstreamPath path; +// name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) +// { +// Poco::File bin_file(getFullPath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"); +// if (bin_file.exists()) +// column_to_size[name_type.name] += bin_file.getSize(); +// }, path); +// } +// } -void MergeTreeDataPart::loadColumns(bool require) +void MergeTreeDataPartWide::loadColumns(bool require) { String path = getFullPath() + "columns.txt"; Poco::File poco_file_path{path}; @@ -829,7 +572,7 @@ void MergeTreeDataPart::loadColumns(bool require) columns.readText(file); } -void MergeTreeDataPart::checkConsistency(bool require_part_metadata) +void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) { String path = getFullPath(); @@ -932,77 +675,22 @@ void MergeTreeDataPart::checkConsistency(bool require_part_metadata) } } -bool MergeTreeDataPart::hasColumnFiles(const String & column_name, const IDataType & type) const -{ - bool res = true; +// bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDataType & type) const +// { +// bool res = true; - type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(column_name, substream_path); +// type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) +// { +// String file_name = IDataType::getFileNameForStream(column_name, substream_path); - auto bin_checksum = checksums.files.find(file_name + ".bin"); - auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); +// auto bin_checksum = checksums.files.find(file_name + ".bin"); +// auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); - if (bin_checksum == checksums.files.end() || mrk_checksum == checksums.files.end()) - res = false; - }, {}); +// if (bin_checksum == checksums.files.end() || mrk_checksum == checksums.files.end()) +// res = false; +// }, {}); - return res; -} - - -UInt64 MergeTreeDataPart::getIndexSizeInBytes() const -{ - UInt64 res = 0; - for (const ColumnPtr & column : index) - res += column->byteSize(); - return res; -} - -UInt64 MergeTreeDataPart::getIndexSizeInAllocatedBytes() const -{ - UInt64 res = 0; - for (const ColumnPtr & column : index) - res += column->allocatedBytes(); - return res; -} - -String MergeTreeDataPart::stateToString(MergeTreeDataPart::State state) -{ - switch (state) - { - case State::Temporary: - return "Temporary"; - case State::PreCommitted: - return "PreCommitted"; - case State::Committed: - return "Committed"; - case State::Outdated: - return "Outdated"; - case State::Deleting: - return "Deleting"; - case State::DeleteOnDestroy: - return "DeleteOnDestroy"; - } - - __builtin_unreachable(); -} - -String MergeTreeDataPart::stateString() const -{ - return stateToString(state); -} - -void MergeTreeDataPart::assertState(const std::initializer_list & affordable_states) const -{ - if (!checkState(affordable_states)) - { - String states_str; - for (auto affordable_state : affordable_states) - states_str += stateToString(affordable_state) + " "; - - throw Exception("Unexpected state of part " + getNameWithState() + ". Expected: " + states_str, ErrorCodes::NOT_FOUND_EXPECTED_DATA_PART); - } -} +// return res; +// } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h new file mode 100644 index 00000000000..5bad56a94ed --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -0,0 +1,116 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + + +namespace DB +{ + +struct ColumnSize; +class MergeTreeData; + + +/// Description of the data part. +class MergeTreeDataPartWide : public IMergeTreeDataPart +{ +public: + using Checksums = MergeTreeDataPartChecksums; + using Checksum = MergeTreeDataPartChecksums::Checksum; + + MergeTreeDataPartWide( + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const DiskSpace::DiskPtr & disk, + const std::optional & relative_path = {}); + + MergeTreeDataPartWide( + MergeTreeData & storage_, + const String & name_, + const DiskSpace::DiskPtr & disk, + const std::optional & relative_path = {}); + + MergeTreeReaderPtr getReader( + const NamesAndTypesList & columns, + const MarkRanges & mark_ranges, + UncompressedCache * uncompressed_cache, + MarkCache * mark_cache, + const ReaderSettings & reader_settings_, + const ValueSizeMap & avg_value_size_hints = ValueSizeMap{}, + const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{}) const override; + + bool isStoredOnDisk() const override { return true; } + + void remove() const override; + + bool supportsVerticalMerge() const override { return true; } + + /// NOTE: Returns zeros if column files are not found in checksums. + /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. + /// (either by locking columns_lock, or by locking table structure). + ColumnSize getColumnSize(const String & name, const IDataType & type) const override; + + /// Initialize columns (from columns.txt if exists, or create from column files if not). + /// Load checksums from checksums.txt if exists. Load index if required. + void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) override; + + /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). + /// If no checksums are present returns the name of the first physically existing column. + String getColumnNameWithMinumumCompressedSize() const override; + + virtual Type getType() const override { return Type::WIDE; } + + ~MergeTreeDataPartWide() override; + + /// Calculate the total size of the entire directory with all the files + static UInt64 calculateTotalSizeOnDisk(const String & from); + + +private: + /// Reads columns names and types from columns.txt + void loadColumns(bool require); + + /// If checksums.txt exists, reads files' checksums (and sizes) from it + void loadChecksums(bool require); + + /// Loads marks index granularity into memory + void loadIndexGranularity(); + + /// Loads index file. + void loadIndex(); + + /// Load rows count for this part from disk (for the newer storage format version). + /// For the older format version calculates rows count from the size of a column with a fixed size. + void loadRowsCount(); + + /// Loads ttl infos in json format from file ttl.txt. If file doesn`t exists assigns ttl infos with all zeros + void loadTTLInfos(); + + void loadPartitionAndMinMaxIndex(); + + ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const override; + + void checkConsistency(bool require_part_metadata); +}; + + +// using MergeTreeDataPartState =IMergeTreeDataPart::State; + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 99b4a49d111..8060159bf16 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -567,6 +567,13 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( BlockInputStreams res; + ReaderSettings reader_settings = + { + .min_bytes_to_use_direct_io = settings.min_bytes_to_use_direct_io, + .max_read_buffer_size = settings.max_read_buffer_size, + .save_marks_in_cache = true + }; + if (select.final()) { /// Add columns needed to calculate the sorting expression and the sign. @@ -588,7 +595,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( settings.use_uncompressed_cache, query_info, virt_column_names, - settings); + settings, + reader_settings); } else if (settings.optimize_read_in_order && query_info.sorting_info) { @@ -608,7 +616,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( query_info, sorting_key_prefix_expr, virt_column_names, - settings); + settings, + reader_settings); } else { @@ -620,7 +629,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( settings.use_uncompressed_cache, query_info, virt_column_names, - settings); + settings, + reader_settings); } if (use_sampling) @@ -666,7 +676,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( bool use_uncompressed_cache, const SelectQueryInfo & query_info, const Names & virt_columns, - const Settings & settings) const + const Settings & settings, + const ReaderSettings & reader_settings) const { /// Count marks for each part. std::vector sum_marks_in_parts(parts.size()); @@ -727,7 +738,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( res.emplace_back(std::make_shared( i, pool, min_marks_for_concurrent_read, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, data, use_uncompressed_cache, - query_info.prewhere_info, settings, virt_columns)); + query_info.prewhere_info, reader_settings, virt_columns)); if (i == 0) { @@ -803,8 +814,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( BlockInputStreamPtr source_stream = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, ranges_to_get_from_part, - use_uncompressed_cache, query_info.prewhere_info, true, settings.min_bytes_to_use_direct_io, - settings.max_read_buffer_size, true, virt_columns, part.part_index_in_query); + use_uncompressed_cache, query_info.prewhere_info, true, reader_settings, + virt_columns, part.part_index_in_query); res.push_back(source_stream); } @@ -826,7 +837,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO const SelectQueryInfo & query_info, const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, - const Settings & settings) const + const Settings & settings, + const ReaderSettings & reader_settings) const { size_t sum_marks = 0; SortingInfoPtr sorting_info = query_info.sorting_info; @@ -988,16 +1000,16 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO source_stream = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, ranges_to_get_from_part, - use_uncompressed_cache, query_info.prewhere_info, true, settings.min_bytes_to_use_direct_io, - settings.max_read_buffer_size, true, virt_columns, part.part_index_in_query); + use_uncompressed_cache, query_info.prewhere_info, true, reader_settings, + virt_columns, part.part_index_in_query); } else { source_stream = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, ranges_to_get_from_part, - use_uncompressed_cache, query_info.prewhere_info, true, settings.min_bytes_to_use_direct_io, - settings.max_read_buffer_size, true, virt_columns, part.part_index_in_query); + use_uncompressed_cache, query_info.prewhere_info, true, reader_settings, + virt_columns, part.part_index_in_query); source_stream = std::make_shared(source_stream); } @@ -1033,7 +1045,8 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal bool use_uncompressed_cache, const SelectQueryInfo & query_info, const Names & virt_columns, - const Settings & settings) const + const Settings & settings, + const ReaderSettings & reader_settings) const { const auto data_settings = data.getSettings(); size_t sum_marks = 0; @@ -1071,7 +1084,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal BlockInputStreamPtr source_stream = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, part.ranges, use_uncompressed_cache, - query_info.prewhere_info, true, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size, true, + query_info.prewhere_info, true, reader_settings, virt_columns, part.part_index_in_query); to_merge.emplace_back(std::make_shared(source_stream, data.sorting_key_expr)); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 44857799d01..61b4a59b363 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -54,7 +54,8 @@ private: bool use_uncompressed_cache, const SelectQueryInfo & query_info, const Names & virt_columns, - const Settings & settings) const; + const Settings & settings, + const ReaderSettings & reader_settings) const; BlockInputStreams spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, @@ -65,7 +66,8 @@ private: const SelectQueryInfo & query_info, const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, - const Settings & settings) const; + const Settings & settings, + const ReaderSettings & reader_settings) const; BlockInputStreams spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, @@ -74,7 +76,8 @@ private: bool use_uncompressed_cache, const SelectQueryInfo & query_info, const Names & virt_columns, - const Settings & settings) const; + const Settings & settings, + const ReaderSettings & reader_settings) const; /// Get the approximate value (bottom estimate - only by full marks) of the number of rows falling under the index. size_t getApproximateTotalRowsToRead( diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index e29ae01b3b3..b1589f5a3e0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -75,7 +76,7 @@ void buildScatterSelector( } /// Computes ttls and updates ttl infos -void updateTTL(const MergeTreeData::TTLEntry & ttl_entry, MergeTreeDataPart::TTLInfos & ttl_infos, Block & block, const String & column_name) +void updateTTL(const MergeTreeData::TTLEntry & ttl_entry,IMergeTreeDataPart::TTLInfos & ttl_infos, Block & block, const String & column_name) { if (!block.has(ttl_entry.result_column)) ttl_entry.expression->execute(block); @@ -173,7 +174,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// This will generate unique name in scope of current server process. Int64 temp_index = data.insert_increment.get(); - MergeTreeDataPart::MinMaxIndex minmax_idx; + IMergeTreeDataPart::MinMaxIndex minmax_idx; minmax_idx.update(block, data.minmax_idx_columns); MergeTreePartition partition(std::move(block_with_partition.partition)); @@ -204,11 +205,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa MergeTreeData::MutableDataPartPtr new_data_part = - std::make_shared(data, reservation->getDisk(), part_name, new_part_info); + createPart(data, reservation->getDisk(), part_name, new_part_info, TMP_PREFIX + part_name); new_data_part->partition = std::move(partition); new_data_part->minmax_idx = std::move(minmax_idx); - new_data_part->relative_path = TMP_PREFIX + part_name; new_data_part->is_temp = true; /// The name could be non-unique in case of stale files from previous runs. diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h index ff391be596c..5a006855632 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -5,7 +5,7 @@ namespace DB { -/// Class contains information about index granularity in rows of MergeTreeDataPart +/// Class contains information about index granularity in rows ofIMergeTreeDataPart /// Inside it contains vector of partial sums of rows after mark: /// |-----|---|----|----| /// | 5 | 8 | 12 | 16 | diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 05f09041fed..ce852d1fa46 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -8,8 +8,9 @@ MergeTreeIndexReader::MergeTreeIndexReader( MergeTreeIndexPtr index_, MergeTreeData::DataPartPtr part_, size_t marks_count_, const MarkRanges & all_mark_ranges_) : index(index_), stream( part_->getFullPath() + index->getFileName(), ".idx", marks_count_, - all_mark_ranges_, nullptr, false, nullptr, - part_->getFileSizeOrZero(index->getFileName() + ".idx"), 0, DBMS_DEFAULT_BUFFER_SIZE, + all_mark_ranges_, + { 0, DBMS_DEFAULT_BUFFER_SIZE, false}, nullptr, nullptr, + part_->getFileSizeOrZero(index->getFileName() + ".idx"), &part_->index_granularity_info, ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE) { diff --git a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp index d26cc7fb627..8273a915237 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartition.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp index 9076c053900..7234ac6743a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -1,5 +1,7 @@ #include #include +#include + #include #include @@ -76,7 +78,7 @@ bool MergeTreePartsMover::selectPartsForMove( const AllowedMovingPredicate & can_move, const std::lock_guard & /* moving_parts_lock */) { - MergeTreeData::DataPartsVector data_parts = data->getDataPartsVector(); + auto data_parts = data->getDataPartsVector(); if (data_parts.empty()) return false; @@ -108,7 +110,7 @@ bool MergeTreePartsMover::selectPartsForMove( /// Don't report message to log, because logging is excessive if (!can_move(part, &reason)) continue; - + auto to_insert = need_to_move.find(part->disk); if (to_insert != need_to_move.end()) to_insert->second.add(part); @@ -143,8 +145,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt moving_part.part->makeCloneOnDiskDetached(moving_part.reserved_space); MergeTreeData::MutableDataPartPtr cloned_part = - std::make_shared(*data, moving_part.reserved_space->getDisk(), moving_part.part->name); - cloned_part->relative_path = "detached/" + moving_part.part->name; + createPart(*data, moving_part.reserved_space->getDisk(), moving_part.part->name, "detached/" + moving_part.part->name); LOG_TRACE(log, "Part " << moving_part.part->name << " was cloned to " << cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.h b/dbms/src/Storages/MergeTree/MergeTreePartsMover.h index 5a3f8e2cefc..f931c4dc3b7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include @@ -15,10 +15,10 @@ namespace DB /// it have to be moved. struct MergeTreeMoveEntry { - std::shared_ptr part; + std::shared_ptr part; DiskSpace::ReservationPtr reserved_space; - MergeTreeMoveEntry(const std::shared_ptr & part_, DiskSpace::ReservationPtr reservation_) + MergeTreeMoveEntry(const std::shared_ptr & part_, DiskSpace::ReservationPtr reservation_) : part(part_), reserved_space(std::move(reservation_)) { } @@ -33,7 +33,7 @@ class MergeTreePartsMover { private: /// Callback tells that part is not participating in background process - using AllowedMovingPredicate = std::function &, String * reason)>; + using AllowedMovingPredicate = std::function &, String * reason)>; public: MergeTreePartsMover(MergeTreeData * data_) @@ -50,14 +50,14 @@ public: const std::lock_guard & moving_parts_lock); /// Copies part to selected reservation in detached folder. Throws exception if part alredy exists. - std::shared_ptr clonePart(const MergeTreeMoveEntry & moving_part) const; + std::shared_ptr clonePart(const MergeTreeMoveEntry & moving_part) const; /// Replaces cloned part from detached directory into active data parts set. /// Replacing part changes state to DeleteOnDestroy and will be removed from disk after destructor of - /// MergeTreeDataPart called. If replacing part doesn't exists or not active (commited) than + ///IMergeTreeDataPart called. If replacing part doesn't exists or not active (commited) than /// cloned part will be removed and loge message will be reported. It may happen in case of concurrent /// merge or mutation. - void swapClonedPart(const std::shared_ptr & cloned_parts) const; + void swapClonedPart(const std::shared_ptr & cloned_parts) const; public: /// Can stop background moves and moves from queries diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 932721eb028..84f9ee04299 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -13,7 +13,7 @@ namespace DB { MergeTreeRangeReader::DelayedStream::DelayedStream( - size_t from_mark, MergeTreeReader * merge_tree_reader_) + size_t from_mark, IMergeTreeReader * merge_tree_reader_) : current_mark(from_mark), current_offset(0), num_delayed_rows(0) , merge_tree_reader(merge_tree_reader_) , index_granularity(&(merge_tree_reader->data_part->index_granularity)) @@ -108,7 +108,7 @@ size_t MergeTreeRangeReader::DelayedStream::finalize(Block & block) MergeTreeRangeReader::Stream::Stream( - size_t from_mark, size_t to_mark, MergeTreeReader * merge_tree_reader_) + size_t from_mark, size_t to_mark, IMergeTreeReader * merge_tree_reader_) : current_mark(from_mark), offset_after_current_mark(0) , last_mark(to_mark) , merge_tree_reader(merge_tree_reader_) @@ -406,7 +406,7 @@ void MergeTreeRangeReader::ReadResult::setFilter(const ColumnPtr & new_filter) MergeTreeRangeReader::MergeTreeRangeReader( - MergeTreeReader * merge_tree_reader_, MergeTreeRangeReader * prev_reader_, + IMergeTreeReader * merge_tree_reader_, MergeTreeRangeReader * prev_reader_, ExpressionActionsPtr alias_actions_, ExpressionActionsPtr prewhere_actions_, const String * prewhere_column_name_, const Names * ordered_names_, bool always_reorder_, bool remove_prewhere_column_, bool last_reader_in_chain_) diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h index 0eae69ee17e..e0161cc2421 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -11,7 +11,7 @@ template class ColumnVector; using ColumnUInt8 = ColumnVector; -class MergeTreeReader; +class IMergeTreeReader; class MergeTreeIndexGranularity; /// MergeTreeReader iterator which allows sequential reading for arbitrary number of rows between pairs of marks in the same part. @@ -20,7 +20,7 @@ class MergeTreeIndexGranularity; class MergeTreeRangeReader { public: - MergeTreeRangeReader(MergeTreeReader * merge_tree_reader_, MergeTreeRangeReader * prev_reader_, + MergeTreeRangeReader(IMergeTreeReader * merge_tree_reader_, MergeTreeRangeReader * prev_reader_, ExpressionActionsPtr alias_actions_, ExpressionActionsPtr prewhere_actions_, const String * prewhere_column_name_, const Names * ordered_names_, bool always_reorder_, bool remove_prewhere_column_, bool last_reader_in_chain_); @@ -41,7 +41,7 @@ public: { public: DelayedStream() = default; - DelayedStream(size_t from_mark, MergeTreeReader * merge_tree_reader); + DelayedStream(size_t from_mark, IMergeTreeReader * merge_tree_reader); /// Read @num_rows rows from @from_mark starting from @offset row /// Returns the number of rows added to block. @@ -62,7 +62,7 @@ public: size_t num_delayed_rows = 0; /// Actual reader of data from disk - MergeTreeReader * merge_tree_reader = nullptr; + IMergeTreeReader * merge_tree_reader = nullptr; const MergeTreeIndexGranularity * index_granularity = nullptr; bool continue_reading = false; bool is_finished = true; @@ -78,7 +78,7 @@ public: { public: Stream() = default; - Stream(size_t from_mark, size_t to_mark, MergeTreeReader * merge_tree_reader); + Stream(size_t from_mark, size_t to_mark, IMergeTreeReader * merge_tree_reader); /// Returns the number of rows added to block. size_t read(Block & block, size_t num_rows, bool skip_remaining_rows_in_current_granule); @@ -103,7 +103,7 @@ public: size_t last_mark = 0; - MergeTreeReader * merge_tree_reader = nullptr; + IMergeTreeReader * merge_tree_reader = nullptr; const MergeTreeIndexGranularity * index_granularity = nullptr; size_t current_mark_index_granularity = 0; @@ -191,7 +191,7 @@ private: void executePrewhereActionsAndFilterColumns(ReadResult & result); void filterBlock(Block & block, const IColumn::Filter & filter) const; - MergeTreeReader * merge_tree_reader = nullptr; + IMergeTreeReader * merge_tree_reader = nullptr; const MergeTreeIndexGranularity * index_granularity = nullptr; MergeTreeRangeReader * prev_reader = nullptr; /// If not nullptr, read from prev_reader firstly. diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index 6298c098220..7093bef21e4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -33,7 +33,7 @@ MergeTreeReadPool::MergeTreeReadPool( for (auto & part_ranges : parts_ranges) std::reverse(std::begin(part_ranges.ranges), std::end(part_ranges.ranges)); - /// parts don't contain duplicate MergeTreeDataPart's. + /// parts don't contain duplicateIMergeTreeDataPart's. const auto per_part_sum_marks = fillPerPartInfo(parts_, check_columns_); fillPerThreadInfo(threads_, sum_marks_, per_part_sum_marks, parts_, min_marks_for_concurrent_read_); } @@ -129,7 +129,7 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(const size_t min_marks_to_read, prewhere_info && prewhere_info->remove_prewhere_column, per_part_should_reorder[part_idx], std::move(curr_task_size_predictor)); } -MarkRanges MergeTreeReadPool::getRestMarks(const MergeTreeDataPart & part, const MarkRange & from) const +MarkRanges MergeTreeReadPool::getRestMarks(const IMergeTreeDataPart & part, const MarkRange & from) const { MarkRanges all_part_ranges; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h index 2e9cb76f0cd..aa3b81aafe9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.h @@ -81,7 +81,7 @@ public: void profileFeedback(const ReadBufferFromFileBase::ProfileInfo info); /// This method tells which mark ranges we have to read if we start from @from mark range - MarkRanges getRestMarks(const MergeTreeDataPart & part, const MarkRange & from) const; + MarkRanges getRestMarks(const IMergeTreeDataPart & part, const MarkRange & from) const; Block getHeader() const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h new file mode 100644 index 00000000000..45038debbe9 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -0,0 +1,42 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +/// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. +/// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. +/// Avoids loading the marks file if it is not needed (e.g. when reading the whole part). +class MergeTreeReaderCompact : public IMergeTreeReader +{ +public: + MergeTreeReaderCompact(const MergeTreeData::DataPartPtr & data_part_, + const NamesAndTypesList & columns_, + UncompressedCache * uncompressed_cache_, + MarkCache * mark_cache_, + const MarkRanges & mark_ranges_, + const ReaderSettings & settings_, + const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}, + clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); + + /// Return the number of rows has been read or zero if there is no columns to read. + /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark + size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) override; + +private: + ReadBuffer * data_buffer; + + using Offsets = std::vector; + Offsets offsets; + + /// Columns that are read. + + friend class MergeTreeRangeReader::DelayedStream; +}; + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h new file mode 100644 index 00000000000..e0332ce3d3a --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h @@ -0,0 +1,13 @@ +#pragma once +#include + +namespace DB +{ + struct ReaderSettings + { + size_t min_bytes_to_use_direct_io = 0; + size_t max_read_buffer_size = 0; + bool save_marks_in_cache = false; + }; +} + diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 3dbfc61a00b..f7f158f7d3e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -17,13 +17,13 @@ namespace ErrorCodes MergeTreeReaderStream::MergeTreeReaderStream( const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, - MarkCache * mark_cache_, bool save_marks_in_cache_, - UncompressedCache * uncompressed_cache, - size_t file_size, size_t aio_threshold, size_t max_read_buffer_size, + const ReaderSettings & settings, + MarkCache * mark_cache_, + UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type) : path_prefix(path_prefix_), data_file_extension(data_file_extension_), marks_count(marks_count_) - , mark_cache(mark_cache_), save_marks_in_cache(save_marks_in_cache_) + , mark_cache(mark_cache_), save_marks_in_cache(settings.save_marks_in_cache) , index_granularity_info(index_granularity_info_) { /// Compute the size of the buffer. @@ -71,15 +71,15 @@ MergeTreeReaderStream::MergeTreeReaderStream( /// Avoid empty buffer. May happen while reading dictionary for DataTypeLowCardinality. /// For example: part has single dictionary and all marks point to the same position. if (max_mark_range_bytes == 0) - max_mark_range_bytes = max_read_buffer_size; + max_mark_range_bytes = settings.max_read_buffer_size; - size_t buffer_size = std::min(max_read_buffer_size, max_mark_range_bytes); + size_t buffer_size = std::min(settings.max_read_buffer_size, max_mark_range_bytes); /// Initialize the objects that shall be used to perform read operations. if (uncompressed_cache) { auto buffer = std::make_unique( - path_prefix + data_file_extension, uncompressed_cache, sum_mark_range_bytes, aio_threshold, buffer_size); + path_prefix + data_file_extension, uncompressed_cache, sum_mark_range_bytes, settings.min_bytes_to_use_direct_io, buffer_size); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); @@ -90,7 +90,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( else { auto buffer = std::make_unique( - path_prefix + data_file_extension, sum_mark_range_bytes, aio_threshold, buffer_size); + path_prefix + data_file_extension, sum_mark_range_bytes, settings.min_bytes_to_use_direct_io, buffer_size); if (profile_callback) buffer->setProfileCallback(profile_callback, clock_type); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index d60689f1e91..85ab78dde9f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -17,10 +17,9 @@ public: MergeTreeReaderStream( const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, - MarkCache * mark_cache, bool save_marks_in_cache, - UncompressedCache * uncompressed_cache, - size_t file_size, size_t aio_threshold, size_t max_read_buffer_size, - const MergeTreeIndexGranularityInfo * index_granularity_info_, + const ReaderSettings & settings_, + MarkCache * mark_cache, UncompressedCache * uncompressed_cache, + size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); void seekToMark(size_t index); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp similarity index 54% rename from dbms/src/Storages/MergeTree/MergeTreeReader.cpp rename to dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index d9732c8ac6f..dcc12e2e800 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include #include #include @@ -27,20 +27,14 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; } - -MergeTreeReader::~MergeTreeReader() = default; - - -MergeTreeReader::MergeTreeReader(const String & path_, - const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, - UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, bool save_marks_in_cache_, - const MergeTreeData & storage_, const MarkRanges & all_mark_ranges_, - size_t aio_threshold_, size_t max_read_buffer_size_, const ValueSizeMap & avg_value_size_hints_, +MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data_part_, + const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, + const MarkRanges & mark_ranges_, const ReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) - : data_part(data_part_), avg_value_size_hints(avg_value_size_hints_), path(path_), columns(columns_) - , uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_), save_marks_in_cache(save_marks_in_cache_), storage(storage_) - , all_mark_ranges(all_mark_ranges_), aio_threshold(aio_threshold_), max_read_buffer_size(max_read_buffer_size_) + : IMergeTreeReader(data_part_, columns_ + , uncompressed_cache_, mark_cache_, mark_ranges_ + , settings_ , avg_value_size_hints_) { try { @@ -54,14 +48,7 @@ MergeTreeReader::MergeTreeReader(const String & path_, } } - -const MergeTreeReader::ValueSizeMap & MergeTreeReader::getAvgValueSizeHints() const -{ - return avg_value_size_hints; -} - - -size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) +size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) { size_t read_rows = 0; try @@ -150,7 +137,7 @@ size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t return read_rows; } -void MergeTreeReader::addStreams(const String & name, const IDataType & type, +void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type) { IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path) @@ -170,9 +157,8 @@ void MergeTreeReader::addStreams(const String & name, const IDataType & type, streams.emplace(stream_name, std::make_unique( path + stream_name, DATA_FILE_EXTENSION, data_part->getMarksCount(), - all_mark_ranges, mark_cache, save_marks_in_cache, + all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), - aio_threshold, max_read_buffer_size, &data_part->index_granularity_info, profile_callback, clock_type)); }; @@ -182,7 +168,7 @@ void MergeTreeReader::addStreams(const String & name, const IDataType & type, } -void MergeTreeReader::readData( +void MergeTreeReaderWide::readData( const String & name, const IDataType & type, IColumn & column, size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) @@ -232,147 +218,4 @@ void MergeTreeReader::readData( IDataType::updateAvgValueSizeHint(column, avg_value_size_hint); } - -static bool arrayHasNoElementsRead(const IColumn & column) -{ - const ColumnArray * column_array = typeid_cast(&column); - - if (!column_array) - return false; - - size_t size = column_array->size(); - if (!size) - return false; - - size_t data_size = column_array->getData().size(); - if (data_size) - return false; - - size_t last_offset = column_array->getOffsets()[size - 1]; - return last_offset != 0; -} - - -void MergeTreeReader::fillMissingColumns(Block & res, bool & should_reorder, bool & should_evaluate_missing_defaults, size_t num_rows) -{ - try - { - /// For a missing column of a nested data structure we must create not a column of empty - /// arrays, but a column of arrays of correct length. - - /// First, collect offset columns for all arrays in the block. - OffsetColumns offset_columns; - for (size_t i = 0; i < res.columns(); ++i) - { - const ColumnWithTypeAndName & column = res.safeGetByPosition(i); - - if (const ColumnArray * array = typeid_cast(column.column.get())) - { - String offsets_name = Nested::extractTableName(column.name); - auto & offsets_column = offset_columns[offsets_name]; - - /// If for some reason multiple offsets columns are present for the same nested data structure, - /// choose the one that is not empty. - if (!offsets_column || offsets_column->empty()) - offsets_column = array->getOffsetsPtr(); - } - } - - should_evaluate_missing_defaults = false; - should_reorder = false; - - /// insert default values only for columns without default expressions - for (const auto & requested_column : columns) - { - bool has_column = res.has(requested_column.name); - if (has_column) - { - const auto & col = *res.getByName(requested_column.name).column; - if (arrayHasNoElementsRead(col)) - { - res.erase(requested_column.name); - has_column = false; - } - } - - if (!has_column) - { - should_reorder = true; - if (storage.getColumns().hasDefault(requested_column.name)) - { - should_evaluate_missing_defaults = true; - continue; - } - - ColumnWithTypeAndName column_to_add; - column_to_add.name = requested_column.name; - column_to_add.type = requested_column.type; - - String offsets_name = Nested::extractTableName(column_to_add.name); - if (offset_columns.count(offsets_name)) - { - ColumnPtr offsets_column = offset_columns[offsets_name]; - DataTypePtr nested_type = typeid_cast(*column_to_add.type).getNestedType(); - size_t nested_rows = typeid_cast(*offsets_column).getData().back(); - - ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(nested_rows)->convertToFullColumnIfConst(); - - column_to_add.column = ColumnArray::create(nested_column, offsets_column); - } - else - { - /// We must turn a constant column into a full column because the interpreter could infer that it is constant everywhere - /// but in some blocks (from other parts) it can be a full column. - column_to_add.column = column_to_add.type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst(); - } - - res.insert(std::move(column_to_add)); - } - } - } - catch (Exception & e) - { - /// Better diagnostics. - e.addMessage("(while reading from part " + path + ")"); - throw; - } -} - -void MergeTreeReader::reorderColumns(Block & res, const Names & ordered_names, const String * filter_name) -{ - try - { - Block ordered_block; - - for (const auto & name : ordered_names) - if (res.has(name)) - ordered_block.insert(res.getByName(name)); - - if (filter_name && !ordered_block.has(*filter_name) && res.has(*filter_name)) - ordered_block.insert(res.getByName(*filter_name)); - - std::swap(res, ordered_block); - } - catch (Exception & e) - { - /// Better diagnostics. - e.addMessage("(while reading from part " + path + ")"); - throw; - } -} - -void MergeTreeReader::evaluateMissingDefaults(Block & res) -{ - try - { - DB::evaluateMissingDefaults(res, columns, storage.getColumns().getDefaults(), storage.global_context); - } - catch (Exception & e) - { - /// Better diagnostics. - e.addMessage("(while reading from part " + path + ")"); - throw; - } -} - } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h new file mode 100644 index 00000000000..782f79ead56 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +/// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. +/// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. +/// Avoids loading the marks file if it is not needed (e.g. when reading the whole part). +class MergeTreeReaderWide : public IMergeTreeReader +{ +public: + MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data_part_, + const NamesAndTypesList & columns_, + UncompressedCache * uncompressed_cache_, + MarkCache * mark_cache_, + const MarkRanges & mark_ranges_, + const ReaderSettings & settings_, + const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}, + clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); + + /// Return the number of rows has been read or zero if there is no columns to read. + /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark + size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) override; + +private: + using FileStreams = std::map>; + + FileStreams streams; + + /// Columns that are read. + + void addStreams(const String & name, const IDataType & type, + const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); + + void readData( + const String & name, const IDataType & type, IColumn & column, + size_t from_mark, bool continue_reading, size_t max_rows_to_read, + bool read_offsets = true); + + friend class MergeTreeRangeReader::DelayedStream; +}; + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp index 9b78517e742..2f991a53b01 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include @@ -24,16 +24,14 @@ MergeTreeReverseSelectBlockInputStream::MergeTreeReverseSelectBlockInputStream( bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, bool check_columns, - size_t min_bytes_to_use_direct_io_, - size_t max_read_buffer_size_, - bool save_marks_in_cache_, + const ReaderSettings & reader_settings_, const Names & virt_column_names_, size_t part_index_in_query_, bool quiet) : MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, min_bytes_to_use_direct_io_, - max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names_}, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, + use_uncompressed_cache_, virt_column_names_}, required_columns{required_columns_}, data_part{owned_data_part_}, part_columns_lock(data_part->columns_lock), @@ -89,19 +87,14 @@ MergeTreeReverseSelectBlockInputStream::MergeTreeReverseSelectBlockInputStream( owned_mark_cache = storage.global_context.getMarkCache(); - reader = std::make_unique( - path, data_part, task_columns.columns, owned_uncompressed_cache.get(), - owned_mark_cache.get(), save_marks_in_cache, storage, - all_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size); + reader = data_part->getReader(task_columns.columns, all_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); if (prewhere_info) - pre_reader = std::make_unique( - path, data_part, task_columns.pre_columns, owned_uncompressed_cache.get(), - owned_mark_cache.get(), save_marks_in_cache, storage, - all_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size); + pre_reader = data_part->getReader(task_columns.pre_columns, all_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); } - Block MergeTreeReverseSelectBlockInputStream::getHeader() const { return header; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h index 40af5d5d92a..e7abeb7a933 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h @@ -27,9 +27,7 @@ public: bool use_uncompressed_cache, const PrewhereInfoPtr & prewhere_info, bool check_columns, - size_t min_bytes_to_use_direct_io, - size_t max_read_buffer_size, - bool save_marks_in_cache, + const ReaderSettings & reader_settings, const Names & virt_column_names = {}, size_t part_index_in_query = 0, bool quiet = false); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp index 7a6e6f197dd..d3763e30dd3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp @@ -1,6 +1,6 @@ #include +#include #include -#include #include @@ -24,16 +24,14 @@ MergeTreeSelectBlockInputStream::MergeTreeSelectBlockInputStream( bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, bool check_columns_, - size_t min_bytes_to_use_direct_io_, - size_t max_read_buffer_size_, - bool save_marks_in_cache_, + const ReaderSettings & reader_settings_, const Names & virt_column_names_, size_t part_index_in_query_, bool quiet) : MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, min_bytes_to_use_direct_io_, - max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names_}, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, + reader_settings_, use_uncompressed_cache_, virt_column_names_}, required_columns{required_columns_}, data_part{owned_data_part_}, part_columns_lock(data_part->columns_lock), @@ -126,16 +124,12 @@ try owned_mark_cache = storage.global_context.getMarkCache(); - reader = std::make_unique( - path, data_part, task_columns.columns, owned_uncompressed_cache.get(), - owned_mark_cache.get(), save_marks_in_cache, storage, - all_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size); + reader = data_part->getReader(task_columns.columns, all_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); if (prewhere_info) - pre_reader = std::make_unique( - path, data_part, task_columns.pre_columns, owned_uncompressed_cache.get(), - owned_mark_cache.get(), save_marks_in_cache, storage, - all_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size); + pre_reader = data_part->getReader(task_columns.pre_columns, all_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); } return true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h index 0fc9830f5d0..b3ebf8b2d7e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h @@ -27,9 +27,7 @@ public: bool use_uncompressed_cache, const PrewhereInfoPtr & prewhere_info, bool check_columns, - size_t min_bytes_to_use_direct_io, - size_t max_read_buffer_size, - bool save_marks_in_cache, + const ReaderSettings & reader_settings, const Names & virt_column_names = {}, size_t part_index_in_query = 0, bool quiet = false); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 74cff479e5f..6051b1c36d0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -51,13 +51,17 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( columns_for_reader = data_part->columns.addTypes(columns_to_read); } - reader = std::make_unique( - data_part->getFullPath(), data_part, columns_for_reader, /* uncompressed_cache = */ nullptr, - mark_cache.get(), /* save_marks_in_cache = */ false, storage, + ReaderSettings reader_settings = + { + /// This is hack + .min_bytes_to_use_direct_io = read_with_direct_io ? 1UL : std::numeric_limits::max(), + .max_read_buffer_size = DBMS_DEFAULT_BUFFER_SIZE, + .save_marks_in_cache = false + }; + + reader = data_part->getReader(columns_for_reader, MarkRanges{MarkRange(0, data_part->getMarksCount())}, - /* bytes to use AIO (this is hack) */ - read_with_direct_io ? 1UL : std::numeric_limits::max(), - DBMS_DEFAULT_BUFFER_SIZE); + /* uncompressed_cache = */ nullptr, mark_cache.get(), reader_settings); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.h index aab1b89d31f..35d6b4dfd27 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.h @@ -1,7 +1,7 @@ #pragma once #include #include -#include +#include #include #include @@ -58,7 +58,7 @@ private: Logger * log = &Logger::get("MergeTreeSequentialBlockInputStream"); std::shared_ptr mark_cache; - using MergeTreeReaderPtr = std::unique_ptr; + using MergeTreeReaderPtr = std::unique_ptr; MergeTreeReaderPtr reader; /// current mark at which we stop reading diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp index 69cf173212d..36a0b194a58 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -17,12 +17,12 @@ MergeTreeThreadSelectBlockInputStream::MergeTreeThreadSelectBlockInputStream( const MergeTreeData & storage_, const bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - const Settings & settings, + const ReaderSettings & reader_settings_, const Names & virt_column_names_) : MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, settings.min_bytes_to_use_direct_io, - settings.max_read_buffer_size, use_uncompressed_cache_, true, virt_column_names_}, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, + reader_settings_, use_uncompressed_cache_, virt_column_names_}, thread{thread_}, pool{pool_} { @@ -80,15 +80,14 @@ bool MergeTreeThreadSelectBlockInputStream::getNewTask() owned_uncompressed_cache = storage.global_context.getUncompressedCache(); owned_mark_cache = storage.global_context.getMarkCache(); - reader = std::make_unique( - path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, rest_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, MergeTreeReader::ValueSizeMap{}, profile_callback); + reader = task->data_part->getReader(task->columns, rest_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, + IMergeTreeReader::ValueSizeMap{}, profile_callback); if (prewhere_info) - pre_reader = std::make_unique( - path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, rest_mark_ranges, min_bytes_to_use_direct_io, - max_read_buffer_size, MergeTreeReader::ValueSizeMap{}, profile_callback); + pre_reader = task->data_part->getReader(task->pre_columns, rest_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, + IMergeTreeReader::ValueSizeMap{}, profile_callback); } else { @@ -97,18 +96,17 @@ bool MergeTreeThreadSelectBlockInputStream::getNewTask() { auto rest_mark_ranges = pool->getRestMarks(*task->data_part, task->mark_ranges[0]); /// retain avg_value_size_hints - reader = std::make_unique( - path, task->data_part, task->columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, rest_mark_ranges, min_bytes_to_use_direct_io, max_read_buffer_size, + reader = task->data_part->getReader(task->columns, rest_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, reader->getAvgValueSizeHints(), profile_callback); if (prewhere_info) - pre_reader = std::make_unique( - path, task->data_part, task->pre_columns, owned_uncompressed_cache.get(), owned_mark_cache.get(), save_marks_in_cache, - storage, rest_mark_ranges, min_bytes_to_use_direct_io, - max_read_buffer_size, pre_reader->getAvgValueSizeHints(), profile_callback); + pre_reader = task->data_part->getReader(task->pre_columns, rest_mark_ranges, + owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, + reader->getAvgValueSizeHints(), profile_callback); } } + last_readed_part_path = path; return true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h index 3c7dfb7927d..e47513909e7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h @@ -24,7 +24,7 @@ public: const MergeTreeData & storage_, const bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - const Settings & settings_, + const ReaderSettings & reader_settings_, const Names & virt_column_names_); String getName() const override { return "MergeTreeThread"; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeWriter.h b/dbms/src/Storages/MergeTree/MergeTreeWriter.h new file mode 100644 index 00000000000..5176b153a6d --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeWriter.h @@ -0,0 +1,13 @@ +#include + +namespace DB +{ + +class MergeTreeWriter +{ + +}; + +using MergeTreeWriterPtr = std::shared_ptr; + +} diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 0e116969f31..6bb78b07f2b 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -2,7 +2,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQuorumAddedParts.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQuorumAddedParts.h index dc251a02dcc..4c45681c22c 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQuorumAddedParts.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQuorumAddedParts.h @@ -8,7 +8,7 @@ #include #include -#include +#include namespace DB { diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 0e15a5660a9..2c2a0748cfc 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include #include @@ -12,11 +12,11 @@ namespace DB { /// A Storage that allows reading from a single MergeTree data part. -class StorageFromMergeTreeDataPart : public ext::shared_ptr_helper, public IStorage +class StorageFroIMergeTreeDataPart : public ext::shared_ptr_helper, public IStorage { - friend struct ext::shared_ptr_helper; + friend struct ext::shared_ptr_helper; public: - String getName() const override { return "FromMergeTreeDataPart"; } + String getName() const override { return "FroIMergeTreeDataPart"; } String getTableName() const override { return part->storage.getTableName() + " (part " + part->name + ")"; } String getDatabaseName() const override { return part->storage.getDatabaseName(); } @@ -40,7 +40,7 @@ public: } protected: - StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) + StorageFroIMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(part_->storage.getVirtuals()), part(part_) { setColumns(part_->storage.getColumns()); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index c55378d8526..be0f1fbb7ee 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include #include @@ -2332,7 +2332,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( } -bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const MergeTreeDataPart & part, Int64 mutation_version) +bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const IMergeTreeDataPart & part, Int64 mutation_version) { auto zookeeper = getZooKeeper(); @@ -2691,7 +2691,7 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Strin { const auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); - if (auto part = getPartIfExists(part_info, {MergeTreeDataPart::State::Outdated, MergeTreeDataPart::State::Deleting})) + if (auto part = getPartIfExists(part_info, {IMergeTreeDataPart::State::Outdated, IMergeTreeDataPart::State::Deleting})) { LOG_DEBUG(log, "Part " << part->name << " should be deleted after previous attempt before fetch"); /// Force immediate parts cleanup to delete the part that was left from the previous fetch attempt. diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index facdb1660f0..17ba543978c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -434,7 +434,7 @@ private: bool force_ttl, ReplicatedMergeTreeLogEntryData * out_log_entry = nullptr); - bool createLogEntryToMutatePart(const MergeTreeDataPart & part, Int64 mutation_version); + bool createLogEntryToMutatePart(const IMergeTreeDataPart & part, Int64 mutation_version); /// Exchange parts. diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp index 54626198373..35d6a0205c5 100644 --- a/dbms/src/Storages/System/StorageSystemParts.cpp +++ b/dbms/src/Storages/System/StorageSystemParts.cpp @@ -58,7 +58,7 @@ StorageSystemParts::StorageSystemParts(const std::string & name_) void StorageSystemParts::processNextStorage(MutableColumns & columns_, const StoragesInfo & info, bool has_state_column) { - using State = MergeTreeDataPart::State; + using State =IMergeTreeDataPart::State; MergeTreeData::DataPartStateVector all_parts_state; MergeTreeData::DataPartsVector all_parts; @@ -117,7 +117,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns_, const Sto MinimalisticDataPartChecksums helper; { - /// TODO: MergeTreeDataPart structure is too error-prone. + /// TODO:IMergeTreeDataPart structure is too error-prone. std::shared_lock lock(part->columns_lock); helper.computeTotalChecksums(part->checksums); } diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 11a14dac326..3aad49eb942 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -98,7 +98,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns_, co auto index_size_in_bytes = part->getIndexSizeInBytes(); auto index_size_in_allocated_bytes = part->getIndexSizeInAllocatedBytes(); - using State = MergeTreeDataPart::State; + using State =IMergeTreeDataPart::State; for (const auto & column : part->columns) From 21a252f0ceb4bfc85e0b1aefbedaced9911a0262 Mon Sep 17 00:00:00 2001 From: Clement Rodriguez Date: Fri, 11 Oct 2019 09:25:22 +0200 Subject: [PATCH 0037/2007] Fixed integration test --- .../configs/dictionaries/mysql_dict2.xml | 4 +-- .../test_dictionaries_mysql/test.py | 26 ++++++++++++------- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml b/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml index 024d13b14b2..91506481cc9 100644 --- a/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml +++ b/dbms/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict2.xml @@ -46,7 +46,7 @@ 3306 root clickhouse - test2
+ test3
true true @@ -82,7 +82,7 @@ 3306 root clickhouse - test2
+ test4
true true diff --git a/dbms/tests/integration/test_dictionaries_mysql/test.py b/dbms/tests/integration/test_dictionaries_mysql/test.py index 52f82cc9b39..78aeb747dba 100644 --- a/dbms/tests/integration/test_dictionaries_mysql/test.py +++ b/dbms/tests/integration/test_dictionaries_mysql/test.py @@ -23,7 +23,7 @@ create_table_mysql_template = """ """ create_clickhouse_dictionary_table_template = """ - CREATE TABLE `test`.`{}` (`id` Int32, `value` String) ENGINE = Dictionary({}) + CREATE TABLE `test`.`dict_table_{}` (`id` Int32, `value` String) ENGINE = Dictionary({}) ORDER BY `id` DESC SETTINGS index_granularity = 8192 """ @@ -31,10 +31,14 @@ create_clickhouse_dictionary_table_template = """ def started_cluster(): try: cluster.start() - instance.query("CREATE DATABASE IF NOT EXISTS test") - + # Create a MySQL database create_mysql_db(get_mysql_conn(), 'test') + + # Create database in ClickHouse + instance.query("CREATE DATABASE IF NOT EXISTS test") + + # Create database in ClickChouse using MySQL protocol (will be used for data insertion) instance.query("CREATE DATABASE clickhouse_mysql ENGINE = MySQL('mysql1:3306', 'test', 'root', 'clickhouse')") yield cluster @@ -49,18 +53,17 @@ def test_load_mysql_dictionaries(started_cluster): query("SYSTEM RELOAD DICTIONARIES") for n in range(0, 5): - # Create MySQL tables and fills them - prepare_mysql_table('test' + n) + # Create MySQL tables, fill them and create CH dict tables + prepare_tables('test' + n) - #Create Dictionary tables based on MySQL tables - query(create_clickhouse_dictionary_table_template.format('test' + n), 'dict' + n) - # Check dictionaries are loaded and have correct number of elements for n in range(0, 100): + # Force reload of dictionaries (each 10 iteration) if (n % 10) == 0: - # Force reload of dictionaries query("SYSTEM RELOAD DICTIONARIES") - assert query("SELECT count() FROM `test`.{}".format('test' + (n % 5))).rstrip() == '10000' + + # Check number of rows + assert query("SELECT count() FROM `dict_table_`.{}".format('test' + (n % 5))).rstrip() == '10000' def create_mysql_db(mysql_connection, name): with mysql_connection.cursor() as cursor: @@ -78,6 +81,9 @@ def prepare_mysql_table(table_name): query("INSERT INTO `clickhouse_mysql`.{}(id, value) select number, concat('{} value ', toString(number)) from numbers(10000) ".format(table_name, table_name)) assert query("SELECT count() FROM `clickhouse_mysql`.{}".format(table_name)).rstrip() == '10000' mysql_connection.close() + + #Create CH Dictionary tables based on MySQL tables + query(create_clickhouse_dictionary_table_template.format(table_name + n), 'dict' + n) def get_mysql_conn(): conn = pymysql.connect(user='root', password='clickhouse', host='mysql1', port=3308) From 18163e4d7f51440bed695c2b0d31d49450c7f016 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Fri, 11 Oct 2019 18:37:16 +0300 Subject: [PATCH 0038/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 4 +- .../Storages/MergeTree/IMergeTreeDataPart.h | 4 ++ .../Storages/MergeTree/IMergeTreeReader.cpp | 26 +++++++ .../src/Storages/MergeTree/IMergeTreeReader.h | 6 ++ .../MergeTree/MergeTreeDataPartCompact.h | 7 ++ .../MergeTree/MergeTreeDataPartWide.cpp | 2 +- .../MergeTree/MergeTreeDataPartWide.h | 8 +++ .../MergeTreeIndexGranularityInfo.cpp | 32 ++++++--- .../MergeTree/MergeTreeIndexGranularityInfo.h | 14 ++-- .../MergeTree/MergeTreeReaderCompact.cpp | 68 +++++++++++++++++++ .../MergeTree/MergeTreeReaderCompact.h | 41 +++++++++-- .../MergeTree/MergeTreeReaderWide.cpp | 2 +- 12 files changed, 189 insertions(+), 25 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index ea6182c2fae..124638f6b52 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -144,7 +144,7 @@ IMergeTreeDataPart::IMergeTreeDataPart( , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) , disk(disk_) , relative_path(relative_path_.value_or(name_)) - , index_granularity_info(storage) {} + , index_granularity_info(shared_from_this()) {} IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, @@ -157,7 +157,7 @@ IMergeTreeDataPart::IMergeTreeDataPart( , info(info_) , disk(disk_) , relative_path(relative_path_.value_or(name_)) - , index_granularity_info(storage) {} + , index_granularity_info(shared_from_this()) {} ColumnSize IMergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index f30d72f45de..84b6eaf1760 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -84,6 +84,10 @@ public: /// If no checksums are present returns the name of the first physically existing column. virtual String getColumnNameWithMinumumCompressedSize() const = 0; + virtual String getMarkExtension(bool /* is_adaptive */) const { return ""; } + + virtual size_t getMarkSize(bool /* is_adaptive */) const { return 0; } + // virtual void detach() = 0; // virtual Checksums check( diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp index 75b01a22081..5c585b7ce60 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -65,6 +65,32 @@ static bool arrayHasNoElementsRead(const IColumn & column) return last_offset != 0; } +IMergeTreeReader::MarksPtr IMergeTreeReader::loadMarks(const String & mrk_path, const LoadFunc & load_func) +{ + MarksPtr marks; + if (mark_cache) + { + auto key = mark_cache->hash(mrk_path); + if (settings.save_marks_in_cache) + { + marks = mark_cache->getOrSet(key, load_func); + } + else + { + marks = mark_cache->get(key); + if (!marks) + marks = load_func(); + } + } + else + marks = load_func(); + + if (!marks) + throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); + + return marks; +} + void IMergeTreeReader::fillMissingColumns(Block & res, bool & should_reorder, bool & should_evaluate_missing_defaults, size_t num_rows) { diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 8ff463639e8..02cb8be0505 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -55,7 +55,13 @@ public: return all_mark_ranges.back().begin; } + using MarksPtr = MarkCache::MappedPtr; + protected: + + using LoadFunc = std::function; + MarksPtr loadMarks(const String & mrk_path, const LoadFunc & load_func); + /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index a6a2bf48dcf..6655c418428 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -58,6 +58,13 @@ public: bool isStoredOnDisk() const override { return true; } + String getMarkExtension(bool /* is_adaptive */) const override { return ".mrk3"; } + + bool getMarkSize(bool is_adaptive) + { + return sizeof(size_t) + columns.size() * sizeof(size_t) * 2; + } + void remove() const override; /// NOTE: Returns zeros if column files are not found in checksums. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index a187460e38c..3e77249b324 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -315,7 +315,7 @@ void MergeTreeDataPartWide::loadColumnsChecksumsIndexes(bool require_columns_che void MergeTreeDataPartWide::loadIndexGranularity() { String full_path = getFullPath(); - index_granularity_info.changeGranularityIfRequired(full_path); + index_granularity_info.changeGranularityIfRequired(shared_from_this()); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 5bad56a94ed..61007a4b955 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -62,6 +62,14 @@ public: bool supportsVerticalMerge() const override { return true; } + String getMarkExtension(bool is_adaptive) const override { return is_adaptive ? ".mrk2" : ".mrk"; } + + size_t getMarkSize(bool is_adaptive) const override + { + size_t nums = is_adaptive ? 3 : 2; + return sizeof(size_t) * nums; + } + /// NOTE: Returns zeros if column files are not found in checksums. /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. /// (either by locking columns_lock, or by locking table structure). diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index c25968480c8..73baee85569 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -7,6 +7,13 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(const std::string & path_to_part) const { if (Poco::File(path_to_part).exists()) @@ -22,39 +29,42 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( return {}; } -MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo( - const MergeTreeData & storage) +MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeDataPartPtr & part) { - const auto storage_settings = storage.getSettings(); + const auto storage_settings = part->storage.getSettings(); fixed_index_granularity = storage_settings->index_granularity; /// Granularity is fixed - if (!storage.canUseAdaptiveGranularity()) + if (!part->storage.canUseAdaptiveGranularity()) setNonAdaptive(); else setAdaptive(storage_settings->index_granularity_bytes); + + mark_size_in_bytes = part->getMarkSize(is_adaptive); + marks_file_extension = part->getMarkExtension(is_adaptive); } -void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const std::string & path_to_part) +void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const MergeTreeDataPartPtr & part) { - auto mrk_ext = getMrkExtensionFromFS(path_to_part); - if (mrk_ext && *mrk_ext == getNonAdaptiveMrkExtension()) + auto mrk_ext = getMrkExtensionFromFS(part->getFullPath()); + if (mrk_ext && *mrk_ext == ".mrk") /// TODO + { setNonAdaptive(); + mark_size_in_bytes = part->getMarkSize(is_adaptive); + marks_file_extension = part->getMarkExtension(is_adaptive); + } } void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_) { is_adaptive = true; - mark_size_in_bytes = getAdaptiveMrkSize(); - marks_file_extension = getAdaptiveMrkExtension(); index_granularity_bytes = index_granularity_bytes_; + } void MergeTreeIndexGranularityInfo::setNonAdaptive() { is_adaptive = false; - mark_size_in_bytes = getNonAdaptiveMrkSize(); - marks_file_extension = getNonAdaptiveMrkExtension(); index_granularity_bytes = 0; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index 0c203aca8ab..7a51f9031cc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -2,15 +2,20 @@ #include #include +// #include namespace DB { class MergeTreeData; +class IMergeTreeDataPart; + /// Meta information about index granularity struct MergeTreeIndexGranularityInfo { public: + using MergeTreeDataPartPtr = std::shared_ptr; + /// Marks file extension '.mrk' or '.mrk2' String marks_file_extension; @@ -26,14 +31,13 @@ public: /// Approximate bytes size of one granule size_t index_granularity_bytes; - MergeTreeIndexGranularityInfo( - const MergeTreeData & storage); + MergeTreeIndexGranularityInfo(const MergeTreeDataPartPtr & part); - void changeGranularityIfRequired(const std::string & path_to_part); + void changeGranularityIfRequired(const MergeTreeDataPartPtr & part); - String getMarksFilePath(const String & column_path) const + String getMarksFilePath(const String & path_prefix) const { - return column_path + marks_file_extension; + return path_prefix + marks_file_extension; } private: diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index e69de29bb2d..7049a4d2028 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -0,0 +1,68 @@ +#include +#include + +namespace DB +{ + +MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr & data_part_, + const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, + const MarkRanges & mark_ranges_, const ReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_) + : IMergeTreeReader(data_part_, columns_ + , uncompressed_cache_, mark_cache_, mark_ranges_ + , settings_, avg_value_size_hints_) +{ +} + +size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) +{ + UNUSED(from_mark); + UNUSED(continue_reading); + UNUSED(max_rows_to_read); + UNUSED(res); + + return 0; +} + +void MergeTreeReaderCompact::loadMarks() +{ + const auto & index_granularity_info = data_part->index_granularity_info; + size_t marks_count = data_part->getMarksCount(); + std::string mrk_path = index_granularity_info.getMarksFilePath(NAME_OF_FILE_WITH_DATA); + + auto load_func = [&]() -> MarkCache::MappedPtr + { + size_t file_size = Poco::File(mrk_path).getSize(); + size_t expected_file_size = index_granularity_info.mark_size_in_bytes * marks_count; + if (expected_file_size != file_size) + throw Exception( + "Bad size of marks file '" + mrk_path + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), + ErrorCodes::CORRUPTED_DATA); + + /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); + + auto res = std::make_shared(marks_count * columns.size()); + + ReadBufferFromFile buffer(mrk_path, file_size); + size_t i = 0; + + while (!buffer.eof()) + { + buffer.seek(sizeof(size_t)); + buffer.read(marks.getRowAddress(i), marks.getRowSize()); + ++i; + } + + if (i * index_granularity_info.mark_size_in_bytes != file_size) + throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); + + res->protect(); + return res; + }; + + auto marks_array = IMergeTreeReader::loadMarks(mrk_path, load_func); + marks = MarksInCompressedFileCompact(marks_array, columns.size()); +} + +} + diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 45038debbe9..4b4c5bde85c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -8,6 +8,36 @@ namespace DB { +class MarksInCompressedFileCompact +{ +public: + using MarksPtr = MarkCache::MappedPtr; + + MarksInCompressedFileCompact() = default; + + MarksInCompressedFileCompact(const MarksPtr & data_, size_t columns_num_) + : data(data_), columns_num(columns_num_) {} + + const MarkInCompressedFile & getMark(size_t index, size_t column) const + { + return (*data)[index * columns_num + column]; + } + + char * getRowAddress(size_t index) const + { + return reinterpret_cast(data->data() + index * columns_num); + } + + size_t getRowSize() const + { + return sizeof(MarkInCompressedFile) * columns_num; + } + +private: + MarksPtr data; + size_t columns_num; +}; + /// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. /// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. /// Avoids loading the marks file if it is not needed (e.g. when reading the whole part). @@ -20,9 +50,7 @@ public: MarkCache * mark_cache_, const MarkRanges & mark_ranges_, const ReaderSettings & settings_, - const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}, - clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); + const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}); /// Return the number of rows has been read or zero if there is no columns to read. /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark @@ -31,8 +59,11 @@ public: private: ReadBuffer * data_buffer; - using Offsets = std::vector; - Offsets offsets; + MarksInCompressedFileCompact marks; + + void loadMarks(); + + static auto constexpr NAME_OF_FILE_WITH_DATA = "data"; /// Columns that are read. diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index dcc12e2e800..327afb26a85 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -34,7 +34,7 @@ MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data clockid_t clock_type_) : IMergeTreeReader(data_part_, columns_ , uncompressed_cache_, mark_cache_, mark_ranges_ - , settings_ , avg_value_size_hints_) + , settings_, avg_value_size_hints_) { try { From 8ba37da2ef096573d7a36912b8b89a9adc1c5ebf Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 16 Oct 2019 21:27:53 +0300 Subject: [PATCH 0039/2007] polymorphic parts (development) --- dbms/src/Interpreters/SystemLog.h | 1 + .../Storages/MergeTree/IMergeTreeDataPart.cpp | 10 +- .../Storages/MergeTree/IMergeTreeDataPart.h | 17 +- .../MergeTree/IMergeTreeDataPartWriter.h | 95 +++ .../MergeTree/IMergedBlockOutputStream.cpp | 7 + .../MergeTreeBaseSelectBlockInputStream.cpp | 2 + dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 + dbms/src/Storages/MergeTree/MergeTreeData.h | 1 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 4 + .../MergeTree/MergeTreeDataPartCompact.cpp | 687 ++++++++++++++++++ .../MergeTree/MergeTreeDataPartCompact.h | 27 +- .../MergeTree/MergeTreeDataPartFactory.cpp | 7 +- .../MergeTree/MergeTreeDataPartWide.cpp | 18 +- .../MergeTree/MergeTreeDataPartWide.h | 17 +- .../MergeTreeDataPartWriterCompact.cpp | 116 +++ .../MergeTreeDataPartWriterCompact.h | 42 ++ .../MergeTree/MergeTreeDataPartWriterWide.cpp | 67 ++ .../MergeTree/MergeTreeDataPartWriterWide.h | 28 + .../MergeTree/MergeTreeDataWriter.cpp | 3 +- .../MergeTreeIndexGranularityInfo.cpp | 19 +- .../MergeTree/MergeTreeIndexGranularityInfo.h | 5 +- .../MergeTree/MergeTreeReaderCompact.cpp | 130 +++- .../MergeTree/MergeTreeReaderCompact.h | 10 + .../MergeTree/MergeTreeReaderWide.cpp | 3 + .../MergeTree/MergedBlockOutputStream.cpp | 52 +- 25 files changed, 1272 insertions(+), 100 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index b00f77b7622..01c963a53ef 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -354,6 +354,7 @@ void SystemLog::flushImpl(EntryType reason) /// In case of exception, also clean accumulated data - to avoid locking. data.clear(); } + if (reason == EntryType::FORCE_FLUSH) { std::lock_guard lock(condvar_mutex); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 124638f6b52..64860788586 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -137,27 +137,33 @@ void IMergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other) IMergeTreeDataPart::IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) : storage(storage_) , name(name_) , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) + , index_granularity_info(index_granularity_info_) , disk(disk_) , relative_path(relative_path_.value_or(name_)) - , index_granularity_info(shared_from_this()) {} +{ +} IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) : storage(storage_) , name(name_) , info(info_) + , index_granularity_info(index_granularity_info_) , disk(disk_) , relative_path(relative_path_.value_or(name_)) - , index_granularity_info(shared_from_this()) {} +{ +} ColumnSize IMergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 84b6eaf1760..17a799ac9d0 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -74,7 +74,7 @@ public: /// NOTE: Returns zeros if column files are not found in checksums. /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. /// (either by locking columns_lock, or by locking table structure). - virtual ColumnSize getColumnSize(const String & name, const IDataType & type) const = 0; + ColumnSize getColumnSize(const String & name, const IDataType & type); /// Initialize columns (from columns.txt if exists, or create from column files if not). /// Load checksums from checksums.txt if exists. Load index if required. @@ -84,10 +84,6 @@ public: /// If no checksums are present returns the name of the first physically existing column. virtual String getColumnNameWithMinumumCompressedSize() const = 0; - virtual String getMarkExtension(bool /* is_adaptive */) const { return ""; } - - virtual size_t getMarkSize(bool /* is_adaptive */) const { return 0; } - // virtual void detach() = 0; // virtual Checksums check( @@ -110,7 +106,7 @@ public: enum class Type { WIDE, - STRIPED, + COMPACT, IN_MEMORY, }; @@ -124,7 +120,7 @@ public: { case Type::WIDE: return "Wide"; - case Type::STRIPED: + case Type::COMPACT: return "Striped"; case Type::IN_MEMORY: return "InMemory"; @@ -141,17 +137,21 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk = {}, const std::optional & relative_path = {}); IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk = {}, const std::optional & relative_path = {}); void assertOnDisk() const; + ColumnSize getColumnSize(const String & column_name, const IDataType & type) const; + ColumnSize getTotalColumnsSize() const; /// Generate the new name for this part according to `new_part_info` and min/max dates from the old name. @@ -174,6 +174,7 @@ public: String name; MergeTreePartInfo info; + MergeTreeIndexGranularityInfo index_granularity_info; DiskSpace::DiskPtr disk; @@ -317,8 +318,6 @@ public: */ mutable std::mutex alter_mutex; - MergeTreeIndexGranularityInfo index_granularity_info; - /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h new file mode 100644 index 00000000000..687179ece26 --- /dev/null +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -0,0 +1,95 @@ +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +class IMergeTreeDataPartWriter +{ +public: + using WrittenOffsetColumns = std::set; + + struct ColumnStream + { + ColumnStream( + const String & escaped_column_name_, + const String & data_path_, + const std::string & data_file_extension_, + const std::string & marks_path_, + const std::string & marks_file_extension_, + const CompressionCodecPtr & compression_codec_, + size_t max_compress_block_size_, + size_t estimated_size_, + size_t aio_threshold_); + + String escaped_column_name; + std::string data_file_extension; + std::string marks_file_extension; + + /// compressed -> compressed_buf -> plain_hashing -> plain_file + std::unique_ptr plain_file; + HashingWriteBuffer plain_hashing; + CompressedWriteBuffer compressed_buf; + HashingWriteBuffer compressed; + + /// marks -> marks_file + WriteBufferFromFile marks_file; + HashingWriteBuffer marks; + + void finalize(); + + void sync(); + + void addToChecksums(MergeTreeData::DataPart::Checksums & checksums); + }; + + virtual size_t write( + const Block & block, size_t from_mark, size_t offset, + /* Blocks with already sorted index columns */ + const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; + + virtual std::pair writeColumn( + const String & name, + const IDataType & type, + const IColumn & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + size_t from_mark) = 0; + + // /// Write single granule of one column (rows between 2 marks) + // virtual size_t writeSingleGranule( + // const String & name, + // const IDataType & type, + // const IColumn & column, + // WrittenOffsetColumns & offset_columns, + // bool skip_offsets, + // IDataType::SerializeBinaryBulkStatePtr & serialization_state, + // IDataType::SerializeBinaryBulkSettings & serialize_settings, + // size_t from_row, + // size_t number_of_rows, + // bool write_marks) = 0; + + // /// Write mark for column + // virtual void writeSingleMark( + // const String & name, + // const IDataType & type, + // WrittenOffsetColumns & offset_columns, + // bool skip_offsets, + // size_t number_of_rows, + // DB::IDataType::SubstreamPath & path) = 0; +protected: + void start(); + + const NamesAndTypesList & columns_list; + IColumn::Permutation * permutation; + bool started = false; +}; + +} \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 71e55015d77..01d9de2f60c 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -288,6 +288,13 @@ std::pair IMergedBlockOutputStream::writeColumn( } }, serialize_settings.path); + std::cerr << "(IMergedBlockOutputStream::writeColumn) name: " << name << "\n"; + std::cerr << "(IMergedBlockOutputStream::writeColumn) from_mark: " << from_mark << "\n"; + std::cerr << "(IMergedBlockOutputStream::writeColumn) current_column_mark: " << current_column_mark << "\n"; + std::cerr << "(IMergedBlockOutputStream::writeColumn) current_row: " << current_row << "\n"; + std::cerr << "(IMergedBlockOutputStream::writeColumn) total_rows: " << total_rows; + std::cerr << "(IMergedBlockOutputStream::writeColumn) blocks_are_granules_size: " << blocks_are_granules_size << "\n"; + return std::make_pair(current_column_mark, current_row - total_rows); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp index be3f7496439..83614e7bf34 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp @@ -149,6 +149,8 @@ Block MergeTreeBaseSelectBlockInputStream::readFromPartImpl() UInt64 recommended_rows = estimateNumRows(*task, task->range_reader); UInt64 rows_to_read = std::max(UInt64(1), std::min(current_max_block_size_rows, recommended_rows)); + std::cerr << "(readFromPartImpl) rows_to_read: " << rows_to_read << "\n"; + auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); /// All rows were filtered. Repeat. diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 67215efd082..70ee3b717ec 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -283,12 +283,16 @@ void MergeTreeData::setProperties( Names new_primary_key_columns; Names new_sorting_key_columns; + NameSet sorting_key_columns_set; for (size_t i = 0; i < sorting_key_size; ++i) { String sorting_key_column = new_sorting_key_expr_list->children[i]->getColumnName(); new_sorting_key_columns.push_back(sorting_key_column); + if (!sorting_key_columns_set.emplace(sorting_key_column).second) + throw Exception("Sorting key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); + if (i < primary_key_size) { String pk_column = new_primary_key_expr_list->children[i]->getColumnName(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 9e537c03616..4f06e651808 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -746,6 +746,7 @@ protected: friend class IMergeTreeDataPart; friend class MergeTreeDataPartWide; + friend class MergeTreeDataPartCompact; friend class MergeTreeDataMergerMutator; friend class ReplicatedMergeTreeAlterThread; friend struct ReplicatedMergeTreeTableMetadata; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 892fc50f0a8..b26a20efeb1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1170,6 +1170,10 @@ MergeTreeDataMergerMutator::MergeAlgorithm MergeTreeDataMergerMutator::chooseMer if (need_remove_expired_values) return MergeAlgorithm::Horizontal; + for (const auto & part : parts) + if (!part->supportsVerticalMerge()) + return MergeAlgorithm::Horizontal; + bool is_supported_storage = data.merging_params.mode == MergeTreeData::MergingParams::Ordinary || data.merging_params.mode == MergeTreeData::MergingParams::Collapsing || diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index e69de29bb2d..c209b4d5c67 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -0,0 +1,687 @@ +#include "MergeTreeDataPartCompact.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + +// namespace +// { +// } + +namespace ErrorCodes +{ + extern const int FILE_DOESNT_EXIST; + extern const int NO_FILE_IN_DATA_PART; + extern const int EXPECTED_END_OF_FILE; + extern const int CORRUPTED_DATA; + extern const int NOT_FOUND_EXPECTED_DATA_PART; + extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; + extern const int BAD_TTL_FILE; + extern const int CANNOT_UNLINK; +} + + +static ReadBufferFromFile openForReading(const String & path) +{ + return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); +} + +MergeTreeDataPartCompact::MergeTreeDataPartCompact( + MergeTreeData & storage_, + const String & name_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_) + : IMergeTreeDataPart(storage_, name_, index_granularity_info_, disk_, relative_path_) +{ +} + +MergeTreeDataPartCompact::MergeTreeDataPartCompact( + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_) + : IMergeTreeDataPart(storage_, name_, info_, index_granularity_info_, disk_, relative_path_) +{ +} + +IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( + const NamesAndTypesList & columns_to_read, + const MarkRanges & mark_ranges, + UncompressedCache * uncompressed_cache, + MarkCache * mark_cache, + const ReaderSettings & reader_settings, + const ValueSizeMap & avg_value_size_hints, + const ReadBufferFromFileBase::ProfileCallback & /* profile_callback */) const +{ + return std::make_unique(shared_from_this(), columns_to_read, uncompressed_cache, + mark_cache, mark_ranges, reader_settings, avg_value_size_hints); +} + + +/// Takes into account the fact that several columns can e.g. share their .size substreams. +/// When calculating totals these should be counted only once. +ColumnSize MergeTreeDataPartCompact::getColumnSizeImpl( + const String & column_name, const IDataType & type, std::unordered_set * processed_substreams) const +{ + UNUSED(column_name); + UNUSED(type); + UNUSED(processed_substreams); + // ColumnSize size; + // if (checksums.empty()) + // return size; + + // type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + // { + // String file_name = IDataType::getFileNameForStream(column_name, substream_path); + + // if (processed_substreams && !processed_substreams->insert(file_name).second) + // return; + + // auto bin_checksum = checksums.files.find(file_name + ".bin"); + // if (bin_checksum != checksums.files.end()) + // { + // size.data_compressed += bin_checksum->second.file_size; + // size.data_uncompressed += bin_checksum->second.uncompressed_size; + // } + + // auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); + // if (mrk_checksum != checksums.files.end()) + // size.marks += mrk_checksum->second.file_size; + // }, {}); + + return ColumnSize{}; +} + +/** Returns the name of a column with minimum compressed size (as returned by getColumnSize()). + * If no checksums are present returns the name of the first physically existing column. + */ +String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const +{ + const auto & storage_columns = storage.getColumns().getAllPhysical(); + const std::string * minimum_size_column = nullptr; + UInt64 minimum_size = std::numeric_limits::max(); + + for (const auto & column : storage_columns) + { + if (!hasColumnFiles(column.name, *column.type)) + continue; + + const auto size = getColumnSize(column.name, *column.type).data_compressed; + if (size < minimum_size) + { + minimum_size = size; + minimum_size_column = &column.name; + } + } + + if (!minimum_size_column) + throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); + + return *minimum_size_column; +} + +MergeTreeDataPartCompact::~MergeTreeDataPartCompact() +{ + if (state == State::DeleteOnDestroy || is_temp) + { + try + { + std::string path = getFullPath(); + + Poco::File dir(path); + if (!dir.exists()) + return; + + if (is_temp) + { + String file_name = Poco::Path(relative_path).getFileName(); + + if (file_name.empty()) + throw Exception("relative_path " + relative_path + " of part " + name + " is invalid or not set", ErrorCodes::LOGICAL_ERROR); + + if (!startsWith(file_name, "tmp")) + { + LOG_ERROR(storage.log, "~DataPart() should remove part " << path + << " but its name doesn't start with tmp. Too suspicious, keeping the part."); + return; + } + } + + dir.remove(true); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } +} + +UInt64 MergeTreeDataPartCompact::calculateTotalSizeOnDisk(const String & from) +{ + Poco::File cur(from); + if (cur.isFile()) + return cur.getSize(); + std::vector files; + cur.list(files); + UInt64 res = 0; + for (const auto & file : files) + res += calculateTotalSizeOnDisk(from + file); + return res; +} + +void MergeTreeDataPartCompact::remove() const +{ + if (relative_path.empty()) + throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); + + /** Atomic directory removal: + * - rename directory to temporary name; + * - remove it recursive. + * + * For temporary name we use "delete_tmp_" prefix. + * + * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, + * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. + * But when we removing data part, it can be old enough. And rename doesn't change mtime. + * And a race condition can happen that will lead to "File not found" error here. + */ + + String full_path = storage.getFullPathOnDisk(disk); + String from = full_path + relative_path; + String to = full_path + "delete_tmp_" + name; + // TODO directory delete_tmp_ is never removed if server crashes before returning from this function + + + Poco::File from_dir{from}; + Poco::File to_dir{to}; + + if (to_dir.exists()) + { + LOG_WARNING(storage.log, "Directory " << to << " (to which part must be renamed before removing) already exists." + " Most likely this is due to unclean restart. Removing it."); + + try + { + to_dir.remove(true); + } + catch (...) + { + LOG_ERROR(storage.log, "Cannot remove directory " << to << ". Check owner and access rights."); + throw; + } + } + + try + { + from_dir.renameTo(to); + } + catch (const Poco::FileNotFoundException &) + { + LOG_ERROR(storage.log, "Directory " << from << " (part to remove) doesn't exist or one of nested files has gone." + " Most likely this is due to manual removing. This should be discouraged. Ignoring."); + + return; + } + + try + { + /// Remove each expected file in directory, then remove directory itself. + +#if !__clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + std::shared_lock lock(columns_lock); + + for (const auto & [file, _] : checksums.files) + { + String path_to_remove = to + "/" + file; + if (0 != unlink(path_to_remove.c_str())) + throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, + ErrorCodes::CANNOT_UNLINK); + } +#if !__clang__ +#pragma GCC diagnostic pop +#endif + + for (const auto & file : {"checksums.txt", "columns.txt"}) + { + String path_to_remove = to + "/" + file; + if (0 != unlink(path_to_remove.c_str())) + throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, + ErrorCodes::CANNOT_UNLINK); + } + + if (0 != rmdir(to.c_str())) + throwFromErrnoWithPath("Cannot rmdir file " + to, to, ErrorCodes::CANNOT_UNLINK); + } + catch (...) + { + /// Recursive directory removal does many excessive "stat" syscalls under the hood. + + LOG_ERROR(storage.log, "Cannot quickly remove directory " << to << " by removing files; fallback to recursive removal. Reason: " + << getCurrentExceptionMessage(false)); + + to_dir.remove(true); + } +} + +void MergeTreeDataPartCompact::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool /* check_consistency */) +{ + /// Memory should not be limited during ATTACH TABLE query. + /// This is already true at the server startup but must be also ensured for manual table ATTACH. + /// Motivation: memory for index is shared between queries - not belong to the query itself. + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); + + loadColumns(require_columns_checksums); + loadChecksums(require_columns_checksums); + loadIndexGranularity(); + loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` + loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. + loadPartitionAndMinMaxIndex(); + loadTTLInfos(); + // if (check_consistency) + // checkConsistency(require_columns_checksums); +} + +void MergeTreeDataPartCompact::loadIndexGranularity() +{ + String full_path = getFullPath(); + + if (columns.empty()) + throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + /// We can use any column, it doesn't matter + std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + "data"); + if (!Poco::File(marks_file_path).exists()) + throw Exception("Marks file '" + marks_file_path + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); + + size_t marks_file_size = Poco::File(marks_file_path).getSize(); + + /// old version of marks with static index granularity + ReadBufferFromFile buffer(marks_file_path, marks_file_size, -1); + while (!buffer.eof()) + { + size_t granularity; + readIntBinary(granularity, buffer); + index_granularity.appendMark(granularity); + /// Skip offsets for columns + buffer.seek(sizeof(size_t) * 2 * columns.size()); + } + if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) + throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); +} + +void MergeTreeDataPartCompact::loadIndex() +{ + /// It can be empty in case of mutations + if (!index_granularity.isInitialized()) + throw Exception("Index granularity is not loaded before index loading", ErrorCodes::LOGICAL_ERROR); + + size_t key_size = storage.primary_key_columns.size(); + + if (key_size) + { + MutableColumns loaded_index; + loaded_index.resize(key_size); + + for (size_t i = 0; i < key_size; ++i) + { + loaded_index[i] = storage.primary_key_data_types[i]->createColumn(); + loaded_index[i]->reserve(index_granularity.getMarksCount()); + } + + String index_path = getFullPath() + "primary.idx"; + ReadBufferFromFile index_file = openForReading(index_path); + + for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 + for (size_t j = 0; j < key_size; ++j) + storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], index_file); + + for (size_t i = 0; i < key_size; ++i) + { + loaded_index[i]->protect(); + if (loaded_index[i]->size() != index_granularity.getMarksCount()) + throw Exception("Cannot read all data from index file " + index_path + + "(expected size: " + toString(index_granularity.getMarksCount()) + ", read: " + toString(loaded_index[i]->size()) + ")", + ErrorCodes::CANNOT_READ_ALL_DATA); + } + + if (!index_file.eof()) + throw Exception("Index file " + index_path + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); + + index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); + } +} + +void MergeTreeDataPartCompact::loadPartitionAndMinMaxIndex() +{ + if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + DayNum min_date; + DayNum max_date; + MergeTreePartInfo::parseMinMaxDatesFromPartName(name, min_date, max_date); + + const auto & date_lut = DateLUT::instance(); + partition = MergeTreePartition(date_lut.toNumYYYYMM(min_date)); + minmax_idx = MinMaxIndex(min_date, max_date); + } + else + { + String path = getFullPath(); + partition.load(storage, path); + if (!isEmpty()) + minmax_idx.load(storage, path); + } + + String calculated_partition_id = partition.getID(storage.partition_key_sample); + if (calculated_partition_id != info.partition_id) + throw Exception( + "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id + + " differs from partition ID in part name: " + info.partition_id, + ErrorCodes::CORRUPTED_DATA); +} + +void MergeTreeDataPartCompact::loadChecksums(bool require) +{ + String path = getFullPath() + "checksums.txt"; + Poco::File checksums_file(path); + if (checksums_file.exists()) + { + ReadBufferFromFile file = openForReading(path); + if (checksums.read(file)) + { + assertEOF(file); + bytes_on_disk = checksums.getTotalSizeOnDisk(); + } + else + bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); + } + else + { + if (require) + throw Exception("No checksums.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); + } +} + +void MergeTreeDataPartCompact::loadRowsCount() +{ + if (index_granularity.empty()) + { + rows_count = 0; + } + else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + String path = getFullPath() + "count.txt"; + if (!Poco::File(path).exists()) + throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + ReadBufferFromFile file = openForReading(path); + readIntText(rows_count, file); + assertEOF(file); + } + else + { + for (const NameAndTypePair & column : columns) + { + ColumnPtr column_col = column.type->createColumn(); + if (!column_col->isFixedAndContiguous() || column_col->lowCardinality()) + continue; + + size_t column_size = getColumnSize(column.name, *column.type).data_uncompressed; + if (!column_size) + continue; + + size_t sizeof_field = column_col->sizeOfValueIfFixed(); + rows_count = column_size / sizeof_field; + + if (column_size % sizeof_field != 0) + { + throw Exception( + "Uncompressed size of column " + column.name + "(" + toString(column_size) + + ") is not divisible by the size of value (" + toString(sizeof_field) + ")", + ErrorCodes::LOGICAL_ERROR); + } + + size_t last_mark_index_granularity = index_granularity.getLastNonFinalMarkRows(); + size_t rows_approx = index_granularity.getTotalRows(); + if (!(rows_count <= rows_approx && rows_approx < rows_count + last_mark_index_granularity)) + throw Exception( + "Unexpected size of column " + column.name + ": " + toString(rows_count) + " rows, expected " + + toString(rows_approx) + "+-" + toString(last_mark_index_granularity) + " rows according to the index", + ErrorCodes::LOGICAL_ERROR); + + return; + } + + throw Exception("Data part doesn't contain fixed size column (even Date column)", ErrorCodes::LOGICAL_ERROR); + } +} + +void MergeTreeDataPartCompact::loadTTLInfos() +{ + String path = getFullPath() + "ttl.txt"; + if (Poco::File(path).exists()) + { + ReadBufferFromFile in = openForReading(path); + assertString("ttl format version: ", in); + size_t format_version; + readText(format_version, in); + assertChar('\n', in); + + if (format_version == 1) + { + try + { + ttl_infos.read(in); + } + catch (const JSONException &) + { + throw Exception("Error while parsing file ttl.txt in part: " + name, ErrorCodes::BAD_TTL_FILE); + } + } + else + throw Exception("Unknown ttl format version: " + toString(format_version), ErrorCodes::BAD_TTL_FILE); + } +} + +// void MergeTreeDataPartCompact::accumulateColumnSizes(ColumnToSize & column_to_size) const +// { +// std::shared_lock part_lock(columns_lock); + +// for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical()) +// { +// IDataType::SubstreamPath path; +// name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) +// { +// Poco::File bin_file(getFullPath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"); +// if (bin_file.exists()) +// column_to_size[name_type.name] += bin_file.getSize(); +// }, path); +// } +// } + +void MergeTreeDataPartCompact::loadColumns(bool require) +{ + String path = getFullPath() + "columns.txt"; + Poco::File poco_file_path{path}; + if (!poco_file_path.exists()) + { + if (require) + throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + /// If there is no file with a list of columns, write it down. + for (const NameAndTypePair & column : storage.getColumns().getAllPhysical()) + if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) + columns.push_back(column); + + if (columns.empty()) + throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + { + WriteBufferFromFile out(path + ".tmp", 4096); + columns.writeText(out); + } + Poco::File(path + ".tmp").renameTo(path); + + return; + } + + is_frozen = !poco_file_path.canWrite(); + + ReadBufferFromFile file = openForReading(path); + columns.readText(file); +} + +void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) +{ + String path = getFullPath(); + + if (!checksums.empty()) + { + if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) + throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (require_part_metadata) + { + for (const NameAndTypePair & name_type : columns) + { + IDataType::SubstreamPath stream_path; + name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); + String mrk_file_name = file_name + index_granularity_info.marks_file_extension; + String bin_file_name = file_name + ".bin"; + if (!checksums.files.count(mrk_file_name)) + throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + path, + ErrorCodes::NO_FILE_IN_DATA_PART); + if (!checksums.files.count(bin_file_name)) + throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + path, + ErrorCodes::NO_FILE_IN_DATA_PART); + }, stream_path); + } + } + + if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + if (!checksums.files.count("count.txt")) + throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (storage.partition_key_expr && !checksums.files.count("partition.dat")) + throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (!isEmpty()) + { + for (const String & col_name : storage.minmax_idx_columns) + { + if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) + throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); + } + } + } + + checksums.checkSizes(path); + } + else + { + auto check_file_not_empty = [&path](const String & file_path) + { + Poco::File file(file_path); + if (!file.exists() || file.getSize() == 0) + throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + return file.getSize(); + }; + + /// Check that the primary key index is not empty. + if (!storage.primary_key_columns.empty()) + check_file_not_empty(path + "primary.idx"); + + if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + check_file_not_empty(path + "count.txt"); + + if (storage.partition_key_expr) + check_file_not_empty(path + "partition.dat"); + + for (const String & col_name : storage.minmax_idx_columns) + check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); + } + + /// Check that all marks are nonempty and have the same size. + + std::optional marks_size; + for (const NameAndTypePair & name_type : columns) + { + name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + Poco::File file(IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension); + + /// Missing file is Ok for case when new column was added. + if (file.exists()) + { + UInt64 file_size = file.getSize(); + + if (!file_size) + throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + + if (!marks_size) + marks_size = file_size; + else if (file_size != *marks_size) + throw Exception("Part " + path + " is broken: marks have different sizes.", + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + } + }); + } + } +} + +// bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const IDataType & type) const +// { +// bool res = true; + +// type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) +// { +// String file_name = IDataType::getFileNameForStream(column_name, substream_path); + +// auto bin_checksum = checksums.files.find(file_name + ".bin"); +// auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); + +// if (bin_checksum == checksums.files.end() || mrk_checksum == checksums.files.end()) +// res = false; +// }, {}); + +// return res; +// } + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 6655c418428..be6b06de68d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -34,18 +34,20 @@ public: using Checksums = MergeTreeDataPartChecksums; using Checksum = MergeTreeDataPartChecksums::Checksum; - MergeTreeDataPartCompact( + MergeTreeDataPartCompact( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskSpace::DiskPtr & disk, - const std::optional & relative_path = {}); + const MergeTreeIndexGranularityInfo & index_granularity_info_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_ = {}); - MergeTreeDataPartCompact( + MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const DiskSpace::DiskPtr & disk, - const std::optional & relative_path = {}); + const MergeTreeIndexGranularityInfo & index_granularity_info_, + const DiskSpace::DiskPtr & disk_, + const std::optional & relative_path_ = {}); MergeTreeReaderPtr getReader( const NamesAndTypesList & columns, @@ -58,19 +60,8 @@ public: bool isStoredOnDisk() const override { return true; } - String getMarkExtension(bool /* is_adaptive */) const override { return ".mrk3"; } - - bool getMarkSize(bool is_adaptive) - { - return sizeof(size_t) + columns.size() * sizeof(size_t) * 2; - } - void remove() const override; - /// NOTE: Returns zeros if column files are not found in checksums. - /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. - /// (either by locking columns_lock, or by locking table structure). - ColumnSize getColumnSize(const String & name, const IDataType & type) const override; /// Initialize columns (from columns.txt if exists, or create from column files if not). /// Load checksums from checksums.txt if exists. Load index if required. @@ -80,7 +71,7 @@ public: /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; - virtual Type getType() const override { return Type::WIDE; } + virtual Type getType() const override { return Type::COMPACT; } ~MergeTreeDataPartCompact() override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp index 1abfbbe3708..d2066385dba 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp @@ -1,16 +1,19 @@ #include "MergeTreeDataPartFactory.h" +#include namespace DB { std::shared_ptr createPart(const MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const MergeTreePartInfo & info, const String & relative_path) { - return std::make_shared(storage, name, info, disk, relative_path); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", 3 * sizeof(size_t)); + return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); } std::shared_ptr createPart(MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const String & relative_path) { - return std::make_shared(storage, name, disk, relative_path); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", 3 * sizeof(size_t)); + return std::make_shared(storage, name, index_granularity_info, disk, relative_path); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 3e77249b324..7ec1c282095 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -54,9 +54,10 @@ static ReadBufferFromFile openForReading(const String & path) MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, index_granularity_info_, disk_, relative_path_) { } @@ -64,9 +65,10 @@ MergeTreeDataPartWide::MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, info_, index_granularity_info_, disk_, relative_path_) { } @@ -115,11 +117,6 @@ ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( return size; } -ColumnSize MergeTreeDataPartWide::getColumnSize(const String & column_name, const IDataType & type) const -{ - return getColumnSizeImpl(column_name, type, nullptr); -} - /** Returns the name of a column with minimum compressed size (as returned by getColumnSize()). * If no checksums are present returns the name of the first physically existing column. */ @@ -134,7 +131,7 @@ String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const if (!hasColumnFiles(column.name, *column.type)) continue; - const auto size = getColumnSize(column.name, *column.type).data_compressed; + const auto size = getColumnSizeImpl(column.name, *column.type, nullptr).data_compressed; if (size < minimum_size) { minimum_size = size; @@ -315,7 +312,7 @@ void MergeTreeDataPartWide::loadColumnsChecksumsIndexes(bool require_columns_che void MergeTreeDataPartWide::loadIndexGranularity() { String full_path = getFullPath(); - index_granularity_info.changeGranularityIfRequired(shared_from_this()); + index_granularity_info.changeGranularityIfRequired(full_path); if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); @@ -327,7 +324,6 @@ void MergeTreeDataPartWide::loadIndexGranularity() size_t marks_file_size = Poco::File(marks_file_path).getSize(); - /// old version of marks with static index granularity if (!index_granularity_info.is_adaptive) { size_t marks_count = marks_file_size / index_granularity_info.mark_size_in_bytes; @@ -467,7 +463,7 @@ void MergeTreeDataPartWide::loadRowsCount() if (!column_col->isFixedAndContiguous() || column_col->lowCardinality()) continue; - size_t column_size = getColumnSize(column.name, *column.type).data_uncompressed; + size_t column_size = getColumnSizeImpl(column.name, *column.type, nullptr).data_uncompressed; if (!column_size) continue; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 61007a4b955..9e91bea55bf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -38,12 +38,14 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk, const std::optional & relative_path = {}); MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk, const std::optional & relative_path = {}); @@ -62,19 +64,6 @@ public: bool supportsVerticalMerge() const override { return true; } - String getMarkExtension(bool is_adaptive) const override { return is_adaptive ? ".mrk2" : ".mrk"; } - - size_t getMarkSize(bool is_adaptive) const override - { - size_t nums = is_adaptive ? 3 : 2; - return sizeof(size_t) * nums; - } - - /// NOTE: Returns zeros if column files are not found in checksums. - /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. - /// (either by locking columns_lock, or by locking table structure). - ColumnSize getColumnSize(const String & name, const IDataType & type) const override; - /// Initialize columns (from columns.txt if exists, or create from column files if not). /// Load checksums from checksums.txt if exists. Load index if required. void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) override; @@ -83,7 +72,7 @@ public: /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; - virtual Type getType() const override { return Type::WIDE; } + Type getType() const override { return Type::WIDE; } ~MergeTreeDataPartWide() override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp new file mode 100644 index 00000000000..86a2ff3f188 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -0,0 +1,116 @@ +#include + +namespace DB +{ + +size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule( + const ColumnWithTypeAndName & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + IDataType::SerializeBinaryBulkSettings & serialize_settings, + size_t from_row, + size_t number_of_rows) +{ +} + + +size_t MergeTreeDataPartWriterCompact::write(const Block & block, size_t from_mark, size_t offset, + const Block & primary_key_block, const Block & skip_indexes_block) +{ + if (!started) + start(); + + size_t total_rows = block.rows(); + size_t current_mark = from_mark; + size_t current_row = 0; + + + IDataType::SerializeBinaryBulkSettings serialize_settings; + serialize_settings.getter = [&ostr](IDataType::SubstreamPath) -> WriteBuffer * { return &ostr; }; + serialize_settings.position_independent_encoding = false; + serialize_settings.low_cardinality_max_dictionary_size = 0; + + ColumnsWithTypeAndName columns_to_write(columns_list.size()); + auto it = columns_list.begin(); + for (size_t i = 0; i < columns_list.size(); ++i, ++it) + { + if (permutation) + { + if (primary_key_block.has(it->name)) + columns_to_write[i] = primary_key_block.getByName(it->name); + else if (skip_indexes_block.has(it->name)) + columns_to_write[i] = skip_indexes_block.getByName(it->name); + else + { + columns_to_write[i] = block.getByName(it->name); + columns_to_write[i].column = columns_to_write[i].column->permute(*permutation, 0); + } + } + else + columns_to_write[i] = block.getByName(it->name); + } + + while (current_row < total_rows) + { + bool write_marks = true; + size_t rows_to_write; + if (current_row == 0 && offset != 0) + { + rows_to_write = offset; + write_marks = false; + } + else + { + rows_to_write = index_granularity->getMarkRows(current_mark); + } + + for (size_t i = 0; i < columns_to_write.size(); ++i) + { + current_row = writeColumnSingleGranule(columns_to_write[i], offset_columns, skip_offsets, serialization_states[i], serialize_settings, current_row, rows_to_write); + } + + if (write_marks) + { + writeMark(); + ++current_mark; + } + } + + /// We always write end granule for block in Compact parts. + return 0; +} + +size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + IDataType::SerializeBinaryBulkSettings & serialize_settings, + size_t from_row, + size_t number_of_rows) +{ + column.type->serializeBinaryBulkStatePrefix(serialize_settings, serialization_state); + column.type->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, serialization_state); + column.type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_state); +} + +void MergeTreeDataPartWriterWide::start() +{ + if (started) + return; + + started = true; + + serialization_states.reserve(columns_list.size()); + WrittenOffsetColumns tmp_offset_columns; + IDataType::SerializeBinaryBulkSettings settings; + + for (const auto & col : columns_list) + { + settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); + serialization_states.emplace_back(nullptr); + col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); + } +} + +} \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h new file mode 100644 index 00000000000..e5990faab77 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -0,0 +1,42 @@ +#include + +namespace DB +{ + +class MergeTreeDataPartWriterCompact : IMergeTreeDataPartWriter +{ +public: + size_t write(const Block & block, size_t from_mark, size_t offset, + const Block & primary_key_block, const Block & skip_indexes_block) override; + + std::pair writeColumn( + const String & name, + const IDataType & type, + const IColumn & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + size_t from_mark) override; + + /// Write single granule of one column (rows between 2 marks) + size_t writeColumnSingleGranule( + const ColumnWithTypeAndName & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + IDataType::SerializeBinaryBulkSettings & serialize_settings, + size_t from_row, + size_t number_of_rows); + + void writeSingleMark() + +protected: + void start() override; + +private: + ColumnStream stream; + MergeTreeIndexGranularity * index_granularity = nullptr; + Columns columns_to_write; +}; + +} \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp new file mode 100644 index 00000000000..cac516ca10f --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -0,0 +1,67 @@ +// #include + +// namespace DB +// { + +// size_t MergeTreeDataPartWriterWide::write(size_t current_mark, const Block & block) +// { +// if (!started) +// start(); + +// size_t index_offset = 0; +// auto it = columns_list.begin(); +// for (size_t i = 0; i < columns_list.size(); ++i, ++it) +// { +// const ColumnWithTypeAndName & column = block.getByName(it->name); + +// if (permutation) +// { +// auto primary_column_it = primary_key_column_name_to_position.find(it->name); +// auto skip_index_column_it = skip_indexes_column_name_to_position.find(it->name); + +// if (primary_key_column_name_to_position.end() != primary_column_it) +// { +// const auto & primary_column = *primary_key_columns[primary_column_it->second].column; +// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, primary_column, offset_columns, false, serialization_states[i], current_mark); +// } +// else if (skip_indexes_column_name_to_position.end() != skip_index_column_it) +// { +// const auto & index_column = *skip_indexes_columns[skip_index_column_it->second].column; +// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, index_column, offset_columns, false, serialization_states[i], current_mark); +// } +// else +// { +// /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. +// ColumnPtr permuted_column = column.column->permute(*permutation, 0); +// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, *permuted_column, offset_columns, false, serialization_states[i], current_mark); +// } +// } +// else +// { +// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, false, serialization_states[i], current_mark); +// } +// } + +// return index_offset; +// } + +// void MergeTreeDataPartWriterWide::start() +// { +// if (started) +// return; + +// started = true; + +// serialization_states.reserve(columns_list.size()); +// WrittenOffsetColumns tmp_offset_columns; +// IDataType::SerializeBinaryBulkSettings settings; + +// for (const auto & col : columns_list) +// { +// settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); +// serialization_states.emplace_back(nullptr); +// col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); +// } +// } + +// } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h new file mode 100644 index 00000000000..7de78a12da3 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -0,0 +1,28 @@ +// #include + +// namespace DB +// { + +// class MergeTreeDataPartWriterWide : IMergeTreeDataPartWriter +// { +// public: +// size_t write(size_t current_mark, const Block & block) override; + +// std::pair writeColumn( +// const String & name, +// const IDataType & type, +// const IColumn & column, +// WrittenOffsetColumns & offset_columns, +// bool skip_offsets, +// IDataType::SerializeBinaryBulkStatePtr & serialization_state, +// size_t from_mark) override; + +// protected: +// void start() override; + +// private: +// SerializationStates serialization_states; +// NameSet permuted_columns; +// }; + +// } \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index b1589f5a3e0..de23dd21b1f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -168,13 +168,14 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPartition & block_with_partition) { Block & block = block_with_partition.block; + std::cerr << "(MergeTreeDataWriter::writeTempPart) block.rows(): " << block.rows() << "\n"; static const String TMP_PREFIX = "tmp_insert_"; /// This will generate unique name in scope of current server process. Int64 temp_index = data.insert_increment.get(); - IMergeTreeDataPart::MinMaxIndex minmax_idx; + IMergeTreeDataPart::MinMaxIndex minmax_idx; minmax_idx.update(block, data.minmax_idx_columns); MergeTreePartition partition(std::move(block_with_partition.partition)); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 73baee85569..83952e0c625 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -29,30 +29,25 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( return {}; } -MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeDataPartPtr & part) +MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeData & storage, + const String & marks_file_extension_, UInt8 mark_size_in_bytes_) + : marks_file_extension(marks_file_extension_), mark_size_in_bytes(mark_size_in_bytes_) { - const auto storage_settings = part->storage.getSettings(); + const auto storage_settings = storage.getSettings(); fixed_index_granularity = storage_settings->index_granularity; /// Granularity is fixed - if (!part->storage.canUseAdaptiveGranularity()) + if (!storage.canUseAdaptiveGranularity()) setNonAdaptive(); else setAdaptive(storage_settings->index_granularity_bytes); - - mark_size_in_bytes = part->getMarkSize(is_adaptive); - marks_file_extension = part->getMarkExtension(is_adaptive); } -void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const MergeTreeDataPartPtr & part) +void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const String & path) { - auto mrk_ext = getMrkExtensionFromFS(part->getFullPath()); + auto mrk_ext = getMrkExtensionFromFS(path); if (mrk_ext && *mrk_ext == ".mrk") /// TODO - { setNonAdaptive(); - mark_size_in_bytes = part->getMarkSize(is_adaptive); - marks_file_extension = part->getMarkExtension(is_adaptive); - } } void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index 7a51f9031cc..da3bcad7dbc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -31,9 +31,10 @@ public: /// Approximate bytes size of one granule size_t index_granularity_bytes; - MergeTreeIndexGranularityInfo(const MergeTreeDataPartPtr & part); + MergeTreeIndexGranularityInfo(const MergeTreeData & storage, + const String & mark_file_extension_, UInt8 mark_size_in_bytes_); - void changeGranularityIfRequired(const MergeTreeDataPartPtr & part); + void changeGranularityIfRequired(const String & path); String getMarksFilePath(const String & path_prefix) const { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 7049a4d2028..b2130b9704e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -4,6 +4,14 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NOT_FOUND_EXPECTED_DATA_PART; + extern const int MEMORY_LIMIT_EXCEEDED; + extern const int ARGUMENT_OUT_OF_BOUND; +} + MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & mark_ranges_, const ReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_) @@ -11,6 +19,30 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr , uncompressed_cache_, mark_cache_, mark_ranges_ , settings_, avg_value_size_hints_) { + size_t buffer_size = settings.max_read_buffer_size; + + if (uncompressed_cache) + { + auto buffer = std::make_unique( + "data.bin", uncompressed_cache, 0, settings.min_bytes_to_use_direct_io, buffer_size); + + // if (profile_callback) + // buffer->setProfileCallback(profile_callback, clock_type); + + cached_buffer = std::move(buffer); + data_buffer = cached_buffer.get(); + } + else + { + auto buffer = std::make_unique( + "data.bin", 0, settings.min_bytes_to_use_direct_io, buffer_size); + + // if (profile_callback) + // buffer->setProfileCallback(profile_callback, clock_type); + + non_cached_buffer = std::move(buffer); + data_buffer = non_cached_buffer.get(); + } } size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) @@ -20,9 +52,68 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, UNUSED(max_rows_to_read); UNUSED(res); - return 0; + size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); + size_t read_rows = 0; + + size_t ind = 0; + for (const auto & it : columns) + { + bool append = res.has(it.name); + if (!append) + res.insert(ColumnWithTypeAndName(it.type->createColumn(), it.type, it.name)); + + /// To keep offsets shared. TODO Very dangerous. Get rid of this. + MutableColumnPtr column = res.getByName(it.name).column->assumeMutable(); + + try + { + size_t column_size_before_reading = column->size(); + + readData(it.name, *it.type, *column, from_mark, ind++, rows_to_read); + + /// For elements of Nested, column_size_before_reading may be greater than column size + /// if offsets are not empty and were already read, but elements are empty. + if (column->size()) + read_rows = std::max(read_rows, column->size() - column_size_before_reading); + } + catch (Exception & e) + { + /// Better diagnostics. + e.addMessage("(while reading column " + it.name + ")"); + throw; + } + + if (column->size()) + res.getByName(it.name).column = std::move(column); + else + res.erase(it.name); + } + + return read_rows; } + +void MergeTreeReaderCompact::readData( + const String & name, const IDataType & type, IColumn & column, + size_t from_mark, size_t column_position, size_t rows_to_read) +{ + seekToMark(from_mark, column_position); + + IDataType::DeserializeBinaryBulkSettings deserialize_settings; + deserialize_settings.getter = [&](IDataType::SubstreamPath) -> ReadBuffer * { return data_buffer; }; + deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; + deserialize_settings.position_independent_encoding = false; + + IDataType::DeserializeBinaryBulkStatePtr state; + type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); + type.deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state); + + if (column.size() != rows_to_read) + throw Exception("Cannot read all data in NativeBlockInputStream. Rows read: " + toString(column.size()) + ". Rows expected: "+ toString(rows_to_read) + ".", + ErrorCodes::CANNOT_READ_ALL_DATA); +} + + void MergeTreeReaderCompact::loadMarks() { const auto & index_granularity_info = data_part->index_granularity_info; @@ -64,5 +155,42 @@ void MergeTreeReaderCompact::loadMarks() marks = MarksInCompressedFileCompact(marks_array, columns.size()); } +const MarkInCompressedFile & MergeTreeReaderCompact::getMark(size_t row, size_t col) +{ + if (!marks.initialized()) + loadMarks(); + return marks.getMark(row, col); +} + +void MergeTreeReaderCompact::seekToMark(size_t row, size_t col) +{ + MarkInCompressedFile mark = getMark(row, col); + + try + { + if (cached_buffer) + cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + if (non_cached_buffer) + non_cached_buffer->seek(mark.offset_in_compressed_file, mark.offset_in_decompressed_block); + } + catch (Exception & e) + { + /// Better diagnostics. + if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) + e.addMessage("(while seeking to mark (" + toString(row) + ", " + toString(col) + ")"); + + throw; + } +} + + +void MergeTreeReaderCompact::seekToStart() +{ + if (cached_buffer) + cached_buffer->seek(0, 0); + if (non_cached_buffer) + non_cached_buffer->seek(0, 0); +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 4b4c5bde85c..f9d78fb22ab 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -33,6 +33,8 @@ public: return sizeof(MarkInCompressedFile) * columns_num; } + bool initialized() { return data != nullptr; } + private: MarksPtr data; size_t columns_num; @@ -58,10 +60,18 @@ public: private: ReadBuffer * data_buffer; + std::unique_ptr cached_buffer; + std::unique_ptr non_cached_buffer; MarksInCompressedFileCompact marks; void loadMarks(); + void seekToStart(); + void seekToMark(size_t row, size_t col); + const MarkInCompressedFile & getMark(size_t row, size_t col); + + void readData(const String & name, const IDataType & type, IColumn & column, + size_t from_mark, size_t column_position, size_t rows_to_read); static auto constexpr NAME_OF_FILE_WITH_DATA = "data"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 327afb26a85..46013172846 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -134,6 +134,8 @@ size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, si throw; } + std::cerr << "(MergeTreeReaderWide::readRows) read_rows: " << read_rows << "\n"; + return read_rows; } @@ -173,6 +175,7 @@ void MergeTreeReaderWide::readData( size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) { + std::cerr << "(MergeTreeReaderWide::readData) max_rows_to_read: " << max_rows_to_read << "\n"; auto get_stream_getter = [&](bool stream_for_prefix) -> IDataType::InputStreamGetter { return [&, stream_for_prefix](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 9e33b4594f3..4d9407fd22a 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -35,11 +35,9 @@ MergedBlockOutputStream::MergedBlockOutputStream( , columns_list(columns_list_) { init(); + const auto & columns = storage.getColumns(); for (const auto & it : columns_list) - { - const auto columns = storage.getColumns(); addStreams(part_path, it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec_), 0, false); - } } MergedBlockOutputStream::MergedBlockOutputStream( @@ -71,11 +69,9 @@ MergedBlockOutputStream::MergedBlockOutputStream( } } + const auto & columns = storage.getColumns(); for (const auto & it : columns_list) - { - const auto columns = storage.getColumns(); addStreams(part_path, it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec_), total_size, false); - } } std::string MergedBlockOutputStream::getPartPath() const @@ -234,6 +230,7 @@ void MergedBlockOutputStream::init() void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation) { + std::cerr << "(MergedBlockOutputStream::writeImpl) block.rows(): " << block.rows() << "\n"; block.checkNumberOfRows(); size_t rows = block.rows(); if (!rows) @@ -248,44 +245,41 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm /// The set of written offset columns so that you do not write shared offsets of nested structures columns several times WrittenOffsetColumns offset_columns; + Block primary_key_block; + Block skip_indexes_block; + auto primary_key_column_names = storage.primary_key_columns; + std::set skip_indexes_column_names_set; for (const auto & index : storage.skip_indices) std::copy(index->columns.cbegin(), index->columns.cend(), std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); - /// Here we will add the columns related to the Primary Key, then write the index. - std::vector primary_key_columns(primary_key_column_names.size()); - std::map primary_key_column_name_to_position; - for (size_t i = 0, size = primary_key_column_names.size(); i < size; ++i) { const auto & name = primary_key_column_names[i]; - - if (!primary_key_column_name_to_position.emplace(name, i).second) - throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); - - primary_key_columns[i] = block.getByName(name); + primary_key_block.insert(i, block.getByName(name)); /// Reorder primary key columns in advance and add them to `primary_key_columns`. if (permutation) - primary_key_columns[i].column = primary_key_columns[i].column->permute(*permutation, 0); + { + auto & column = primary_key_block.getByPosition(i); + column.column = column.column->permute(*permutation, 0); + } } - /// The same for skip indexes columns - std::vector skip_indexes_columns(skip_indexes_column_names.size()); - std::map skip_indexes_column_name_to_position; - for (size_t i = 0, size = skip_indexes_column_names.size(); i < size; ++i) { const auto & name = skip_indexes_column_names[i]; - skip_indexes_column_name_to_position.emplace(name, i); - skip_indexes_columns[i] = block.getByName(name); + skip_indexes_block.insert(i, block.getByName(name)); /// Reorder index columns in advance. if (permutation) - skip_indexes_columns[i].column = skip_indexes_columns[i].column->permute(*permutation, 0); + { + auto & column = skip_indexes_block.getByPosition(i); + column.column = column.column->permute(*permutation, 0); + } } if (index_columns.empty()) @@ -294,8 +288,8 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm last_index_row.resize(primary_key_column_names.size()); for (size_t i = 0, size = primary_key_column_names.size(); i < size; ++i) { - index_columns[i] = primary_key_columns[i].column->cloneEmpty(); - last_index_row[i] = primary_key_columns[i].cloneEmpty(); + last_index_row[i] = primary_key_block.getByPosition(i).cloneEmpty(); + index_columns[i] = last_index_row[i].column->cloneEmpty(); } } @@ -313,6 +307,8 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } + writer->write(block, primary_key_block, skip_indexes_block, current_mark, index_offset); + size_t new_index_offset = 0; /// Now write the data. auto it = columns_list.begin(); @@ -322,9 +318,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm if (permutation) { - auto primary_column_it = primary_key_column_name_to_position.find(it->name); - auto skip_index_column_it = skip_indexes_column_name_to_position.find(it->name); - if (primary_key_column_name_to_position.end() != primary_column_it) + if (primary_key_block.has(it->name)) { const auto & primary_column = *primary_key_columns[primary_column_it->second].column; std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, primary_column, offset_columns, false, serialization_states[i], current_mark); @@ -347,6 +341,8 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } + std::cerr << "(MergedBlockOutputStream::writeImpl) new_index_offset: " << new_index_offset << "\n"; + rows_count += rows; /// Should be written before index offset update, because we calculate, From 3ebb2ab7c6c4c56a11c2646d6c4a3dc53056bc86 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Sat, 19 Oct 2019 19:49:36 +0300 Subject: [PATCH 0040/2007] polymorphic parts (development) --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 67 +++++++++ .../MergeTree/IMergeTreeDataPartWriter.h | 69 ++++----- .../MergeTree/IMergedBlockOutputStream.cpp | 139 ++---------------- .../MergeTree/IMergedBlockOutputStream.h | 98 ++---------- .../MergeTree/MergeTreeReaderSettings.h | 11 ++ .../MergeTree/MergedBlockOutputStream.cpp | 15 +- .../MergeTree/MergedBlockOutputStream.h | 4 +- .../MergedColumnOnlyOutputStream.cpp | 2 +- 8 files changed, 140 insertions(+), 265 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp new file mode 100644 index 00000000000..0518529d88c --- /dev/null +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -0,0 +1,67 @@ +#include + +namespace DB +{ + IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( + const String & part_path_, + const NamesAndTypesList & columns_list_, + const IColumn::Permutation * permutation_, + const String & marks_file_extension_, + const CompressionCodecPtr & default_codec_, + size_t max_compress_block_size_, + size_t aio_threshold_) + : part_path(part_path_) + , columns_list(columns_list_) + , permutation(permutation_) + , marks_file_extension(marks_file_extension_) + , default_codec(default_codec_) + , max_compress_block_size(max_compress_block_size_) + , aio_threshold(aio_threshold_) {} + +void IMergeTreeDataPartWriter::ColumnStream::finalize() +{ + compressed.next(); + plain_file->next(); + marks.next(); +} + +void IMergeTreeDataPartWriter::ColumnStream::sync() +{ + plain_file->sync(); + marks_file.sync(); +} + +IMergeTreeDataPartWriter::ColumnStream::ColumnStream( + const String & escaped_column_name_, + const String & data_path_, + const std::string & data_file_extension_, + const std::string & marks_path_, + const std::string & marks_file_extension_, + const CompressionCodecPtr & compression_codec_, + size_t max_compress_block_size_, + size_t estimated_size_, + size_t aio_threshold_) : + escaped_column_name(escaped_column_name_), + data_file_extension{data_file_extension_}, + marks_file_extension{marks_file_extension_}, + plain_file(createWriteBufferFromFileBase(data_path_ + data_file_extension, estimated_size_, aio_threshold_, max_compress_block_size_)), + plain_hashing(*plain_file), compressed_buf(plain_hashing, compression_codec_), compressed(compressed_buf), + marks_file(marks_path_ + marks_file_extension, 4096, O_TRUNC | O_CREAT | O_WRONLY), marks(marks_file) +{ +} + +void IMergeTreeDataPartWriter::ColumnStream::addToChecksums(MergeTreeData::DataPart::Checksums & checksums) +{ + String name = escaped_column_name; + + checksums.files[name + data_file_extension].is_compressed = true; + checksums.files[name + data_file_extension].uncompressed_size = compressed.count(); + checksums.files[name + data_file_extension].uncompressed_hash = compressed.getHash(); + checksums.files[name + data_file_extension].file_size = plain_hashing.count(); + checksums.files[name + data_file_extension].file_hash = plain_hashing.getHash(); + + checksums.files[name + marks_file_extension].file_size = marks.count(); + checksums.files[name + marks_file_extension].file_hash = marks.getHash(); +} + +} \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 687179ece26..3287eb20647 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -34,7 +35,7 @@ public: /// compressed -> compressed_buf -> plain_hashing -> plain_file std::unique_ptr plain_file; - HashingWriteBuffer plain_hashing; + HashingWriteBuffer plain_hashi ng; CompressedWriteBuffer compressed_buf; HashingWriteBuffer compressed; @@ -49,47 +50,41 @@ public: void addToChecksums(MergeTreeData::DataPart::Checksums & checksums); }; + using ColumnStreamPtr = std::unique_ptr; + + IMergeTreeDataPartWriter( + const String & part_path, + const MergeTreeData & storage, + const NamesAndTypesList & columns_list, + const IColumn::Permutation * permutation, + const String & marks_file_extension, + const CompressionCodecPtr & default_codec, + size_t max_compress_block_size, + size_t aio_threshold); + virtual size_t write( - const Block & block, size_t from_mark, size_t offset, + const Block & block, size_t from_mark, size_t offset, const MergeTreeIndexGranularity & index_granularity, /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; - virtual std::pair writeColumn( - const String & name, - const IDataType & type, - const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - size_t from_mark) = 0; - - // /// Write single granule of one column (rows between 2 marks) - // virtual size_t writeSingleGranule( - // const String & name, - // const IDataType & type, - // const IColumn & column, - // WrittenOffsetColumns & offset_columns, - // bool skip_offsets, - // IDataType::SerializeBinaryBulkStatePtr & serialization_state, - // IDataType::SerializeBinaryBulkSettings & serialize_settings, - // size_t from_row, - // size_t number_of_rows, - // bool write_marks) = 0; - - // /// Write mark for column - // virtual void writeSingleMark( - // const String & name, - // const IDataType & type, - // WrittenOffsetColumns & offset_columns, - // bool skip_offsets, - // size_t number_of_rows, - // DB::IDataType::SubstreamPath & path) = 0; protected: - void start(); + using SerializationState = IDataType::SerializeBinaryBulkStatePtr; + using SerializationStates = std::vector; - const NamesAndTypesList & columns_list; - IColumn::Permutation * permutation; - bool started = false; + String part_path; + NamesAndTypesList columns_list; + const IColumn::Permutation * permutation; + const String marks_file_extension; + + const MergeTreeData & storage; + + CompressionCodecPtr default_codec; + + size_t min_compress_block_size; + size_t max_compress_block_size; + size_t aio_threshold; }; -} \ No newline at end of file +using MergeTreeDataPartWriterPtr = std::unique_ptr; + +} diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 01d9de2f60c..d8ff61fbda2 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -1,6 +1,6 @@ #include #include - +#include namespace DB { @@ -17,25 +17,19 @@ namespace IMergedBlockOutputStream::IMergedBlockOutputStream( - MergeTreeData & storage_, - const String & part_path_, - size_t min_compress_block_size_, - size_t max_compress_block_size_, + const MergeTreeDataPartPtr & data_part, CompressionCodecPtr codec_, - size_t aio_threshold_, + const WriterSettings & writer_settings_, bool blocks_are_granules_size_, const std::vector & indices_to_recalc, const MergeTreeIndexGranularity & index_granularity_, - const MergeTreeIndexGranularityInfo * index_granularity_info_) - : storage(storage_) - , part_path(part_path_) - , min_compress_block_size(min_compress_block_size_) - , max_compress_block_size(max_compress_block_size_) - , aio_threshold(aio_threshold_) - , can_use_adaptive_granularity(index_granularity_info_ ? index_granularity_info_->is_adaptive : storage.canUseAdaptiveGranularity()) - , marks_file_extension(can_use_adaptive_granularity ? getAdaptiveMrkExtension() : getNonAdaptiveMrkExtension()) + bool can_use_adaptive_granularity_) + : storage(data_part->storage) + , part_path(data_part->getFullPath()) + , writer_settings(writer_settings_) + , can_use_adaptive_granularity(can_use_adaptive_granularity_) , blocks_are_granules_size(blocks_are_granules_size_) - , index_granularity(index_granularity_) + , index_granularity(data_part->index_granularity) , compute_granularity(index_granularity.empty()) , codec(std::move(codec_)) , skip_indices(indices_to_recalc) @@ -43,42 +37,10 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( { if (blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); + + writer = data_part->getWriter(columns_list, permutation, default_codec, writer_settings); } -void IMergedBlockOutputStream::addStreams( - const String & path, - const String & name, - const IDataType & type, - const CompressionCodecPtr & effective_codec, - size_t estimated_size, - bool skip_offsets) -{ - IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path) - { - if (skip_offsets && !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) - return; - - String stream_name = IDataType::getFileNameForStream(name, substream_path); - - /// Shared offsets for Nested type. - if (column_streams.count(stream_name)) - return; - - column_streams[stream_name] = std::make_unique( - stream_name, - path + stream_name, DATA_FILE_EXTENSION, - path + stream_name, marks_file_extension, - effective_codec, - max_compress_block_size, - estimated_size, - aio_threshold); - }; - - IDataType::SubstreamPath stream_path; - type.enumerateStreams(callback, stream_path); -} - - IDataType::OutputStreamGetter IMergedBlockOutputStream::createStreamGetter( const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets) { @@ -150,39 +112,6 @@ void IMergedBlockOutputStream::fillIndexGranularity(const Block & block) can_use_adaptive_granularity); } -void IMergedBlockOutputStream::writeSingleMark( - const String & name, - const IDataType & type, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - size_t number_of_rows, - DB::IDataType::SubstreamPath & path) -{ - type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) - { - bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && skip_offsets) - return; - - String stream_name = IDataType::getFileNameForStream(name, substream_path); - - /// Don't write offsets more than one time for Nested type. - if (is_offsets && offset_columns.count(stream_name)) - return; - - ColumnStream & stream = *column_streams[stream_name]; - - /// There could already be enough data to compress into the new block. - if (stream.compressed.offset() >= min_compress_block_size) - stream.compressed.next(); - - writeIntBinary(stream.plain_hashing.count(), stream.marks); - writeIntBinary(stream.compressed.offset(), stream.marks); - if (can_use_adaptive_granularity) - writeIntBinary(number_of_rows, stream.marks); - }, path); -} - size_t IMergedBlockOutputStream::writeSingleGranule( const String & name, const IDataType & type, @@ -421,50 +350,4 @@ void IMergedBlockOutputStream::finishSkipIndicesSerialization( /// Implementation of IMergedBlockOutputStream::ColumnStream. -IMergedBlockOutputStream::ColumnStream::ColumnStream( - const String & escaped_column_name_, - const String & data_path_, - const std::string & data_file_extension_, - const std::string & marks_path_, - const std::string & marks_file_extension_, - const CompressionCodecPtr & compression_codec_, - size_t max_compress_block_size_, - size_t estimated_size_, - size_t aio_threshold_) : - escaped_column_name(escaped_column_name_), - data_file_extension{data_file_extension_}, - marks_file_extension{marks_file_extension_}, - plain_file(createWriteBufferFromFileBase(data_path_ + data_file_extension, estimated_size_, aio_threshold_, max_compress_block_size_)), - plain_hashing(*plain_file), compressed_buf(plain_hashing, compression_codec_), compressed(compressed_buf), - marks_file(marks_path_ + marks_file_extension, 4096, O_TRUNC | O_CREAT | O_WRONLY), marks(marks_file) -{ -} - -void IMergedBlockOutputStream::ColumnStream::finalize() -{ - compressed.next(); - plain_file->next(); - marks.next(); -} - -void IMergedBlockOutputStream::ColumnStream::sync() -{ - plain_file->sync(); - marks_file.sync(); -} - -void IMergedBlockOutputStream::ColumnStream::addToChecksums(MergeTreeData::DataPart::Checksums & checksums) -{ - String name = escaped_column_name; - - checksums.files[name + data_file_extension].is_compressed = true; - checksums.files[name + data_file_extension].uncompressed_size = compressed.count(); - checksums.files[name + data_file_extension].uncompressed_hash = compressed.getHash(); - checksums.files[name + data_file_extension].file_size = plain_hashing.count(); - checksums.files[name + data_file_extension].file_hash = plain_hashing.getHash(); - - checksums.files[name + marks_file_extension].file_size = marks.count(); - checksums.files[name + marks_file_extension].file_hash = marks.getHash(); -} - } diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index 37aa2203a72..5eba217e373 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -16,16 +17,13 @@ class IMergedBlockOutputStream : public IBlockOutputStream { public: IMergedBlockOutputStream( - MergeTreeData & storage_, - const String & part_path_, - size_t min_compress_block_size_, - size_t max_compress_block_size_, - CompressionCodecPtr default_codec_, - size_t aio_threshold_, + const MergeTreeDataPartPtr & data_part, + CompressionCodecPtr codec_, + const WriterSettings & writer_settings_, bool blocks_are_granules_size_, const std::vector & indices_to_recalc, const MergeTreeIndexGranularity & index_granularity_, - const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr); + bool can_use_adaptive_granularity_); using WrittenOffsetColumns = std::set; @@ -33,83 +31,8 @@ protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; using SerializationStates = std::vector; - struct ColumnStream - { - ColumnStream( - const String & escaped_column_name_, - const String & data_path_, - const std::string & data_file_extension_, - const std::string & marks_path_, - const std::string & marks_file_extension_, - const CompressionCodecPtr & compression_codec_, - size_t max_compress_block_size_, - size_t estimated_size_, - size_t aio_threshold_); - - String escaped_column_name; - std::string data_file_extension; - std::string marks_file_extension; - - /// compressed -> compressed_buf -> plain_hashing -> plain_file - std::unique_ptr plain_file; - HashingWriteBuffer plain_hashing; - CompressedWriteBuffer compressed_buf; - HashingWriteBuffer compressed; - - /// marks -> marks_file - WriteBufferFromFile marks_file; - HashingWriteBuffer marks; - - void finalize(); - - void sync(); - - void addToChecksums(MergeTreeData::DataPart::Checksums & checksums); - }; - - using ColumnStreams = std::map>; - - void addStreams(const String & path, const String & name, const IDataType & type, - const CompressionCodecPtr & codec, size_t estimated_size, bool skip_offsets); - - IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); - /// Write data of one column. - /// Return how many marks were written and - /// how many rows were written for last mark - std::pair writeColumn( - const String & name, - const IDataType & type, - const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - size_t from_mark - ); - - /// Write single granule of one column (rows between 2 marks) - size_t writeSingleGranule( - const String & name, - const IDataType & type, - const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - IDataType::SerializeBinaryBulkSettings & serialize_settings, - size_t from_row, - size_t number_of_rows, - bool write_marks); - - /// Write mark for column - void writeSingleMark( - const String & name, - const IDataType & type, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - size_t number_of_rows, - DB::IDataType::SubstreamPath & path); - /// Count index_granularity for block and store in `index_granularity` void fillIndexGranularity(const Block & block); @@ -130,15 +53,10 @@ protected: SerializationStates serialization_states; String part_path; - ColumnStreams column_streams; - /// The offset to the first row of the block for which you want to write the index. size_t index_offset = 0; - size_t min_compress_block_size; - size_t max_compress_block_size; - - size_t aio_threshold; + WriterSettings writer_settings; size_t current_mark = 0; @@ -156,10 +74,12 @@ protected: CompressionCodecPtr codec; std::vector skip_indices; - std::vector> skip_indices_streams; + std::vector> skip_indices_streams; MergeTreeIndexAggregators skip_indices_aggregators; std::vector skip_index_filling; + std::unique_ptr writer; + const bool with_final_mark; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h index e0332ce3d3a..16a05cde09f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h @@ -9,5 +9,16 @@ namespace DB size_t max_read_buffer_size = 0; bool save_marks_in_cache = false; }; +<<<<<<< HEAD +======= + + struct WriterSettings + { + size_t min_compress_block_size; + size_t max_compress_block_size; + size_t aio_threshold; + String marks_file_extension; + }; +>>>>>>> 03dc18db16... tmp } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 4d9407fd22a..f7318cc594f 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -105,7 +105,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( { /// Finish columns serialization. { - auto & settings = storage.global_context.getSettingsRef(); + const auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; @@ -242,8 +242,8 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm if (compute_granularity) fillIndexGranularity(block); - /// The set of written offset columns so that you do not write shared offsets of nested structures columns several times - WrittenOffsetColumns offset_columns; + Block primary_key_block; + Block skip_indexes_block; Block primary_key_block; Block skip_indexes_block; @@ -293,12 +293,9 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } - if (serialization_states.empty()) - { - serialization_states.reserve(columns_list.size()); - WrittenOffsetColumns tmp_offset_columns; - IDataType::SerializeBinaryBulkSettings settings; + size_t new_index_offset = writer->write(block, primary_key_block, skip_indexes_block, current_mark, index_offset); +<<<<<<< HEAD for (const auto & col : columns_list) { settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); @@ -340,6 +337,8 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, false, serialization_states[i], current_mark); } } +======= +>>>>>>> 03dc18db16... tmp std::cerr << "(MergedBlockOutputStream::writeImpl) new_index_offset: " << new_index_offset << "\n"; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h index ff45934f106..abdd836ddb2 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -14,14 +14,14 @@ class MergedBlockOutputStream final : public IMergedBlockOutputStream { public: MergedBlockOutputStream( - MergeTreeData & storage_, + const MergeTreeData & storage_, const String & part_path_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, bool blocks_are_granules_size_ = false); MergedBlockOutputStream( - MergeTreeData & storage_, + const MergeTreeData & storage_, const String & part_path_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index a9e8f24d588..4a368a2f101 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -4,7 +4,7 @@ namespace DB { MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( - MergeTreeData & storage_, const Block & header_, const String & part_path_, bool sync_, + const MergeTreeData & storage_, const Block & header_, const String & part_path_, bool sync_, CompressionCodecPtr default_codec_, bool skip_offsets_, const std::vector & indices_to_recalc_, WrittenOffsetColumns & already_written_offset_columns_, From ce36cf88acb762f785c0974d084042c9f3037346 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 21 Oct 2019 02:36:27 +0300 Subject: [PATCH 0041/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.h | 22 +++++++++++++++++-- .../MergeTree/IMergeTreeDataPartWriter.cpp | 2 ++ .../MergeTree/IMergeTreeDataPartWriter.h | 4 ++-- .../MergeTree/IMergedBlockOutputStream.h | 4 ++-- .../MergeTree/MergeTreeReaderSettings.h | 5 +---- 5 files changed, 27 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 17a799ac9d0..021da48ce5a 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -15,7 +15,8 @@ #include #include #include -// #include +#include +#include // #include // #include #include @@ -47,6 +48,7 @@ public: using Checksums = MergeTreeDataPartChecksums; using Checksum = MergeTreeDataPartChecksums::Checksum; using MergeTreeReaderPtr = std::unique_ptr; + using MergeTreeWriterPtr = std::unique_ptr; using ValueSizeMap = std::map; // virtual BlockInputStreamPtr readAll() = 0; @@ -62,7 +64,23 @@ public: const ReaderSettings & reader_settings_, const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}) const = 0; - + + const String & part_path_, + const MergeTreeData & storage_, + const NamesAndTypesList & columns_list_, + const IColumn::Permutation * permutation_, + const String & marks_file_extension_, + const CompressionCodecPtr & default_codec_, + size_t max_compress_block_size_, + size_t aio_threshold_ + + virtual MergeTreeWriterPtr getWriter( + const NamesAndTypesList & columns_list, + const IColumn::Permutation * permutation, + const CompressionCodecPtr & default_codec_, + const WriterSettings & writer_settings, + ) + // virtual MergeTreeWriterPtr getWriter() const = 0; virtual bool isStoredOnDisk() const = 0; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 0518529d88c..f64a060438d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -4,6 +4,7 @@ namespace DB { IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const String & part_path_, + const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, const IColumn::Permutation * permutation_, const String & marks_file_extension_, @@ -11,6 +12,7 @@ namespace DB size_t max_compress_block_size_, size_t aio_threshold_) : part_path(part_path_) + , storage(storage_) , columns_list(columns_list_) , permutation(permutation_) , marks_file_extension(marks_file_extension_) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 3287eb20647..a4c214f029e 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -35,7 +35,7 @@ public: /// compressed -> compressed_buf -> plain_hashing -> plain_file std::unique_ptr plain_file; - HashingWriteBuffer plain_hashi ng; + HashingWriteBuffer plain_hashing; CompressedWriteBuffer compressed_buf; HashingWriteBuffer compressed; @@ -72,11 +72,11 @@ protected: using SerializationStates = std::vector; String part_path; + const MergeTreeData & storage; NamesAndTypesList columns_list; const IColumn::Permutation * permutation; const String marks_file_extension; - const MergeTreeData & storage; CompressionCodecPtr default_codec; diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index 5eba217e373..e3338308b36 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -8,7 +8,7 @@ #include #include #include - +#include namespace DB { @@ -48,7 +48,7 @@ protected: void calculateAndSerializeSkipIndices(const ColumnsWithTypeAndName & skip_indexes_columns, size_t rows); void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); protected: - MergeTreeData & storage; + const MergeTreeData & storage; SerializationStates serialization_states; String part_path; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h index 16a05cde09f..5859328eef3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h @@ -9,16 +9,13 @@ namespace DB size_t max_read_buffer_size = 0; bool save_marks_in_cache = false; }; -<<<<<<< HEAD -======= struct WriterSettings { size_t min_compress_block_size; size_t max_compress_block_size; size_t aio_threshold; - String marks_file_extension; + // String marks_file_extension; }; ->>>>>>> 03dc18db16... tmp } From 1297991cef2c90bd964a3ab1909ecfa3bb79ef74 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 21 Oct 2019 03:28:29 +0300 Subject: [PATCH 0042/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.h | 12 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 22 +- .../MergeTree/IMergeTreeDataPartWriter.h | 3 +- .../MergeTreeDataPartWriterCompact.cpp | 84 ++---- .../MergeTreeDataPartWriterCompact.h | 26 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 255 ++++++++++++++---- .../MergeTree/MergeTreeDataPartWriterWide.h | 77 ++++-- .../MergeTree/MergeTreeReaderStream.h | 1 + 8 files changed, 299 insertions(+), 181 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 021da48ce5a..66d7281aadd 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -65,21 +65,11 @@ public: const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}) const = 0; - const String & part_path_, - const MergeTreeData & storage_, - const NamesAndTypesList & columns_list_, - const IColumn::Permutation * permutation_, - const String & marks_file_extension_, - const CompressionCodecPtr & default_codec_, - size_t max_compress_block_size_, - size_t aio_threshold_ - virtual MergeTreeWriterPtr getWriter( const NamesAndTypesList & columns_list, const IColumn::Permutation * permutation, const CompressionCodecPtr & default_codec_, - const WriterSettings & writer_settings, - ) + const WriterSettings & writer_settings) const = 0; // virtual MergeTreeWriterPtr getWriter() const = 0; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index f64a060438d..37b7acdd9fc 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -52,18 +52,18 @@ IMergeTreeDataPartWriter::ColumnStream::ColumnStream( { } -void IMergeTreeDataPartWriter::ColumnStream::addToChecksums(MergeTreeData::DataPart::Checksums & checksums) -{ - String name = escaped_column_name; +// void IMergeTreeDataPartWriter::ColumnStream::addToChecksums(MergeTreeData::DataPart::Checksums & checksums) +// { +// String name = escaped_column_name; - checksums.files[name + data_file_extension].is_compressed = true; - checksums.files[name + data_file_extension].uncompressed_size = compressed.count(); - checksums.files[name + data_file_extension].uncompressed_hash = compressed.getHash(); - checksums.files[name + data_file_extension].file_size = plain_hashing.count(); - checksums.files[name + data_file_extension].file_hash = plain_hashing.getHash(); +// checksums.files[name + data_file_extension].is_compressed = true; +// checksums.files[name + data_file_extension].uncompressed_size = compressed.count(); +// checksums.files[name + data_file_extension].uncompressed_hash = compressed.getHash(); +// checksums.files[name + data_file_extension].file_size = plain_hashing.count(); +// checksums.files[name + data_file_extension].file_hash = plain_hashing.getHash(); - checksums.files[name + marks_file_extension].file_size = marks.count(); - checksums.files[name + marks_file_extension].file_hash = marks.getHash(); -} +// checksums.files[name + marks_file_extension].file_size = marks.count(); +// checksums.files[name + marks_file_extension].file_hash = marks.getHash(); +// } } \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index a4c214f029e..0ff2c7c502a 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -47,10 +47,11 @@ public: void sync(); - void addToChecksums(MergeTreeData::DataPart::Checksums & checksums); + void addToChecksums(IMergeTreeDataPart::Checksums & checksums); }; using ColumnStreamPtr = std::unique_ptr; + using ColumnStreams = std::map; IMergeTreeDataPartWriter( const String & part_path, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 86a2ff3f188..68f9eb52c3a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -3,34 +3,14 @@ namespace DB { -size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule( - const ColumnWithTypeAndName & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - IDataType::SerializeBinaryBulkSettings & serialize_settings, - size_t from_row, - size_t number_of_rows) -{ -} - - -size_t MergeTreeDataPartWriterCompact::write(const Block & block, size_t from_mark, size_t offset, +size_t MergeTreeDataPartWriterCompact::write(const Block & block, size_t from_mark, size_t index_offset, + const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) { - if (!started) - start(); - size_t total_rows = block.rows(); size_t current_mark = from_mark; size_t current_row = 0; - - IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.getter = [&ostr](IDataType::SubstreamPath) -> WriteBuffer * { return &ostr; }; - serialize_settings.position_independent_encoding = false; - serialize_settings.low_cardinality_max_dictionary_size = 0; - ColumnsWithTypeAndName columns_to_write(columns_list.size()); auto it = columns_list.begin(); for (size_t i = 0; i < columns_list.size(); ++i, ++it) @@ -55,62 +35,52 @@ size_t MergeTreeDataPartWriterCompact::write(const Block & block, size_t from_ma { bool write_marks = true; size_t rows_to_write; - if (current_row == 0 && offset != 0) + if (current_row == 0 && index_offset != 0) { - rows_to_write = offset; + rows_to_write = index_offset; write_marks = false; } else { - rows_to_write = index_granularity->getMarkRows(current_mark); - } - - for (size_t i = 0; i < columns_to_write.size(); ++i) - { - current_row = writeColumnSingleGranule(columns_to_write[i], offset_columns, skip_offsets, serialization_states[i], serialize_settings, current_row, rows_to_write); + rows_to_write = index_granularity.getMarkRows(current_mark); } if (write_marks) { - writeMark(); + writeIntBinary(rows_to_write, stream->marks); + for (size_t i = 0; i < columns_to_write.size(); ++i) + { + writeIntBinary(stream->plain_hashing.count(), stream->marks); + writeIntBinary(stream->compressed.offset(), stream->marks); + current_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); + } ++current_mark; } + else + { + for (size_t i = 0; i < columns_to_write.size(); ++i) + current_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); + } } /// We always write end granule for block in Compact parts. return 0; } -size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - IDataType::SerializeBinaryBulkSettings & serialize_settings, - size_t from_row, - size_t number_of_rows) +size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) { - column.type->serializeBinaryBulkStatePrefix(serialize_settings, serialization_state); - column.type->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, serialization_state); - column.type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_state); -} + IDataType::SerializeBinaryBulkStatePtr state; + IDataType::SerializeBinaryBulkSettings serialize_settings; -void MergeTreeDataPartWriterWide::start() -{ - if (started) - return; + serialize_settings.getter = [&stream](IDataType::SubstreamPath) -> WriteBuffer * { return &stream->compressed; }; + serialize_settings.position_independent_encoding = false; + serialize_settings.low_cardinality_max_dictionary_size = 0; - started = true; + column.type->serializeBinaryBulkStatePrefix(serialize_settings, state); + column.type->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, state); + column.type->serializeBinaryBulkStateSuffix(serialize_settings, state); - serialization_states.reserve(columns_list.size()); - WrittenOffsetColumns tmp_offset_columns; - IDataType::SerializeBinaryBulkSettings settings; - - for (const auto & col : columns_list) - { - settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); - serialization_states.emplace_back(nullptr); - col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); - } + return from_row + number_of_rows; } } \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index e5990faab77..614005ca523 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -6,37 +6,17 @@ namespace DB class MergeTreeDataPartWriterCompact : IMergeTreeDataPartWriter { public: - size_t write(const Block & block, size_t from_mark, size_t offset, + size_t write(const Block & block, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) override; - std::pair writeColumn( - const String & name, - const IDataType & type, - const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - size_t from_mark) override; - /// Write single granule of one column (rows between 2 marks) size_t writeColumnSingleGranule( const ColumnWithTypeAndName & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - IDataType::SerializeBinaryBulkSettings & serialize_settings, size_t from_row, size_t number_of_rows); - - void writeSingleMark() - -protected: - void start() override; - + private: - ColumnStream stream; - MergeTreeIndexGranularity * index_granularity = nullptr; - Columns columns_to_write; + ColumnStreamPtr stream; }; } \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index cac516ca10f..9a5b389f956 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -1,67 +1,210 @@ -// #include +#include -// namespace DB -// { +namespace DB +{ -// size_t MergeTreeDataPartWriterWide::write(size_t current_mark, const Block & block) -// { -// if (!started) -// start(); +size_t MergeTreeDataPartWriterWide::write(const Block & block, size_t from_mark, size_t index_offset, + const MergeTreeIndexGranularity & index_granularity, + const Block & primary_key_block, const Block & skip_indexes_block) +{ + if (serialization_states.empty()) + { + serialization_states.reserve(columns_list.size()); + WrittenOffsetColumns tmp_offset_columns; + IDataType::SerializeBinaryBulkSettings settings; -// size_t index_offset = 0; -// auto it = columns_list.begin(); -// for (size_t i = 0; i < columns_list.size(); ++i, ++it) -// { -// const ColumnWithTypeAndName & column = block.getByName(it->name); + for (const auto & col : columns_list) + { + settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); + serialization_states.emplace_back(nullptr); + col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); + } + } -// if (permutation) -// { -// auto primary_column_it = primary_key_column_name_to_position.find(it->name); -// auto skip_index_column_it = skip_indexes_column_name_to_position.find(it->name); + WrittenOffsetColumns offset_columns; + size_t new_index_offset = 0; -// if (primary_key_column_name_to_position.end() != primary_column_it) -// { -// const auto & primary_column = *primary_key_columns[primary_column_it->second].column; -// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, primary_column, offset_columns, false, serialization_states[i], current_mark); -// } -// else if (skip_indexes_column_name_to_position.end() != skip_index_column_it) -// { -// const auto & index_column = *skip_indexes_columns[skip_index_column_it->second].column; -// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, index_column, offset_columns, false, serialization_states[i], current_mark); -// } -// else -// { -// /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. -// ColumnPtr permuted_column = column.column->permute(*permutation, 0); -// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, *permuted_column, offset_columns, false, serialization_states[i], current_mark); -// } -// } -// else -// { -// std::tie(std::ignore, index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, false, serialization_states[i], current_mark); -// } -// } + auto it = columns_list.begin(); + for (size_t i = 0; i < columns_list.size(); ++i, ++it) + { + const ColumnWithTypeAndName & column = block.getByName(it->name); -// return index_offset; -// } + if (permutation) + { + if (primary_key_block.has(it->name)) + { + const auto & primary_column = *primary_key_block.getByName(it->name).column; + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, primary_column, offset_columns, false, serialization_states[i], from_mark, index_offset); + } + else if (skip_indexes_block.has(it->name)) + { + const auto & index_column = *skip_indexes_block.getByName(it->name).column; + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, index_column, offset_columns, false, serialization_states[i], from_mark, index_offset); + } + else + { + /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. + ColumnPtr permuted_column = column.column->permute(*permutation, 0); + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *permuted_column, offset_columns, false, serialization_states[i], from_mark, index_offset); + } + } + else + { + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, false, serialization_states[i], from_mark, index_offset); + } + } -// void MergeTreeDataPartWriterWide::start() -// { -// if (started) -// return; + return new_index_offset; +} -// started = true; +void MergeTreeDataPartWriterWide::writeSingleMark( + const String & name, + const IDataType & type, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + size_t number_of_rows, + DB::IDataType::SubstreamPath & path) +{ + type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) + { + bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; + if (is_offsets && skip_offsets) + return; -// serialization_states.reserve(columns_list.size()); -// WrittenOffsetColumns tmp_offset_columns; -// IDataType::SerializeBinaryBulkSettings settings; + String stream_name = IDataType::getFileNameForStream(name, substream_path); -// for (const auto & col : columns_list) -// { -// settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); -// serialization_states.emplace_back(nullptr); -// col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); -// } -// } + /// Don't write offsets more than one time for Nested type. + if (is_offsets && offset_columns.count(stream_name)) + return; -// } + ColumnStream & stream = *column_streams[stream_name]; + + /// There could already be enough data to compress into the new block. + if (stream.compressed.offset() >= min_compress_block_size) + stream.compressed.next(); + + writeIntBinary(stream.plain_hashing.count(), stream.marks); + writeIntBinary(stream.compressed.offset(), stream.marks); + if (can_use_adaptive_granularity) + writeIntBinary(number_of_rows, stream.marks); + }, path); +} + +size_t MergeTreeDataPartWriterWide::writeSingleGranule( + const String & name, + const IDataType & type, + const IColumn & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + IDataType::SerializeBinaryBulkSettings & serialize_settings, + size_t from_row, + size_t number_of_rows, + bool write_marks) +{ + if (write_marks) + writeSingleMark(name, type, offset_columns, skip_offsets, number_of_rows, serialize_settings.path); + + type.serializeBinaryBulkWithMultipleStreams(column, from_row, number_of_rows, serialize_settings, serialization_state); + + /// So that instead of the marks pointing to the end of the compressed block, there were marks pointing to the beginning of the next one. + type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) + { + bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; + if (is_offsets && skip_offsets) + return; + + String stream_name = IDataType::getFileNameForStream(name, substream_path); + + /// Don't write offsets more than one time for Nested type. + if (is_offsets && offset_columns.count(stream_name)) + return; + + column_streams[stream_name]->compressed.nextIfAtEnd(); + }, serialize_settings.path); + + return from_row + number_of_rows; +} + +/// column must not be empty. (column.size() !== 0) + +std::pair MergeTreeDataPartWriterWide::writeColumn( + const String & name, + const IDataType & type, + const IColumn & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + size_t from_mark) +{ + auto & settings = storage.global_context.getSettingsRef(); + IDataType::SerializeBinaryBulkSettings serialize_settings; + serialize_settings.getter = createStreamGetter(name, offset_columns, skip_offsets); + serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; + serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; + + size_t total_rows = column.size(); + size_t current_row = 0; + size_t current_column_mark = from_mark; + while (current_row < total_rows) + { + size_t rows_to_write; + bool write_marks = true; + + /// If there is `index_offset`, then the first mark goes not immediately, but after this number of rows. + if (current_row == 0 && index_offset != 0) + { + write_marks = false; + rows_to_write = index_offset; + } + else + { + if (index_granularity.getMarksCount() <= current_column_mark) + throw Exception( + "Incorrect size of index granularity expect mark " + toString(current_column_mark) + " totally have marks " + toString(index_granularity.getMarksCount()), + ErrorCodes::LOGICAL_ERROR); + + rows_to_write = index_granularity.getMarkRows(current_column_mark); + } + + current_row = writeSingleGranule( + name, + type, + column, + offset_columns, + skip_offsets, + serialization_state, + serialize_settings, + current_row, + rows_to_write, + write_marks + ); + + if (write_marks) + current_column_mark++; + } + + /// Memoize offsets for Nested types, that are already written. They will not be written again for next columns of Nested structure. + type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) + { + bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; + if (is_offsets) + { + String stream_name = IDataType::getFileNameForStream(name, substream_path); + offset_columns.insert(stream_name); + } + }, serialize_settings.path); + + return std::make_pair(current_column_mark, current_row - total_rows); + + +void MergeTreeDataPartWriterWide::start() +{ + if (started) + return; + + started = true; + +} + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 7de78a12da3..949d71148ad 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -1,28 +1,61 @@ -// #include +#include -// namespace DB -// { +namespace DB +{ -// class MergeTreeDataPartWriterWide : IMergeTreeDataPartWriter -// { -// public: -// size_t write(size_t current_mark, const Block & block) override; +class MergeTreeDataPartWriterWide : IMergeTreeDataPartWriter +{ +public: + size_t write(const Block & block, size_t from_mark, size_t index_offset, + const MergeTreeIndexGranularity & index_granularity, + const Block & primary_key_block, const Block & skip_indexes_block) override; -// std::pair writeColumn( -// const String & name, -// const IDataType & type, -// const IColumn & column, -// WrittenOffsetColumns & offset_columns, -// bool skip_offsets, -// IDataType::SerializeBinaryBulkStatePtr & serialization_state, -// size_t from_mark) override; + void addStreams(const String & path, const String & name, const IDataType & type, + const CompressionCodecPtr & codec, size_t estimated_size, bool skip_offsets); -// protected: -// void start() override; -// private: -// SerializationStates serialization_states; -// NameSet permuted_columns; -// }; + IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); -// } \ No newline at end of file + /// Write data of one column. + /// Return how many marks were written and + /// how many rows were written for last mark + std::pair writeColumn( + const String & name, + const IDataType & type, + const IColumn & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + size_t from_mark, + size_t index_offset); + +private: + /// Write single granule of one column (rows between 2 marks) + size_t writeSingleGranule( + const String & name, + const IDataType & type, + const IColumn & column, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + IDataType::SerializeBinaryBulkStatePtr & serialization_state, + IDataType::SerializeBinaryBulkSettings & serialize_settings, + size_t from_row, + size_t number_of_rows, + bool write_marks); + + /// Write mark for column + void writeSingleMark( + const String & name, + const IDataType & type, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + size_t number_of_rows, + DB::IDataType::SubstreamPath & path); + + SerializationStates serialization_states; + bool can_use_adaptive_granularity; + ColumnStreams column_streams; + +}; + +} \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index 85ab78dde9f..5476b37c866 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB From 8df8bcea9460012604d76e8a80150f138d555fe3 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 21 Oct 2019 18:33:59 +0300 Subject: [PATCH 0043/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.h | 24 +-- .../MergeTree/IMergeTreeDataPartWriter.cpp | 59 ++++--- .../MergeTree/IMergeTreeDataPartWriter.h | 18 +- .../MergeTree/IMergeTreeDataPart_fwd.h | 14 ++ .../src/Storages/MergeTree/IMergeTreeReader.h | 2 + .../MergeTree/IMergedBlockOutputStream.cpp | 159 ++--------------- .../MergeTree/IMergedBlockOutputStream.h | 7 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 4 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 17 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 13 ++ .../MergeTree/MergeTreeDataPartCompact.h | 6 + .../MergeTree/MergeTreeDataPartWide.cpp | 13 ++ .../MergeTree/MergeTreeDataPartWide.h | 6 + .../MergeTreeDataPartWriterCompact.cpp | 4 +- .../MergeTreeDataPartWriterCompact.h | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 109 ++++++++++-- .../MergeTree/MergeTreeDataPartWriterWide.h | 27 ++- .../MergeTree/MergeTreeDataWriter.cpp | 4 +- .../MergeTree/MergedBlockOutputStream.cpp | 165 ++++++------------ .../MergeTree/MergedBlockOutputStream.h | 6 +- .../MergedColumnOnlyOutputStream.cpp | 53 +++--- .../MergeTree/MergedColumnOnlyOutputStream.h | 4 +- 22 files changed, 336 insertions(+), 380 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 66d7281aadd..dccb52246dd 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -15,8 +15,9 @@ #include #include #include -#include -#include +// #include +// #include +// #include // #include // #include #include @@ -30,27 +31,27 @@ namespace DB { - struct ColumnSize; - class MergeTreeData; - class IMergeTreeReader; +struct ColumnSize; +class MergeTreeData; + +class IMergeTreeReader; +class IMergeTreeWriter; namespace ErrorCodes { extern const int NOT_IMPLEMETED; } - -// class MergeTreeDataPartOnDisk; - class IMergeTreeDataPart : public std::enable_shared_from_this { public: using Checksums = MergeTreeDataPartChecksums; using Checksum = MergeTreeDataPartChecksums::Checksum; - using MergeTreeReaderPtr = std::unique_ptr; - using MergeTreeWriterPtr = std::unique_ptr; using ValueSizeMap = std::map; + using MergeTreeReaderPtr = std::unique_ptr; + using MergeTreeWriterPtr = std::unique_ptr; + // virtual BlockInputStreamPtr readAll() = 0; // virtual BlockInputStreamPtr read() = 0; // virtual BlockInputStreamPtr readWithThreadPool() = 0; @@ -71,8 +72,6 @@ public: const CompressionCodecPtr & default_codec_, const WriterSettings & writer_settings) const = 0; - // virtual MergeTreeWriterPtr getWriter() const = 0; - virtual bool isStoredOnDisk() const = 0; virtual void remove() const = 0; @@ -352,5 +351,6 @@ private: }; using MergeTreeDataPartState = IMergeTreeDataPart::State; +using MergeTreeDataPartPtr = std::shared_ptr; } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 37b7acdd9fc..b4a8feb135e 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -1,24 +1,25 @@ #include +#include namespace DB { - IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( - const String & part_path_, - const MergeTreeData & storage_, - const NamesAndTypesList & columns_list_, - const IColumn::Permutation * permutation_, - const String & marks_file_extension_, - const CompressionCodecPtr & default_codec_, - size_t max_compress_block_size_, - size_t aio_threshold_) - : part_path(part_path_) - , storage(storage_) - , columns_list(columns_list_) - , permutation(permutation_) - , marks_file_extension(marks_file_extension_) - , default_codec(default_codec_) - , max_compress_block_size(max_compress_block_size_) - , aio_threshold(aio_threshold_) {} +IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( + const String & part_path_, + const MergeTreeData & storage_, + const NamesAndTypesList & columns_list_, + const IColumn::Permutation * permutation_, + const String & marks_file_extension_, + const CompressionCodecPtr & default_codec_, + const WriterSettings & settings_) +: part_path(part_path_) +, storage(storage_) +, columns_list(columns_list_) +, permutation(permutation_) +, marks_file_extension(marks_file_extension_) +, default_codec(default_codec_) +, settings(settings_) {} + +IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; void IMergeTreeDataPartWriter::ColumnStream::finalize() { @@ -52,18 +53,18 @@ IMergeTreeDataPartWriter::ColumnStream::ColumnStream( { } -// void IMergeTreeDataPartWriter::ColumnStream::addToChecksums(MergeTreeData::DataPart::Checksums & checksums) -// { -// String name = escaped_column_name; +void IMergeTreeDataPartWriter::ColumnStream::addToChecksums(MergeTreeData::DataPart::Checksums & checksums) +{ + String name = escaped_column_name; -// checksums.files[name + data_file_extension].is_compressed = true; -// checksums.files[name + data_file_extension].uncompressed_size = compressed.count(); -// checksums.files[name + data_file_extension].uncompressed_hash = compressed.getHash(); -// checksums.files[name + data_file_extension].file_size = plain_hashing.count(); -// checksums.files[name + data_file_extension].file_hash = plain_hashing.getHash(); + checksums.files[name + data_file_extension].is_compressed = true; + checksums.files[name + data_file_extension].uncompressed_size = compressed.count(); + checksums.files[name + data_file_extension].uncompressed_hash = compressed.getHash(); + checksums.files[name + data_file_extension].file_size = plain_hashing.count(); + checksums.files[name + data_file_extension].file_hash = plain_hashing.getHash(); -// checksums.files[name + marks_file_extension].file_size = marks.count(); -// checksums.files[name + marks_file_extension].file_hash = marks.getHash(); -// } + checksums.files[name + marks_file_extension].file_size = marks.count(); + checksums.files[name + marks_file_extension].file_hash = marks.getHash(); +} -} \ No newline at end of file +} diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 0ff2c7c502a..3adf125a18a 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -1,3 +1,5 @@ +#pragma once + #include #include #include @@ -6,6 +8,8 @@ #include #include #include +// #include +// #include namespace DB @@ -60,13 +64,16 @@ public: const IColumn::Permutation * permutation, const String & marks_file_extension, const CompressionCodecPtr & default_codec, - size_t max_compress_block_size, - size_t aio_threshold); + const WriterSettings & settings); virtual size_t write( const Block & block, size_t from_mark, size_t offset, const MergeTreeIndexGranularity & index_granularity, /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; + + // virtual void writeFinalMarks() = 0; + + virtual ~IMergeTreeDataPartWriter(); protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; @@ -78,14 +85,11 @@ protected: const IColumn::Permutation * permutation; const String marks_file_extension; - CompressionCodecPtr default_codec; - size_t min_compress_block_size; - size_t max_compress_block_size; - size_t aio_threshold; + WriterSettings settings; }; -using MergeTreeDataPartWriterPtr = std::unique_ptr; +using MergeTreeWriterPtr = std::unique_ptr; } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h new file mode 100644 index 00000000000..c5e10608e0f --- /dev/null +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h @@ -0,0 +1,14 @@ +#pragma once +#include + +namespace DB +{ + class IMergeTreeDataPart; + class IMergeTreeReader; + class IMergeTreeWriter; + + using MergeTreeMutableDataPartPtr = std::shared_ptr; + using MergeTreeDataPartPtr = std::shared_ptr; + using MergeTreeReaderPtr = std::unique_ptr; + using MergeTreeWriterPtr = std::unique_ptr; +} diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 02cb8be0505..7f76d6eac88 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -85,4 +85,6 @@ protected: friend class MergeTreeRangeReader::DelayedStream; }; +using MergeTreeReaderPtr = std::unique_ptr; + } diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index d8ff61fbda2..1ac0bdc2a1f 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -1,6 +1,8 @@ #include #include #include +#include + namespace DB { @@ -11,7 +13,7 @@ namespace ErrorCodes namespace { - constexpr auto DATA_FILE_EXTENSION = ".bin"; + // constexpr auto DATA_FILE_EXTENSION = ".bin"; constexpr auto INDEX_FILE_EXTENSION = ".idx"; } @@ -22,7 +24,6 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( const WriterSettings & writer_settings_, bool blocks_are_granules_size_, const std::vector & indices_to_recalc, - const MergeTreeIndexGranularity & index_granularity_, bool can_use_adaptive_granularity_) : storage(data_part->storage) , part_path(data_part->getFullPath()) @@ -37,27 +38,6 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( { if (blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - - writer = data_part->getWriter(columns_list, permutation, default_codec, writer_settings); -} - -IDataType::OutputStreamGetter IMergedBlockOutputStream::createStreamGetter( - const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets) -{ - return [&, skip_offsets] (const IDataType::SubstreamPath & substream_path) -> WriteBuffer * - { - bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && skip_offsets) - return nullptr; - - String stream_name = IDataType::getFileNameForStream(name, substream_path); - - /// Don't write offsets more than one time for Nested type. - if (is_offsets && offset_columns.count(stream_name)) - return nullptr; - - return &column_streams[stream_name]->compressed; - }; } void fillIndexGranularityImpl( @@ -112,129 +92,14 @@ void IMergedBlockOutputStream::fillIndexGranularity(const Block & block) can_use_adaptive_granularity); } -size_t IMergedBlockOutputStream::writeSingleGranule( - const String & name, - const IDataType & type, - const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - IDataType::SerializeBinaryBulkSettings & serialize_settings, - size_t from_row, - size_t number_of_rows, - bool write_marks) -{ - if (write_marks) - writeSingleMark(name, type, offset_columns, skip_offsets, number_of_rows, serialize_settings.path); - - type.serializeBinaryBulkWithMultipleStreams(column, from_row, number_of_rows, serialize_settings, serialization_state); - - /// So that instead of the marks pointing to the end of the compressed block, there were marks pointing to the beginning of the next one. - type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) - { - bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && skip_offsets) - return; - - String stream_name = IDataType::getFileNameForStream(name, substream_path); - - /// Don't write offsets more than one time for Nested type. - if (is_offsets && offset_columns.count(stream_name)) - return; - - column_streams[stream_name]->compressed.nextIfAtEnd(); - }, serialize_settings.path); - - return from_row + number_of_rows; -} - -/// column must not be empty. (column.size() !== 0) - -std::pair IMergedBlockOutputStream::writeColumn( - const String & name, - const IDataType & type, - const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - size_t from_mark) -{ - auto & settings = storage.global_context.getSettingsRef(); - IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.getter = createStreamGetter(name, offset_columns, skip_offsets); - serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; - serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; - - size_t total_rows = column.size(); - size_t current_row = 0; - size_t current_column_mark = from_mark; - while (current_row < total_rows) - { - size_t rows_to_write; - bool write_marks = true; - - /// If there is `index_offset`, then the first mark goes not immediately, but after this number of rows. - if (current_row == 0 && index_offset != 0) - { - write_marks = false; - rows_to_write = index_offset; - } - else - { - if (index_granularity.getMarksCount() <= current_column_mark) - throw Exception( - "Incorrect size of index granularity expect mark " + toString(current_column_mark) + " totally have marks " + toString(index_granularity.getMarksCount()), - ErrorCodes::LOGICAL_ERROR); - - rows_to_write = index_granularity.getMarkRows(current_column_mark); - } - - current_row = writeSingleGranule( - name, - type, - column, - offset_columns, - skip_offsets, - serialization_state, - serialize_settings, - current_row, - rows_to_write, - write_marks - ); - - if (write_marks) - current_column_mark++; - } - - /// Memoize offsets for Nested types, that are already written. They will not be written again for next columns of Nested structure. - type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) - { - bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets) - { - String stream_name = IDataType::getFileNameForStream(name, substream_path); - offset_columns.insert(stream_name); - } - }, serialize_settings.path); - - std::cerr << "(IMergedBlockOutputStream::writeColumn) name: " << name << "\n"; - std::cerr << "(IMergedBlockOutputStream::writeColumn) from_mark: " << from_mark << "\n"; - std::cerr << "(IMergedBlockOutputStream::writeColumn) current_column_mark: " << current_column_mark << "\n"; - std::cerr << "(IMergedBlockOutputStream::writeColumn) current_row: " << current_row << "\n"; - std::cerr << "(IMergedBlockOutputStream::writeColumn) total_rows: " << total_rows; - std::cerr << "(IMergedBlockOutputStream::writeColumn) blocks_are_granules_size: " << blocks_are_granules_size << "\n"; - - return std::make_pair(current_column_mark, current_row - total_rows); -} - void IMergedBlockOutputStream::writeFinalMark( const std::string & column_name, const DataTypePtr column_type, WrittenOffsetColumns & offset_columns, - bool skip_offsets, + bool /* skip_offsets */, DB::IDataType::SubstreamPath & path) { - writeSingleMark(column_name, *column_type, offset_columns, skip_offsets, 0, path); + // writer->writeSingleMark(column_name, *column_type, offset_columns, skip_offsets, 0, path); /// Memoize information about offsets column_type->enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) { @@ -253,22 +118,20 @@ void IMergedBlockOutputStream::initSkipIndices() { String stream_name = index->getFileName(); skip_indices_streams.emplace_back( - std::make_unique( + std::make_unique( stream_name, part_path + stream_name, INDEX_FILE_EXTENSION, part_path + stream_name, marks_file_extension, - codec, max_compress_block_size, - 0, aio_threshold)); + codec, writer_settings.max_compress_block_size, + 0, writer_settings.aio_threshold)); skip_indices_aggregators.push_back(index->createIndexAggregator()); skip_index_filling.push_back(0); } } void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( - const ColumnsWithTypeAndName & skip_indexes_columns, size_t rows) + const Block & skip_indexes_block, size_t rows) { - /// Creating block for update - Block indices_update_block(skip_indexes_columns); size_t skip_index_current_data_mark = 0; /// Filling and writing skip indices like in IMergedBlockOutputStream::writeColumn @@ -293,7 +156,7 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( skip_indices_aggregators[i] = index->createIndexAggregator(); skip_index_filling[i] = 0; - if (stream.compressed.offset() >= min_compress_block_size) + if (stream.compressed.offset() >= writer_settings.min_compress_block_size) stream.compressed.next(); writeIntBinary(stream.plain_hashing.count(), stream.marks); @@ -308,7 +171,7 @@ void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( } size_t pos = prev_pos; - skip_indices_aggregators[i]->update(indices_update_block, &pos, limit); + skip_indices_aggregators[i]->update(skip_indexes_block, &pos, limit); if (pos == prev_pos + limit) { diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index e3338308b36..fe8ad5cee14 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -7,8 +7,8 @@ #include #include #include -#include #include +#include namespace DB { @@ -22,7 +22,6 @@ public: const WriterSettings & writer_settings_, bool blocks_are_granules_size_, const std::vector & indices_to_recalc, - const MergeTreeIndexGranularity & index_granularity_, bool can_use_adaptive_granularity_); using WrittenOffsetColumns = std::set; @@ -45,7 +44,7 @@ protected: DB::IDataType::SubstreamPath & path); void initSkipIndices(); - void calculateAndSerializeSkipIndices(const ColumnsWithTypeAndName & skip_indexes_columns, size_t rows); + void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, size_t rows); void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); protected: const MergeTreeData & storage; @@ -78,7 +77,7 @@ protected: MergeTreeIndexAggregators skip_indices_aggregators; std::vector skip_index_filling; - std::unique_ptr writer; + MergeTreeWriterPtr writer; const bool with_final_mark; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 70ee3b717ec..e0be40c8c99 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1667,16 +1667,14 @@ void MergeTreeData::alterDataPart( IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( - *this, + part, in.getHeader(), - part->getFullPath(), true /* sync */, compression_codec, true /* skip_offsets */, /// Don't recalc indices because indices alter is restricted std::vector{}, unused_written_offsets, - part->index_granularity, &part->index_granularity_info); in.readPrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index b26a20efeb1..14486300df4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -553,6 +553,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor throw Exception("Directory " + new_part_tmp_path + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); MergeTreeData::DataPart::ColumnToSize merged_column_to_size; + + /// FIXME // for (const MergeTreeData::DataPartPtr & part : parts) // part->accumulateColumnSizes(merged_column_to_size); @@ -730,8 +732,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor merged_stream = std::make_shared(merged_stream, data, new_data_part, time_of_merge, force_ttl); MergedBlockOutputStream to{ - data, - new_part_tmp_path, + new_data_part, merging_columns, compression_codec, merged_column_to_size, @@ -831,9 +832,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor ColumnGathererStream column_gathered_stream(column_name, column_part_streams, rows_sources_read_buf); MergedColumnOnlyOutputStream column_to( - data, + new_data_part, column_gathered_stream.getHeader(), - new_part_tmp_path, false, compression_codec, false, @@ -841,8 +841,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// because all of them were already recalculated and written /// as key part of vertical merge std::vector{}, - written_offset_columns, - to.getIndexGranularity()); + written_offset_columns); size_t column_elems_written = 0; @@ -991,7 +990,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor IMergeTreeDataPart::MinMaxIndex minmax_idx; - MergedBlockOutputStream out(data, new_part_tmp_path, all_columns, compression_codec); + MergedBlockOutputStream out(new_data_part, all_columns, compression_codec); in->readPrefix(); out.writePrefix(); @@ -1092,15 +1091,13 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( - data, + new_data_part, updated_header, - new_part_tmp_path, /* sync = */ false, compression_codec, /* skip_offsets = */ false, std::vector(indices_to_recalc.begin(), indices_to_recalc.end()), unused_written_offsets, - source_part->index_granularity, &source_part->index_granularity_info ); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index c209b4d5c67..251779f2d9f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -85,6 +85,19 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( mark_cache, mark_ranges, reader_settings, avg_value_size_hints); } +IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( + const NamesAndTypesList & columns_list, + const IColumn::Permutation * permutation, + const CompressionCodecPtr & default_codec, + const WriterSettings & writer_settings) const +{ + UNUSED(columns_list); + UNUSED(permutation); + UNUSED(default_codec); + UNUSED(writer_settings); + return {}; +} + /// Takes into account the fact that several columns can e.g. share their .size substreams. /// When calculating totals these should be counted only once. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index be6b06de68d..4a233843a1b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -57,6 +57,12 @@ public: const ReaderSettings & reader_settings_, const ValueSizeMap & avg_value_size_hints = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{}) const override; + + MergeTreeWriterPtr getWriter( + const NamesAndTypesList & columns_list, + const IColumn::Permutation * permutation, + const CompressionCodecPtr & default_codec, + const WriterSettings & writer_settings) const override; bool isStoredOnDisk() const override { return true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 7ec1c282095..aeb18c08f44 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -85,6 +85,19 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader( mark_cache, mark_ranges, reader_settings, avg_value_size_hints, profile_callback); } +IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( + const NamesAndTypesList & columns_list, + const IColumn::Permutation * permutation, + const CompressionCodecPtr & default_codec, + const WriterSettings & writer_settings) const +{ + UNUSED(columns_list); + UNUSED(permutation); + UNUSED(default_codec); + UNUSED(writer_settings); + return {}; +} + /// Takes into account the fact that several columns can e.g. share their .size substreams. /// When calculating totals these should be counted only once. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 9e91bea55bf..5f9672ad015 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -58,6 +58,12 @@ public: const ValueSizeMap & avg_value_size_hints = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{}) const override; + MergeTreeWriterPtr getWriter( + const NamesAndTypesList & columns_list, + const IColumn::Permutation * permutation, + const CompressionCodecPtr & default_codec_, + const WriterSettings & writer_settings) const override; + bool isStoredOnDisk() const override { return true; } void remove() const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 68f9eb52c3a..81271badb27 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -72,7 +72,7 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith IDataType::SerializeBinaryBulkStatePtr state; IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.getter = [&stream](IDataType::SubstreamPath) -> WriteBuffer * { return &stream->compressed; }; + serialize_settings.getter = [this](IDataType::SubstreamPath) -> WriteBuffer * { return &stream->compressed; }; serialize_settings.position_independent_encoding = false; serialize_settings.low_cardinality_max_dictionary_size = 0; @@ -83,4 +83,4 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith return from_row + number_of_rows; } -} \ No newline at end of file +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 614005ca523..6fb641d69ce 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -19,4 +19,4 @@ private: ColumnStreamPtr stream; }; -} \ No newline at end of file +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 9a5b389f956..69ec99fcf74 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -3,6 +3,92 @@ namespace DB { +namespace +{ + constexpr auto DATA_FILE_EXTENSION = ".bin"; +} + +MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( + const String & part_path_, + const MergeTreeData & storage_, + const NamesAndTypesList & columns_list_, + const IColumn::Permutation * permutation_, + const String & marks_file_extension_, + const CompressionCodecPtr & default_codec_, + const WriterSettings & settings_, + const ColumnToSize & merged_column_to_size) + : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, + permutation_, marks_file_extension_, + default_codec_, settings_) +{ + size_t total_size = 0; + if (settings.aio_threshold > 0 && !merged_column_to_size.empty()) + { + for (const auto & it : columns_list) + { + auto it2 = merged_column_to_size.find(it.name); + if (it2 != merged_column_to_size.end()) + total_size += it2->second; + } + } + + const auto & columns = storage.getColumns(); + for (const auto & it : columns_list) + addStreams(it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec), total_size, false); +} + +void MergeTreeDataPartWriterWide::addStreams( + const String & name, + const IDataType & type, + const CompressionCodecPtr & effective_codec, + size_t estimated_size, + bool skip_offsets) +{ + IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path) + { + if (skip_offsets && !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) + return; + + String stream_name = IDataType::getFileNameForStream(name, substream_path); + + /// Shared offsets for Nested type. + if (column_streams.count(stream_name)) + return; + + column_streams[stream_name] = std::make_unique( + stream_name, + part_path + stream_name, DATA_FILE_EXTENSION, + part_path + stream_name, marks_file_extension, + effective_codec, + settings.max_compress_block_size, + estimated_size, + settings.aio_threshold); + }; + + IDataType::SubstreamPath stream_path; + type.enumerateStreams(callback, stream_path); +} + + +IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( + const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets) +{ + return [&, skip_offsets] (const IDataType::SubstreamPath & substream_path) -> WriteBuffer * + { + bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; + if (is_offsets && skip_offsets) + return nullptr; + + String stream_name = IDataType::getFileNameForStream(name, substream_path); + + /// Don't write offsets more than one time for Nested type. + if (is_offsets && offset_columns.count(stream_name)) + return nullptr; + + return &column_streams[stream_name]->compressed; + }; +} + size_t MergeTreeDataPartWriterWide::write(const Block & block, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) @@ -34,23 +120,23 @@ size_t MergeTreeDataPartWriterWide::write(const Block & block, size_t from_mark, if (primary_key_block.has(it->name)) { const auto & primary_column = *primary_key_block.getByName(it->name).column; - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, primary_column, offset_columns, false, serialization_states[i], from_mark, index_offset); + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, primary_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } else if (skip_indexes_block.has(it->name)) { const auto & index_column = *skip_indexes_block.getByName(it->name).column; - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, index_column, offset_columns, false, serialization_states[i], from_mark, index_offset); + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, index_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } else { /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. ColumnPtr permuted_column = column.column->permute(*permutation, 0); - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *permuted_column, offset_columns, false, serialization_states[i], from_mark, index_offset); + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *permuted_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } } else { - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, false, serialization_states[i], from_mark, index_offset); + std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *column.column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } } @@ -80,7 +166,7 @@ void MergeTreeDataPartWriterWide::writeSingleMark( ColumnStream & stream = *column_streams[stream_name]; /// There could already be enough data to compress into the new block. - if (stream.compressed.offset() >= min_compress_block_size) + if (stream.compressed.offset() >= settings.min_compress_block_size) stream.compressed.next(); writeIntBinary(stream.plain_hashing.count(), stream.marks); @@ -132,10 +218,12 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( const String & name, const IDataType & type, const IColumn & column, + const MergeTreeIndexGranularity & index_granularity, WrittenOffsetColumns & offset_columns, bool skip_offsets, IDataType::SerializeBinaryBulkStatePtr & serialization_state, - size_t from_mark) + size_t from_mark, + size_t index_offset) { auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -196,15 +284,6 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( }, serialize_settings.path); return std::make_pair(current_column_mark, current_row - total_rows); - - -void MergeTreeDataPartWriterWide::start() -{ - if (started) - return; - - started = true; - } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 949d71148ad..5b9d89e8d4e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -6,14 +6,23 @@ namespace DB class MergeTreeDataPartWriterWide : IMergeTreeDataPartWriter { public: + + using ColumnToSize = std::map; + + MergeTreeDataPartWriterWide( + const String & part_path, + const MergeTreeData & storage, + const NamesAndTypesList & columns_list, + const IColumn::Permutation * permutation, + const String & marks_file_extension, + const CompressionCodecPtr & default_codec, + const WriterSettings & settings, + const ColumnToSize & merged_column_to_size = {}); + size_t write(const Block & block, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) override; - void addStreams(const String & path, const String & name, const IDataType & type, - const CompressionCodecPtr & codec, size_t estimated_size, bool skip_offsets); - - IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); /// Write data of one column. @@ -23,6 +32,7 @@ public: const String & name, const IDataType & type, const IColumn & column, + const MergeTreeIndexGranularity & index_granularity, WrittenOffsetColumns & offset_columns, bool skip_offsets, IDataType::SerializeBinaryBulkStatePtr & serialization_state, @@ -52,10 +62,17 @@ private: size_t number_of_rows, DB::IDataType::SubstreamPath & path); + void addStreams( + const String & name, + const IDataType & type, + const CompressionCodecPtr & effective_codec, + size_t estimated_size, + bool skip_offsets); + SerializationStates serialization_states; bool can_use_adaptive_granularity; ColumnStreams column_streams; }; -} \ No newline at end of file +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index de23dd21b1f..8d73c4a5809 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -175,7 +175,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// This will generate unique name in scope of current server process. Int64 temp_index = data.insert_increment.get(); - IMergeTreeDataPart::MinMaxIndex minmax_idx; + IMergeTreeDataPart::MinMaxIndex minmax_idx; minmax_idx.update(block, data.minmax_idx_columns); MergeTreePartition partition(std::move(block_with_partition.partition)); @@ -263,7 +263,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa auto compression_codec = data.global_context.chooseCompressionCodec(0, 0); NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); - MergedBlockOutputStream out(data, new_data_part->getFullPath(), columns, compression_codec); + MergedBlockOutputStream out(new_data_part, columns, compression_codec); out.writePrefix(); out.writeWithPermutation(block, perm_ptr); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index f7318cc594f..670e7f2b644 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -20,58 +20,44 @@ namespace ErrorCodes MergedBlockOutputStream::MergedBlockOutputStream( - MergeTreeData & storage_, - const String & part_path_, + const MergeTreeDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, bool blocks_are_granules_size_) : IMergedBlockOutputStream( - storage_, part_path_, storage_.global_context.getSettings().min_compress_block_size, - storage_.global_context.getSettings().max_compress_block_size, default_codec_, - storage_.global_context.getSettings().min_bytes_to_use_direct_io, + data_part_, default_codec_, + { + data_part_->storage.global_context.getSettings().min_compress_block_size, + data_part_->storage.global_context.getSettings().max_compress_block_size, + data_part_->storage.global_context.getSettings().min_bytes_to_use_direct_io + }, blocks_are_granules_size_, - std::vector(std::begin(storage_.skip_indices), std::end(storage_.skip_indices)), - {}) + std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), + data_part_->storage.canUseAdaptiveGranularity()) , columns_list(columns_list_) { init(); - const auto & columns = storage.getColumns(); - for (const auto & it : columns_list) - addStreams(part_path, it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec_), 0, false); } MergedBlockOutputStream::MergedBlockOutputStream( - MergeTreeData & storage_, - const String & part_path_, + const MergeTreeDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, - const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size_, + const MergeTreeData::DataPart::ColumnToSize & /* merged_column_to_size_ */, size_t aio_threshold_, bool blocks_are_granules_size_) : IMergedBlockOutputStream( - storage_, part_path_, storage_.global_context.getSettings().min_compress_block_size, - storage_.global_context.getSettings().max_compress_block_size, default_codec_, - aio_threshold_, blocks_are_granules_size_, - std::vector(std::begin(storage_.skip_indices), std::end(storage_.skip_indices)), {}) + data_part_, default_codec_, + { + data_part_->storage.global_context.getSettings().min_compress_block_size, + data_part_->storage.global_context.getSettings().max_compress_block_size, + aio_threshold_ + }, + blocks_are_granules_size_, + std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), {}) , columns_list(columns_list_) { init(); - - /// If summary size is more than threshold than we will use AIO - size_t total_size = 0; - if (aio_threshold > 0) - { - for (const auto & it : columns_list) - { - auto it2 = merged_column_to_size_.find(it.name); - if (it2 != merged_column_to_size_.end()) - total_size += it2->second; - } - } - - const auto & columns = storage.getColumns(); - for (const auto & it : columns_list) - addStreams(part_path, it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec_), total_size, false); } std::string MergedBlockOutputStream::getPartPath() const @@ -105,23 +91,24 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( { /// Finish columns serialization. { - const auto & settings = storage.global_context.getSettingsRef(); - IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; - serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; - WrittenOffsetColumns offset_columns; - auto it = columns_list.begin(); - for (size_t i = 0; i < columns_list.size(); ++i, ++it) - { - if (!serialization_states.empty()) - { - serialize_settings.getter = createStreamGetter(it->name, offset_columns, false); - it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); - } + /// FIXME + // const auto & settings = storage.global_context.getSettingsRef(); + // IDataType::SerializeBinaryBulkSettings serialize_settings; + // serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; + // serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; + // WrittenOffsetColumns offset_columns; + // auto it = columns_list.begin(); + // for (size_t i = 0; i < columns_list.size(); ++i, ++it) + // { + // if (!serialization_states.empty()) + // { + // serialize_settings.getter = createStreamGetter(it->name, offset_columns, false); + // it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); + // } - if (with_final_mark && rows_count != 0) - writeFinalMark(it->name, it->type, offset_columns, false, serialize_settings.path); - } + // if (with_final_mark && rows_count != 0) + // writeFinalMark(it->name, it->type, offset_columns, false, serialize_settings.path); + // } } if (with_final_mark && rows_count != 0) @@ -155,15 +142,16 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( index_stream = nullptr; } - for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) - { - it->second->finalize(); - it->second->addToChecksums(checksums); - } + /// FIXME + // for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) + // { + // it->second->finalize(); + // it->second->addToChecksums(checksums); + // } finishSkipIndicesSerialization(checksums); - column_streams.clear(); + // column_streams.clear(); if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { @@ -245,9 +233,6 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm Block primary_key_block; Block skip_indexes_block; - Block primary_key_block; - Block skip_indexes_block; - auto primary_key_column_names = storage.primary_key_columns; std::set skip_indexes_column_names_set; @@ -293,60 +278,12 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } - size_t new_index_offset = writer->write(block, primary_key_block, skip_indexes_block, current_mark, index_offset); - -<<<<<<< HEAD - for (const auto & col : columns_list) - { - settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); - serialization_states.emplace_back(nullptr); - col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); - } - } - - writer->write(block, primary_key_block, skip_indexes_block, current_mark, index_offset); - - size_t new_index_offset = 0; - /// Now write the data. - auto it = columns_list.begin(); - for (size_t i = 0; i < columns_list.size(); ++i, ++it) - { - const ColumnWithTypeAndName & column = block.getByName(it->name); - - if (permutation) - { - if (primary_key_block.has(it->name)) - { - const auto & primary_column = *primary_key_columns[primary_column_it->second].column; - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, primary_column, offset_columns, false, serialization_states[i], current_mark); - } - else if (skip_indexes_column_name_to_position.end() != skip_index_column_it) - { - const auto & index_column = *skip_indexes_columns[skip_index_column_it->second].column; - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, index_column, offset_columns, false, serialization_states[i], current_mark); - } - else - { - /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. - ColumnPtr permuted_column = column.column->permute(*permutation, 0); - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *permuted_column, offset_columns, false, serialization_states[i], current_mark); - } - } - else - { - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, false, serialization_states[i], current_mark); - } - } -======= ->>>>>>> 03dc18db16... tmp - - std::cerr << "(MergedBlockOutputStream::writeImpl) new_index_offset: " << new_index_offset << "\n"; - + size_t new_index_offset = writer->write(block, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block); rows_count += rows; /// Should be written before index offset update, because we calculate, /// indices of currently written granules - calculateAndSerializeSkipIndices(skip_indexes_columns, rows); + calculateAndSerializeSkipIndices(skip_indexes_block, rows); { /** While filling index (index_columns), disable memory tracker. @@ -362,11 +299,11 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm { if (storage.hasPrimaryKey()) { - for (size_t j = 0, size = primary_key_columns.size(); j < size; ++j) + for (size_t j = 0, size = primary_key_block.rows(); j < size; ++j) { - const IColumn & primary_column = *primary_key_columns[j].column.get(); - index_columns[j]->insertFrom(primary_column, i); - primary_key_columns[j].type->serializeBinary(primary_column, i, *index_stream); + const auto & primary_column = primary_key_block.getByPosition(j); + index_columns[j]->insertFrom(*primary_column.column, i); + primary_column.type->serializeBinary(*primary_column.column, i, *index_stream); } } @@ -379,9 +316,9 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } /// store last index row to write final mark at the end of column - for (size_t j = 0, size = primary_key_columns.size(); j < size; ++j) + for (size_t j = 0, size = primary_key_block.rows(); j < size; ++j) { - const IColumn & primary_column = *primary_key_columns[j].column.get(); + const IColumn & primary_column = *primary_key_block.getByPosition(j).column.get(); auto mutable_column = std::move(*last_index_row[j].column).mutate(); if (!mutable_column->empty()) mutable_column->popBack(1); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h index abdd836ddb2..0701ceaa8c9 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -14,15 +14,13 @@ class MergedBlockOutputStream final : public IMergedBlockOutputStream { public: MergedBlockOutputStream( - const MergeTreeData & storage_, - const String & part_path_, + const MergeTreeDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, bool blocks_are_granules_size_ = false); MergedBlockOutputStream( - const MergeTreeData & storage_, - const String & part_path_, + const MergeTreeDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size_, diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 4a368a2f101..ea0753ee0a1 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -4,20 +4,21 @@ namespace DB { MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( - const MergeTreeData & storage_, const Block & header_, const String & part_path_, bool sync_, + const MergeTreeDataPartPtr & data_part_, const Block & header_, bool sync_, CompressionCodecPtr default_codec_, bool skip_offsets_, const std::vector & indices_to_recalc_, WrittenOffsetColumns & already_written_offset_columns_, - const MergeTreeIndexGranularity & index_granularity_, const MergeTreeIndexGranularityInfo * index_granularity_info_) : IMergedBlockOutputStream( - storage_, part_path_, storage_.global_context.getSettings().min_compress_block_size, - storage_.global_context.getSettings().max_compress_block_size, default_codec_, - storage_.global_context.getSettings().min_bytes_to_use_direct_io, + data_part_, default_codec_, + { + data_part_->storage.global_context.getSettings().min_compress_block_size, + data_part_->storage.global_context.getSettings().max_compress_block_size, + data_part_->storage.global_context.getSettings().min_bytes_to_use_direct_io, + }, false, indices_to_recalc_, - index_granularity_, - index_granularity_info_), + index_granularity_info_ ? index_granularity_info_->is_adaptive : data_part_->storage.canUseAdaptiveGranularity()), header(header_), sync(sync_), skip_offsets(skip_offsets_), already_written_offset_columns(already_written_offset_columns_) { @@ -30,9 +31,11 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( const auto & col = header.getByName(column_name); const auto columns = storage.getColumns(); - addStreams(part_path, col.name, *col.type, columns.getCodecOrDefault(col.name, codec), 0, skip_offsets); + /// FIXME + // addStreams(part_path, col.name, *col.type, columns.getCodecOrDefault(col.name, codec), 0, skip_offsets); serialization_states.emplace_back(nullptr); - settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); + /// FIXME + // settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); } @@ -65,8 +68,9 @@ void MergedColumnOnlyOutputStream::write(const Block & block) WrittenOffsetColumns offset_columns = already_written_offset_columns; for (size_t i = 0; i < header.columns(); ++i) { - const ColumnWithTypeAndName & column = block.getByName(header.getByPosition(i).name); - std::tie(new_current_mark, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, skip_offsets, serialization_states[i], current_mark); + /// FIXME + // const ColumnWithTypeAndName & column = block.getByName(header.getByPosition(i).name); + // std::tie(new_current_mark, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, skip_offsets, serialization_states[i], current_mark); } /// Should be written before index offset update, because we calculate, @@ -94,7 +98,8 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG for (size_t i = 0, size = header.columns(); i < size; ++i) { auto & column = header.getByPosition(i); - serialize_settings.getter = createStreamGetter(column.name, already_written_offset_columns, skip_offsets); + /// FIXME + // serialize_settings.getter = createStreamGetter(column.name, already_written_offset_columns, skip_offsets); column.type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); /// We wrote at least one row @@ -104,19 +109,23 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG MergeTreeData::DataPart::Checksums checksums; - for (auto & column_stream : column_streams) - { - column_stream.second->finalize(); - if (sync) - column_stream.second->sync(); + /// FIXME - column_stream.second->addToChecksums(checksums); - } + UNUSED(sync); - finishSkipIndicesSerialization(checksums); + // for (auto & column_stream : column_streams) + // { + // column_stream.second->finalize(); + // if (sync) + // column_stream.second->sync(); - column_streams.clear(); - serialization_states.clear(); + // column_stream.second->addToChecksums(checksums); + // } + + // finishSkipIndicesSerialization(checksums); + + // column_streams.clear(); + // serialization_states.clear(); return checksums; } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index f19c970ac41..d0cc316b319 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB { @@ -13,11 +14,10 @@ public: /// Pass empty 'already_written_offset_columns' first time then and pass the same object to subsequent instances of MergedColumnOnlyOutputStream /// if you want to serialize elements of Nested data structure in different instances of MergedColumnOnlyOutputStream. MergedColumnOnlyOutputStream( - MergeTreeData & storage_, const Block & header_, const String & part_path_, bool sync_, + const MergeTreeDataPartPtr & data_part, const Block & header_, bool sync_, CompressionCodecPtr default_codec_, bool skip_offsets_, const std::vector & indices_to_recalc_, WrittenOffsetColumns & already_written_offset_columns_, - const MergeTreeIndexGranularity & index_granularity_, const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr); Block getHeader() const override { return header; } From 1d3f005538ca7fc1db04548e61fd25dd5e747568 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 21 Oct 2019 20:23:06 +0300 Subject: [PATCH 0044/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.h | 5 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 2 - .../MergeTree/IMergeTreeDataPartWriter.h | 9 ++- .../MergeTree/MergeTreeDataPartCompact.cpp | 2 - .../MergeTree/MergeTreeDataPartCompact.h | 1 - .../MergeTree/MergeTreeDataPartWide.cpp | 12 ++-- .../MergeTree/MergeTreeDataPartWide.h | 1 - .../MergeTreeDataPartWriterCompact.cpp | 21 +++++- .../MergeTreeDataPartWriterCompact.h | 9 ++- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 65 +++++++++++++++++-- .../MergeTree/MergeTreeDataPartWriterWide.h | 15 ++++- .../MergeTree/MergedBlockOutputStream.cpp | 49 ++++---------- 12 files changed, 122 insertions(+), 69 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index dccb52246dd..d8815986d49 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -35,7 +35,7 @@ struct ColumnSize; class MergeTreeData; class IMergeTreeReader; -class IMergeTreeWriter; +class IMergeTreeDataPartWriter; namespace ErrorCodes { @@ -50,7 +50,7 @@ public: using ValueSizeMap = std::map; using MergeTreeReaderPtr = std::unique_ptr; - using MergeTreeWriterPtr = std::unique_ptr; + using MergeTreeWriterPtr = std::unique_ptr; // virtual BlockInputStreamPtr readAll() = 0; // virtual BlockInputStreamPtr read() = 0; @@ -68,7 +68,6 @@ public: virtual MergeTreeWriterPtr getWriter( const NamesAndTypesList & columns_list, - const IColumn::Permutation * permutation, const CompressionCodecPtr & default_codec_, const WriterSettings & writer_settings) const = 0; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index b4a8feb135e..f1bd137be6b 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -7,14 +7,12 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, - const IColumn::Permutation * permutation_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const WriterSettings & settings_) : part_path(part_path_) , storage(storage_) , columns_list(columns_list_) -, permutation(permutation_) , marks_file_extension(marks_file_extension_) , default_codec(default_codec_) , settings(settings_) {} diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 3adf125a18a..f39399a2818 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -61,17 +61,17 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, - const IColumn::Permutation * permutation, const String & marks_file_extension, const CompressionCodecPtr & default_codec, const WriterSettings & settings); virtual size_t write( - const Block & block, size_t from_mark, size_t offset, const MergeTreeIndexGranularity & index_granularity, + const Block & block, const IColumn::Permutation * permutation, + size_t from_mark, size_t offset, const MergeTreeIndexGranularity & index_granularity, /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; - - // virtual void writeFinalMarks() = 0; + + virtual void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) = 0; virtual ~IMergeTreeDataPartWriter(); @@ -82,7 +82,6 @@ protected: String part_path; const MergeTreeData & storage; NamesAndTypesList columns_list; - const IColumn::Permutation * permutation; const String marks_file_extension; CompressionCodecPtr default_codec; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 251779f2d9f..dba86814959 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -87,12 +87,10 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( const NamesAndTypesList & columns_list, - const IColumn::Permutation * permutation, const CompressionCodecPtr & default_codec, const WriterSettings & writer_settings) const { UNUSED(columns_list); - UNUSED(permutation); UNUSED(default_codec); UNUSED(writer_settings); return {}; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 4a233843a1b..7b9e9631a49 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -60,7 +60,6 @@ public: MergeTreeWriterPtr getWriter( const NamesAndTypesList & columns_list, - const IColumn::Permutation * permutation, const CompressionCodecPtr & default_codec, const WriterSettings & writer_settings) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index aeb18c08f44..8d55ba399b9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -24,6 +24,8 @@ #include #include +#include +#include namespace DB @@ -87,15 +89,13 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader( IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const NamesAndTypesList & columns_list, - const IColumn::Permutation * permutation, const CompressionCodecPtr & default_codec, const WriterSettings & writer_settings) const { - UNUSED(columns_list); - UNUSED(permutation); - UNUSED(default_codec); - UNUSED(writer_settings); - return {}; + return std::make_unique( + getFullPath(), storage, columns_list, + index_granularity_info.marks_file_extension, + default_codec, writer_settings); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 5f9672ad015..81c3bb776a5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -60,7 +60,6 @@ public: MergeTreeWriterPtr getWriter( const NamesAndTypesList & columns_list, - const IColumn::Permutation * permutation, const CompressionCodecPtr & default_codec_, const WriterSettings & writer_settings) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 81271badb27..3fbd61b8dc5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -3,7 +3,9 @@ namespace DB { -size_t MergeTreeDataPartWriterCompact::write(const Block & block, size_t from_mark, size_t index_offset, +size_t MergeTreeDataPartWriterCompact::write( + const Block & block, const IColumn::Permutation * permutation, + size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) { @@ -83,4 +85,21 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith return from_row + number_of_rows; } +void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) +{ + if (write_final_mark) + { + writeIntBinary(0, stream->marks); + for (size_t i = 0; i < columns_list.size(); ++i) + { + writeIntBinary(stream->plain_hashing.count(), stream->marks); + writeIntBinary(stream->compressed.offset(), stream->marks); + } + } + + stream->finalize(); + stream->addToChecksums(checksums); + stream.reset(); +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 6fb641d69ce..514dc8ddff4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -6,16 +6,19 @@ namespace DB class MergeTreeDataPartWriterCompact : IMergeTreeDataPartWriter { public: - size_t write(const Block & block, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, + size_t write(const Block & block, const IColumn::Permutation * permutation, + size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) override; + void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) override; + +private: /// Write single granule of one column (rows between 2 marks) size_t writeColumnSingleGranule( const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows); - -private: + ColumnStreamPtr stream; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 69ec99fcf74..a2fa713f0f5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -12,13 +12,13 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, - const IColumn::Permutation * permutation_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const WriterSettings & settings_, const ColumnToSize & merged_column_to_size) - : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, - permutation_, marks_file_extension_, + : IMergeTreeDataPartWriter(part_path_, + storage_, columns_list_, + marks_file_extension_, default_codec_, settings_) { size_t total_size = 0; @@ -89,9 +89,10 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( }; } -size_t MergeTreeDataPartWriterWide::write(const Block & block, size_t from_mark, size_t index_offset, - const MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block) +size_t MergeTreeDataPartWriterWide::write(const Block & block, + const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, + const MergeTreeIndexGranularity & index_granularity, + const Block & primary_key_block, const Block & skip_indexes_block) { if (serialization_states.empty()) { @@ -286,4 +287,56 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( return std::make_pair(current_column_mark, current_row - total_rows); } +void MergeTreeDataPartWriterWide::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) +{ + const auto & settings = storage.global_context.getSettingsRef(); + IDataType::SerializeBinaryBulkSettings serialize_settings; + serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; + serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; + WrittenOffsetColumns offset_columns; + + { + auto it = columns_list.begin(); + for (size_t i = 0; i < columns_list.size(); ++i, ++it) + { + if (!serialization_states.empty()) + { + serialize_settings.getter = createStreamGetter(it->name, offset_columns, false); + it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); + } + + if (write_final_mark) + writeFinalMark(it->name, it->type, offset_columns, false, serialize_settings.path); + } + } + + for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) + { + it->second->finalize(); + it->second->addToChecksums(checksums); + } + + column_streams.clear(); +} + +void MergeTreeDataPartWriterWide::writeFinalMark( + const std::string & column_name, + const DataTypePtr column_type, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + DB::IDataType::SubstreamPath & path) +{ + writeSingleMark(column_name, *column_type, offset_columns, skip_offsets, 0, path); + /// Memoize information about offsets + column_type->enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) + { + bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; + if (is_offsets) + { + String stream_name = IDataType::getFileNameForStream(column_name, substream_path); + offset_columns.insert(stream_name); + } + }, path); +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 5b9d89e8d4e..fd1887741d4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -3,7 +3,7 @@ namespace DB { -class MergeTreeDataPartWriterWide : IMergeTreeDataPartWriter +class MergeTreeDataPartWriterWide : public IMergeTreeDataPartWriter { public: @@ -13,16 +13,18 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, - const IColumn::Permutation * permutation, const String & marks_file_extension, const CompressionCodecPtr & default_codec, const WriterSettings & settings, const ColumnToSize & merged_column_to_size = {}); - size_t write(const Block & block, size_t from_mark, size_t index_offset, + size_t write(const Block & block, const IColumn::Permutation * permutation, + size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) override; + void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) override; + IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); /// Write data of one column. @@ -61,6 +63,13 @@ private: bool skip_offsets, size_t number_of_rows, DB::IDataType::SubstreamPath & path); + + void writeFinalMark( + const std::string & column_name, + const DataTypePtr column_type, + WrittenOffsetColumns & offset_columns, + bool skip_offsets, + DB::IDataType::SubstreamPath & path); void addStreams( const String & name, diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 670e7f2b644..bb1ddc2400f 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -37,6 +37,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( , columns_list(columns_list_) { init(); + writer = data_part_->getWriter(columns_list_, default_codec_, writer_settings); } MergedBlockOutputStream::MergedBlockOutputStream( @@ -58,6 +59,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( , columns_list(columns_list_) { init(); + writer = data_part_->getWriter(columns_list_, default_codec_, writer_settings); } std::string MergedBlockOutputStream::getPartPath() const @@ -89,37 +91,19 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( const NamesAndTypesList * total_column_list, MergeTreeData::DataPart::Checksums * additional_column_checksums) { + /// Finish write and get checksums. + MergeTreeData::DataPart::Checksums checksums; + /// Finish columns serialization. - { - /// FIXME - // const auto & settings = storage.global_context.getSettingsRef(); - // IDataType::SerializeBinaryBulkSettings serialize_settings; - // serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; - // serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; - // WrittenOffsetColumns offset_columns; - // auto it = columns_list.begin(); - // for (size_t i = 0; i < columns_list.size(); ++i, ++it) - // { - // if (!serialization_states.empty()) - // { - // serialize_settings.getter = createStreamGetter(it->name, offset_columns, false); - // it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); - // } + bool write_final_mark = (with_final_mark && rows_count != 0); + writer->finalize(checksums, write_final_mark); - // if (with_final_mark && rows_count != 0) - // writeFinalMark(it->name, it->type, offset_columns, false, serialize_settings.path); - // } - } - - if (with_final_mark && rows_count != 0) + if (write_final_mark) index_granularity.appendMark(0); /// last mark if (!total_column_list) total_column_list = &columns_list; - /// Finish write and get checksums. - MergeTreeData::DataPart::Checksums checksums; - if (additional_column_checksums) checksums = std::move(*additional_column_checksums); @@ -142,17 +126,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( index_stream = nullptr; } - /// FIXME - // for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) - // { - // it->second->finalize(); - // it->second->addToChecksums(checksums); - // } - finishSkipIndicesSerialization(checksums); - // column_streams.clear(); - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { new_part->partition.store(storage, part_path, checksums); @@ -278,7 +253,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } - size_t new_index_offset = writer->write(block, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block); + size_t new_index_offset = writer->write(block, permutation, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block); rows_count += rows; /// Should be written before index offset update, because we calculate, @@ -299,9 +274,11 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm { if (storage.hasPrimaryKey()) { - for (size_t j = 0, size = primary_key_block.rows(); j < size; ++j) + for (size_t j = 0, size = primary_key_block.columns(); j < size; ++j) { const auto & primary_column = primary_key_block.getByPosition(j); + std::cerr << "(writeImpl) primary_column: " << !!primary_column.column << "\n"; + std::cerr << "(writeImpl) index_column: " << !!index_columns[j] << "\n"; index_columns[j]->insertFrom(*primary_column.column, i); primary_column.type->serializeBinary(*primary_column.column, i, *index_stream); } @@ -316,7 +293,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } /// store last index row to write final mark at the end of column - for (size_t j = 0, size = primary_key_block.rows(); j < size; ++j) + for (size_t j = 0, size = primary_key_block.columns(); j < size; ++j) { const IColumn & primary_column = *primary_key_block.getByPosition(j).column.get(); auto mutable_column = std::move(*last_index_row[j].column).mutate(); From 8b880d65baabc00bfb5e8c6ddc59833c93c55353 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 22 Oct 2019 13:50:17 +0300 Subject: [PATCH 0045/2007] polymorphic parts (development) --- dbms/src/Interpreters/SystemLog.cpp | 1 + .../MergeTree/IMergedBlockOutputStream.cpp | 20 ----------- .../MergeTree/MergeTreeDataPartCompact.cpp | 14 ++++---- .../MergeTree/MergeTreeDataPartFactory.cpp | 13 ++++--- .../MergeTree/MergeTreeDataPartFactory.h | 1 + .../MergeTreeDataPartWriterCompact.cpp | 35 +++++++++++++++++++ .../MergeTreeDataPartWriterCompact.h | 10 +++++- .../MergedColumnOnlyOutputStream.cpp | 7 ++-- 8 files changed, 67 insertions(+), 34 deletions(-) diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index cb87990e4c0..2e279153b73 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -26,6 +26,7 @@ std::shared_ptr createSystemLog( const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { + std::cerr << "config has prefix '" << config_prefix << "' :" << config.has(config_prefix) << "\n"; if (!config.has(config_prefix)) return {}; diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 1ac0bdc2a1f..e4796a202ba 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -92,26 +92,6 @@ void IMergedBlockOutputStream::fillIndexGranularity(const Block & block) can_use_adaptive_granularity); } -void IMergedBlockOutputStream::writeFinalMark( - const std::string & column_name, - const DataTypePtr column_type, - WrittenOffsetColumns & offset_columns, - bool /* skip_offsets */, - DB::IDataType::SubstreamPath & path) -{ - // writer->writeSingleMark(column_name, *column_type, offset_columns, skip_offsets, 0, path); - /// Memoize information about offsets - column_type->enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) - { - bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets) - { - String stream_name = IDataType::getFileNameForStream(column_name, substream_path); - offset_columns.insert(stream_name); - } - }, path); -} - void IMergedBlockOutputStream::initSkipIndices() { for (const auto & index : skip_indices) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index dba86814959..9bf6e98fdbf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -23,6 +23,7 @@ #include #include +#include #include @@ -90,13 +91,12 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( const CompressionCodecPtr & default_codec, const WriterSettings & writer_settings) const { - UNUSED(columns_list); - UNUSED(default_codec); - UNUSED(writer_settings); - return {}; + return std::make_unique( + getFullPath(), storage, columns_list, + index_granularity_info.marks_file_extension, + default_codec, writer_settings); } - /// Takes into account the fact that several columns can e.g. share their .size substreams. /// When calculating totals these should be counted only once. ColumnSize MergeTreeDataPartCompact::getColumnSizeImpl( @@ -145,7 +145,7 @@ String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const if (!hasColumnFiles(column.name, *column.type)) continue; - const auto size = getColumnSize(column.name, *column.type).data_compressed; + const auto size = getColumnSizeImpl(column.name, *column.type, nullptr).data_compressed; if (size < minimum_size) { minimum_size = size; @@ -469,7 +469,7 @@ void MergeTreeDataPartCompact::loadRowsCount() if (!column_col->isFixedAndContiguous() || column_col->lowCardinality()) continue; - size_t column_size = getColumnSize(column.name, *column.type).data_uncompressed; + size_t column_size = getColumnSizeImpl(column.name, *column.type, nullptr).data_uncompressed; if (!column_size) continue; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp index d2066385dba..aa5b379b82c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp @@ -1,4 +1,5 @@ #include "MergeTreeDataPartFactory.h" +#include #include namespace DB @@ -6,14 +7,18 @@ namespace DB std::shared_ptr createPart(const MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const MergeTreePartInfo & info, const String & relative_path) { - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", 3 * sizeof(size_t)); - return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); + /// FIXME + size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); } std::shared_ptr createPart(MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const String & relative_path) { - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", 3 * sizeof(size_t)); - return std::make_shared(storage, name, index_granularity_info, disk, relative_path); + /// FIXME + size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + return std::make_shared(storage, name, index_granularity_info, disk, relative_path); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h index 38091e5d38b..98d808a252a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h @@ -1,5 +1,6 @@ #include #include +#include namespace DB { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 3fbd61b8dc5..a2138ea5cca 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -3,6 +3,35 @@ namespace DB { +namespace +{ + constexpr auto DATA_FILE_NAME = "data"; + constexpr auto DATA_FILE_EXTENSION = ".bin"; +} + + +MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( + const String & part_path_, + const MergeTreeData & storage_, + const NamesAndTypesList & columns_list_, + const String & marks_file_extension_, + const CompressionCodecPtr & default_codec_, + const WriterSettings & settings_) +: IMergeTreeDataPartWriter(part_path_, + storage_, columns_list_, + marks_file_extension_, + default_codec_, settings_) +{ + stream = std::make_unique( + DATA_FILE_NAME, + part_path + DATA_FILE_NAME, DATA_FILE_EXTENSION, + part_path + DATA_FILE_NAME, marks_file_extension, + default_codec, + settings.max_compress_block_size, + 0, + settings.aio_threshold); +} + size_t MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, @@ -33,8 +62,12 @@ size_t MergeTreeDataPartWriterCompact::write( columns_to_write[i] = block.getByName(it->name); } + std::cerr << "(MergeTreeDataPartWriterCompact::write) total_rows: " << total_rows << "\n"; + while (current_row < total_rows) { + std::cerr << "(MergeTreeDataPartWriterCompact::write) current_row: " << current_row << "\n"; + bool write_marks = true; size_t rows_to_write; if (current_row == 0 && index_offset != 0) @@ -47,6 +80,8 @@ size_t MergeTreeDataPartWriterCompact::write( rows_to_write = index_granularity.getMarkRows(current_mark); } + std::cerr << "(MergeTreeDataPartWriterCompact::write) rows_to_write: " << rows_to_write << "\n"; + if (write_marks) { writeIntBinary(rows_to_write, stream->marks); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 514dc8ddff4..14b309cb2a1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -3,9 +3,17 @@ namespace DB { -class MergeTreeDataPartWriterCompact : IMergeTreeDataPartWriter +class MergeTreeDataPartWriterCompact : public IMergeTreeDataPartWriter { public: + MergeTreeDataPartWriterCompact( + const String & part_path, + const MergeTreeData & storage, + const NamesAndTypesList & columns_list, + const String & marks_file_extension, + const CompressionCodecPtr & default_codec, + const WriterSettings & settings); + size_t write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) override; diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index ea0753ee0a1..38d8241aed5 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -102,9 +102,12 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG // serialize_settings.getter = createStreamGetter(column.name, already_written_offset_columns, skip_offsets); column.type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); + UNUSED(skip_offsets); + + /// FIXME /// We wrote at least one row - if (with_final_mark && (index_offset != 0 || current_mark != 0)) - writeFinalMark(column.name, column.type, offset_columns, skip_offsets, serialize_settings.path); + // if (with_final_mark && (index_offset != 0 || current_mark != 0)) + // writeFinalMark(column.name, column.type, offset_columns, skip_offsets, serialize_settings.path); } MergeTreeData::DataPart::Checksums checksums; From 715ae5a21696bb097c458da1ff09395d2b7f7d35 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 22 Oct 2019 20:42:59 +0300 Subject: [PATCH 0046/2007] polymorphic parts (development) --- .../MergeTree/IMergeTreeDataPartWriter.h | 5 +- .../MergeTree/MergeTreeDataPartFactory.cpp | 21 +++++--- .../MergeTreeDataPartWriterCompact.cpp | 36 ++++++++----- .../MergeTreeDataPartWriterCompact.h | 4 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 19 ++++--- .../MergeTree/MergeTreeDataPartWriterWide.h | 6 +-- .../MergeTree/MergedBlockOutputStream.cpp | 2 +- .../MergedColumnOnlyOutputStream.cpp | 51 ++++--------------- .../MergeTree/MergedColumnOnlyOutputStream.h | 2 + 9 files changed, 66 insertions(+), 80 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index f39399a2818..b05faf1d315 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -19,6 +19,7 @@ class IMergeTreeDataPartWriter { public: using WrittenOffsetColumns = std::set; + using MarkWithOffset = std::pair; struct ColumnStream { @@ -65,13 +66,13 @@ public: const CompressionCodecPtr & default_codec, const WriterSettings & settings); - virtual size_t write( + virtual MarkWithOffset write( const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t offset, const MergeTreeIndexGranularity & index_granularity, /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; - virtual void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) = 0; + virtual void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) = 0; virtual ~IMergeTreeDataPartWriter(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp index aa5b379b82c..8f16b088ec0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp @@ -7,18 +7,23 @@ namespace DB std::shared_ptr createPart(const MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const MergeTreePartInfo & info, const String & relative_path) { - /// FIXME - size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); + // /// FIXME + // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + // return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); + + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); + return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); } std::shared_ptr createPart(MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const String & relative_path) { - /// FIXME - size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - return std::make_shared(storage, name, index_granularity_info, disk, relative_path); + // /// FIXME + // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + // return std::make_shared(storage, name, index_granularity_info, disk, relative_path); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); + return std::make_shared(storage, name, index_granularity_info, disk, relative_path); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index a2138ea5cca..d6d6612ad5b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -32,7 +32,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( settings.aio_threshold); } -size_t MergeTreeDataPartWriterCompact::write( +IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, @@ -64,24 +64,31 @@ size_t MergeTreeDataPartWriterCompact::write( std::cerr << "(MergeTreeDataPartWriterCompact::write) total_rows: " << total_rows << "\n"; + UNUSED(index_offset); + while (current_row < total_rows) { std::cerr << "(MergeTreeDataPartWriterCompact::write) current_row: " << current_row << "\n"; + bool write_marks = true; - size_t rows_to_write; - if (current_row == 0 && index_offset != 0) - { - rows_to_write = index_offset; - write_marks = false; - } - else - { - rows_to_write = index_granularity.getMarkRows(current_mark); - } + size_t rows_to_write = std::min(total_rows, index_granularity.getMarkRows(current_mark)); + // if (current_row == 0 && index_offset != 0) + // { + // rows_to_write = index_offset; + // write_marks = false; + // } + // else + // { + // rows_to_write = index_granularity.getMarkRows(current_mark); + // } std::cerr << "(MergeTreeDataPartWriterCompact::write) rows_to_write: " << rows_to_write << "\n"; + /// There could already be enough data to compress into the new block. + if (stream->compressed.offset() >= settings.min_compress_block_size) + stream->compressed.next(); + if (write_marks) { writeIntBinary(rows_to_write, stream->marks); @@ -100,8 +107,7 @@ size_t MergeTreeDataPartWriterCompact::write( } } - /// We always write end granule for block in Compact parts. - return 0; + return {current_mark, total_rows - current_row}; } size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) @@ -120,7 +126,7 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith return from_row + number_of_rows; } -void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) +void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) { if (write_final_mark) { @@ -133,6 +139,8 @@ void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & ch } stream->finalize(); + if (sync) + stream->sync(); stream->addToChecksums(checksums); stream.reset(); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 14b309cb2a1..659feb79ba4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -14,11 +14,11 @@ public: const CompressionCodecPtr & default_codec, const WriterSettings & settings); - size_t write(const Block & block, const IColumn::Permutation * permutation, + MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) override; - void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) override; + void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; private: /// Write single granule of one column (rows between 2 marks) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index a2fa713f0f5..8a7210e9b5f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -89,7 +89,7 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( }; } -size_t MergeTreeDataPartWriterWide::write(const Block & block, +IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) @@ -109,7 +109,7 @@ size_t MergeTreeDataPartWriterWide::write(const Block & block, } WrittenOffsetColumns offset_columns; - size_t new_index_offset = 0; + MarkWithOffset result; auto it = columns_list.begin(); for (size_t i = 0; i < columns_list.size(); ++i, ++it) @@ -121,27 +121,27 @@ size_t MergeTreeDataPartWriterWide::write(const Block & block, if (primary_key_block.has(it->name)) { const auto & primary_column = *primary_key_block.getByName(it->name).column; - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, primary_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, primary_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } else if (skip_indexes_block.has(it->name)) { const auto & index_column = *skip_indexes_block.getByName(it->name).column; - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, index_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, index_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } else { /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. ColumnPtr permuted_column = column.column->permute(*permutation, 0); - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *permuted_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, *permuted_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } } else { - std::tie(std::ignore, new_index_offset) = writeColumn(column.name, *column.type, *column.column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, *column.column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); } } - return new_index_offset; + return result; } void MergeTreeDataPartWriterWide::writeSingleMark( @@ -287,7 +287,7 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( return std::make_pair(current_column_mark, current_row - total_rows); } -void MergeTreeDataPartWriterWide::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) +void MergeTreeDataPartWriterWide::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) { const auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -313,10 +313,13 @@ void MergeTreeDataPartWriterWide::finalize(IMergeTreeDataPart::Checksums & check for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) { it->second->finalize(); + if (sync) + it->second->sync(); it->second->addToChecksums(checksums); } column_streams.clear(); + serialization_states.clear(); } void MergeTreeDataPartWriterWide::writeFinalMark( diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index fd1887741d4..cebf37c5118 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -18,19 +18,19 @@ public: const WriterSettings & settings, const ColumnToSize & merged_column_to_size = {}); - size_t write(const Block & block, const IColumn::Permutation * permutation, + MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block) override; - void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark) override; + void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); /// Write data of one column. /// Return how many marks were written and /// how many rows were written for last mark - std::pair writeColumn( + MarkWithOffset writeColumn( const String & name, const IDataType & type, const IColumn & column, diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index bb1ddc2400f..90cd9381862 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -253,7 +253,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } - size_t new_index_offset = writer->write(block, permutation, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block); + size_t new_index_offset = writer->write(block, permutation, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block).second; rows_count += rows; /// Should be written before index offset update, because we calculate, diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 38d8241aed5..b5ab31ab412 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -22,23 +22,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( header(header_), sync(sync_), skip_offsets(skip_offsets_), already_written_offset_columns(already_written_offset_columns_) { - serialization_states.reserve(header.columns()); - WrittenOffsetColumns tmp_offset_columns; - IDataType::SerializeBinaryBulkSettings settings; - - for (const auto & column_name : header.getNames()) - { - const auto & col = header.getByName(column_name); - - const auto columns = storage.getColumns(); - /// FIXME - // addStreams(part_path, col.name, *col.type, columns.getCodecOrDefault(col.name, codec), 0, skip_offsets); - serialization_states.emplace_back(nullptr); - /// FIXME - // settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); - col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); - } - + writer = data_part_->getWriter(header.getNamesAndTypesList(), default_codec_, writer_settings); initSkipIndices(); } @@ -63,15 +47,11 @@ void MergedColumnOnlyOutputStream::write(const Block & block) if (!rows) return; - size_t new_index_offset = 0; - size_t new_current_mark = 0; - WrittenOffsetColumns offset_columns = already_written_offset_columns; - for (size_t i = 0; i < header.columns(); ++i) - { - /// FIXME - // const ColumnWithTypeAndName & column = block.getByName(header.getByPosition(i).name); - // std::tie(new_current_mark, new_index_offset) = writeColumn(column.name, *column.type, *column.column, offset_columns, skip_offsets, serialization_states[i], current_mark); - } + /// FIXME skip_offsets + UNUSED(skip_offsets); + UNUSED(already_written_offset_columns); + + auto [new_index_offset, new_current_mark] = writer->write(block, nullptr, current_mark, index_offset, index_granularity); /// Should be written before index offset update, because we calculate, /// indices of currently written granules @@ -112,23 +92,10 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG MergeTreeData::DataPart::Checksums checksums; - /// FIXME + bool write_final_mark = with_final_mark && (index_offset != 0 || current_mark != 0); + writer->finalize(checksums, write_final_mark, sync); - UNUSED(sync); - - // for (auto & column_stream : column_streams) - // { - // column_stream.second->finalize(); - // if (sync) - // column_stream.second->sync(); - - // column_stream.second->addToChecksums(checksums); - // } - - // finishSkipIndicesSerialization(checksums); - - // column_streams.clear(); - // serialization_states.clear(); + finishSkipIndicesSerialization(checksums); return checksums; } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index d0cc316b319..6030977fc29 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -6,6 +6,8 @@ namespace DB { +class MergeTreeDataPartWriterWide; + /// Writes only those columns that are in `header` class MergedColumnOnlyOutputStream final : public IMergedBlockOutputStream { From 32858c4f445a7327bdec27370c5d78c566799b19 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 23 Oct 2019 17:32:54 +0300 Subject: [PATCH 0047/2007] polymorphic parts (development) --- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 1 + .../MergedColumnOnlyOutputStream.cpp | 22 ------------------- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 8a7210e9b5f..494db34e5d0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -20,6 +20,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( storage_, columns_list_, marks_file_extension_, default_codec_, settings_) + , can_use_adaptive_granularity(storage_.canUseAdaptiveGranularity()) { size_t total_size = 0; if (settings.aio_threshold > 0 && !merged_column_to_size.empty()) diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index b5ab31ab412..461e4567c09 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -69,29 +69,7 @@ void MergedColumnOnlyOutputStream::writeSuffix() MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums() { /// Finish columns serialization. - auto & settings = storage.global_context.getSettingsRef(); - IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; - serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; - - WrittenOffsetColumns offset_columns; - for (size_t i = 0, size = header.columns(); i < size; ++i) - { - auto & column = header.getByPosition(i); - /// FIXME - // serialize_settings.getter = createStreamGetter(column.name, already_written_offset_columns, skip_offsets); - column.type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); - - UNUSED(skip_offsets); - - /// FIXME - /// We wrote at least one row - // if (with_final_mark && (index_offset != 0 || current_mark != 0)) - // writeFinalMark(column.name, column.type, offset_columns, skip_offsets, serialize_settings.path); - } - MergeTreeData::DataPart::Checksums checksums; - bool write_final_mark = with_final_mark && (index_offset != 0 || current_mark != 0); writer->finalize(checksums, write_final_mark, sync); From 5484f4b771e2e76535d61a9f862f18e05fae44fd Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 28 Oct 2019 14:00:29 +0300 Subject: [PATCH 0048/2007] polymorphic parts (development) --- dbms/src/Storages/MergeTree/IMergeTreeDataPart.h | 2 ++ .../Storages/MergeTree/IMergeTreeDataPartWriter.h | 3 ++- .../MergeTree/IMergedBlockOutputStream.cpp | 1 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 1 + .../Storages/MergeTree/MergeTreeDataPartWide.cpp | 5 +++++ .../MergeTree/MergeTreeDataPartWriterCompact.cpp | 6 +++++- .../MergeTree/MergeTreeDataPartWriterCompact.h | 3 ++- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 13 +++++++------ .../MergeTree/MergeTreeDataPartWriterWide.h | 3 ++- .../Storages/MergeTree/MergedBlockOutputStream.cpp | 14 ++++++-------- .../MergeTree/MergedColumnOnlyOutputStream.cpp | 6 ++++++ 11 files changed, 39 insertions(+), 18 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index d8815986d49..cd64c0d26a0 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -160,6 +160,8 @@ public: ColumnSize getTotalColumnsSize() const; + String getMarksFileExtension() const { return index_granularity_info.marks_file_extension; } + /// Generate the new name for this part according to `new_part_info` and min/max dates from the old name. /// This is useful when you want to change e.g. block numbers or the mutation version of the part. String getNewName(const MergeTreePartInfo & new_part_info) const; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index b05faf1d315..acab159e4ba 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -70,7 +70,8 @@ public: const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t offset, const MergeTreeIndexGranularity & index_granularity, /* Blocks with already sorted index columns */ - const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; + const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, + bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) = 0; virtual void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) = 0; diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index e4796a202ba..59b90639ba0 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -29,6 +29,7 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( , part_path(data_part->getFullPath()) , writer_settings(writer_settings_) , can_use_adaptive_granularity(can_use_adaptive_granularity_) + , marks_file_extension(data_part->getMarksFileExtension()) , blocks_are_granules_size(blocks_are_granules_size_) , index_granularity(data_part->index_granularity) , compute_granularity(index_granularity.empty()) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 14486300df4..040d77a100f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -831,6 +831,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor rows_sources_read_buf.seek(0, 0); ColumnGathererStream column_gathered_stream(column_name, column_part_streams, rows_sources_read_buf); + new_data_part->index_granularity = to.getIndexGranularity(); MergedColumnOnlyOutputStream column_to( new_data_part, column_gathered_stream.getHeader(), diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 8d55ba399b9..bf626b9d68a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -271,8 +271,13 @@ void MergeTreeDataPartWide::remove() const #endif std::shared_lock lock(columns_lock); + std::cerr << "removing: " << name << "\n"; + std::cerr << "checksums size: " << checksums.files.size() << "\n"; + for (const auto & [file, _] : checksums.files) { + std::cerr << "checksums file: " << file << "\n"; + String path_to_remove = to + "/" + file; if (0 != unlink(path_to_remove.c_str())) throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index d6d6612ad5b..ad1ec6993b2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -36,8 +36,12 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block) + const Block & primary_key_block, const Block & skip_indexes_block, + bool skip_offsets, const WrittenOffsetColumns & already_written_offset_columns) { + UNUSED(skip_offsets); + UNUSED(already_written_offset_columns); + size_t total_rows = block.rows(); size_t current_mark = from_mark; size_t current_row = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 659feb79ba4..e92b03a5d78 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -16,7 +16,8 @@ public: MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block) override; + const Block & primary_key_block, const Block & skip_indexes_block, + bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) override; void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 494db34e5d0..c82b2de93cf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -93,7 +93,8 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block) + const Block & primary_key_block, const Block & skip_indexes_block, + bool skip_offsets, const WrittenOffsetColumns & already_written_offset_columns) { if (serialization_states.empty()) { @@ -109,7 +110,7 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(cons } } - WrittenOffsetColumns offset_columns; + WrittenOffsetColumns offset_columns = already_written_offset_columns; MarkWithOffset result; auto it = columns_list.begin(); @@ -122,23 +123,23 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(cons if (primary_key_block.has(it->name)) { const auto & primary_column = *primary_key_block.getByName(it->name).column; - result = writeColumn(column.name, *column.type, primary_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, primary_column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); } else if (skip_indexes_block.has(it->name)) { const auto & index_column = *skip_indexes_block.getByName(it->name).column; - result = writeColumn(column.name, *column.type, index_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, index_column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); } else { /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. ColumnPtr permuted_column = column.column->permute(*permutation, 0); - result = writeColumn(column.name, *column.type, *permuted_column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, *permuted_column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); } } else { - result = writeColumn(column.name, *column.type, *column.column, index_granularity, offset_columns, false, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, *column.column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index cebf37c5118..18ee533f4be 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -21,7 +21,8 @@ public: MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block) override; + const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, + bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) override; void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 90cd9381862..c40c357c22c 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -44,7 +44,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, - const MergeTreeData::DataPart::ColumnToSize & /* merged_column_to_size_ */, + const MergeTreeData::DataPart::ColumnToSize & /* merged_column_to_size_ */, // FIXME size_t aio_threshold_, bool blocks_are_granules_size_) : IMergedBlockOutputStream( @@ -55,7 +55,8 @@ MergedBlockOutputStream::MergedBlockOutputStream( aio_threshold_ }, blocks_are_granules_size_, - std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), {}) + std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), + data_part_->storage.canUseAdaptiveGranularity()) , columns_list(columns_list_) { init(); @@ -94,6 +95,9 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( /// Finish write and get checksums. MergeTreeData::DataPart::Checksums checksums; + if (additional_column_checksums) + checksums = std::move(*additional_column_checksums); + /// Finish columns serialization. bool write_final_mark = (with_final_mark && rows_count != 0); writer->finalize(checksums, write_final_mark); @@ -104,9 +108,6 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (!total_column_list) total_column_list = &columns_list; - if (additional_column_checksums) - checksums = std::move(*additional_column_checksums); - if (index_stream) { if (with_final_mark && rows_count != 0) @@ -193,7 +194,6 @@ void MergedBlockOutputStream::init() void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation) { - std::cerr << "(MergedBlockOutputStream::writeImpl) block.rows(): " << block.rows() << "\n"; block.checkNumberOfRows(); size_t rows = block.rows(); if (!rows) @@ -277,8 +277,6 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm for (size_t j = 0, size = primary_key_block.columns(); j < size; ++j) { const auto & primary_column = primary_key_block.getByPosition(j); - std::cerr << "(writeImpl) primary_column: " << !!primary_column.column << "\n"; - std::cerr << "(writeImpl) index_column: " << !!index_columns[j] << "\n"; index_columns[j]->insertFrom(*primary_column.column, i); primary_column.type->serializeBinary(*primary_column.column, i, *index_stream); } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 461e4567c09..ac4b820d9fb 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -22,6 +22,12 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( header(header_), sync(sync_), skip_offsets(skip_offsets_), already_written_offset_columns(already_written_offset_columns_) { + std::cerr << "(MergedColumnOnlyOutputStream) storage: " << storage.getTableName() << "\n"; + std::cerr << "(MergedColumnOnlyOutputStream) can_use_adaptive_granularity: " << can_use_adaptive_granularity << "\n"; + std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info: " << !!index_granularity_info_ << "\n"; + if (index_granularity_info_) + std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; + writer = data_part_->getWriter(header.getNamesAndTypesList(), default_codec_, writer_settings); initSkipIndices(); } From 35b736380255820d4e4de7265509a687816ed809 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 31 Oct 2019 17:44:17 +0300 Subject: [PATCH 0049/2007] polymorphic parts (development) --- dbms/src/Interpreters/SystemLog.cpp | 1 - .../Storages/MergeTree/IMergeTreeDataPart.cpp | 419 +++++++++++++++- .../Storages/MergeTree/IMergeTreeDataPart.h | 51 +- .../MergeTree/IMergeTreeDataPartWriter.h | 2 +- .../MergeTree/IMergedBlockOutputStream.cpp | 7 +- .../MergeTreeBaseSelectBlockInputStream.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 3 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 1 + .../MergeTree/MergeTreeDataPartCompact.cpp | 389 +------------- .../MergeTree/MergeTreeDataPartCompact.h | 33 +- .../MergeTree/MergeTreeDataPartFactory.cpp | 22 +- .../MergeTree/MergeTreeDataPartOnDisk.cpp | 0 .../MergeTree/MergeTreeDataPartOnDisk.h | 33 -- .../MergeTree/MergeTreeDataPartWide.cpp | 474 +----------------- .../MergeTree/MergeTreeDataPartWide.h | 28 +- .../MergeTreeDataPartWriterCompact.cpp | 24 +- .../MergeTreeDataPartWriterCompact.h | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 8 +- .../MergeTree/MergeTreeDataPartWriterWide.h | 4 +- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../MergeTree/MergeTreeRangeReader.cpp | 6 + .../MergeTree/MergeTreeReaderCompact.cpp | 46 +- .../MergeTree/MergeTreeReaderWide.cpp | 4 +- .../MergedColumnOnlyOutputStream.cpp | 6 +- 24 files changed, 534 insertions(+), 1033 deletions(-) delete mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.cpp delete mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h diff --git a/dbms/src/Interpreters/SystemLog.cpp b/dbms/src/Interpreters/SystemLog.cpp index 2e279153b73..cb87990e4c0 100644 --- a/dbms/src/Interpreters/SystemLog.cpp +++ b/dbms/src/Interpreters/SystemLog.cpp @@ -26,7 +26,6 @@ std::shared_ptr createSystemLog( const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { - std::cerr << "config has prefix '" << config_prefix << "' :" << config.has(config_prefix) << "\n"; if (!config.has(config_prefix)) return {}; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 64860788586..ed9beac9521 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -201,6 +201,19 @@ String IMergeTreeDataPart::getNewName(const MergeTreePartInfo & new_part_info) c return new_part_info.getPartName(); } +size_t IMergeTreeDataPart::getColumnPosition(const String & column_name) const +{ + /// FIXME + size_t i = 0; + for (const auto & it : columns) + { + if (it.name == column_name) + return i; + ++i; + } + return -1; +} + DayNum IMergeTreeDataPart::getMinDate() const { if (storage.minmax_idx_date_column_pos != -1 && minmax_idx.initialized) @@ -237,33 +250,38 @@ time_t IMergeTreeDataPart::getMaxTime() const IMergeTreeDataPart::~IMergeTreeDataPart() { - // if (on_disk && (state == State::DeleteOnDestroy || is_temp)) - // { - // try - // { - // std::string path = on_disk->getFullPath(); + if (state == State::DeleteOnDestroy || is_temp) + { + try + { + std::string path = getFullPath(); - // Poco::File dir(path); - // if (!dir.exists()) - // return; + Poco::File dir(path); + if (!dir.exists()) + return; - // if (is_temp) - // { - // if (!startsWith(on_disk->getNameWithPrefix(), "tmp")) - // { - // LOG_ERROR(storage.log, "~DataPart() should remove part " << path - // << " but its name doesn't start with tmp. Too suspicious, keeping the part."); - // return; - // } - // } + if (is_temp) + { + String file_name = Poco::Path(relative_path).getFileName(); - // dir.remove(true); - // } - // catch (...) - // { - // tryLogCurrentException(__PRETTY_FUNCTION__); - // } - // } + if (file_name.empty()) + throw Exception("relative_path " + relative_path + " of part " + name + " is invalid or not set", ErrorCodes::LOGICAL_ERROR); + + if (!startsWith(file_name, "tmp")) + { + LOG_ERROR(storage.log, "~DataPart() should remove part " << path + << " but its name doesn't start with tmp. Too suspicious, keeping the part."); + return; + } + } + + dir.remove(true); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } } @@ -326,7 +344,7 @@ void IMergeTreeDataPart::assertState(const std::initializer_listcreateColumn(); + loaded_index[i]->reserve(index_granularity.getMarksCount()); + } + + String index_path = getFullPath() + "primary.idx"; + ReadBufferFromFile index_file = openForReading(index_path); + + for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 + for (size_t j = 0; j < key_size; ++j) + storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], index_file); + + for (size_t i = 0; i < key_size; ++i) + { + loaded_index[i]->protect(); + if (loaded_index[i]->size() != index_granularity.getMarksCount()) + throw Exception("Cannot read all data from index file " + index_path + + "(expected size: " + toString(index_granularity.getMarksCount()) + ", read: " + toString(loaded_index[i]->size()) + ")", + ErrorCodes::CANNOT_READ_ALL_DATA); + } + + if (!index_file.eof()) + throw Exception("Index file " + index_path + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); + + index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); + } +} + +void IMergeTreeDataPart::loadPartitionAndMinMaxIndex() +{ + if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + DayNum min_date; + DayNum max_date; + MergeTreePartInfo::parseMinMaxDatesFromPartName(name, min_date, max_date); + + const auto & date_lut = DateLUT::instance(); + partition = MergeTreePartition(date_lut.toNumYYYYMM(min_date)); + minmax_idx = MinMaxIndex(min_date, max_date); + } + else + { + String path = getFullPath(); + partition.load(storage, path); + if (!isEmpty()) + minmax_idx.load(storage, path); + } + + String calculated_partition_id = partition.getID(storage.partition_key_sample); + if (calculated_partition_id != info.partition_id) + throw Exception( + "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id + + " differs from partition ID in part name: " + info.partition_id, + ErrorCodes::CORRUPTED_DATA); +} + +void IMergeTreeDataPart::loadChecksums(bool require) +{ + String path = getFullPath() + "checksums.txt"; + Poco::File checksums_file(path); + if (checksums_file.exists()) + { + ReadBufferFromFile file = openForReading(path); + if (checksums.read(file)) + { + assertEOF(file); + bytes_on_disk = checksums.getTotalSizeOnDisk(); + } + else + bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); + } + else + { + if (require) + throw Exception("No checksums.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); + } +} + +void IMergeTreeDataPart::loadRowsCount() +{ + if (index_granularity.empty()) + { + rows_count = 0; + } + else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + String path = getFullPath() + "count.txt"; + if (!Poco::File(path).exists()) + throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + ReadBufferFromFile file = openForReading(path); + readIntText(rows_count, file); + assertEOF(file); + } + else + { + for (const NameAndTypePair & column : columns) + { + ColumnPtr column_col = column.type->createColumn(); + if (!column_col->isFixedAndContiguous() || column_col->lowCardinality()) + continue; + + size_t column_size = getColumnSizeImpl(column.name, *column.type, nullptr).data_uncompressed; + if (!column_size) + continue; + + size_t sizeof_field = column_col->sizeOfValueIfFixed(); + rows_count = column_size / sizeof_field; + + if (column_size % sizeof_field != 0) + { + throw Exception( + "Uncompressed size of column " + column.name + "(" + toString(column_size) + + ") is not divisible by the size of value (" + toString(sizeof_field) + ")", + ErrorCodes::LOGICAL_ERROR); + } + + size_t last_mark_index_granularity = index_granularity.getLastNonFinalMarkRows(); + size_t rows_approx = index_granularity.getTotalRows(); + if (!(rows_count <= rows_approx && rows_approx < rows_count + last_mark_index_granularity)) + throw Exception( + "Unexpected size of column " + column.name + ": " + toString(rows_count) + " rows, expected " + + toString(rows_approx) + "+-" + toString(last_mark_index_granularity) + " rows according to the index", + ErrorCodes::LOGICAL_ERROR); + + return; + } + + throw Exception("Data part doesn't contain fixed size column (even Date column)", ErrorCodes::LOGICAL_ERROR); + } +} + +void IMergeTreeDataPart::loadTTLInfos() +{ + String path = getFullPath() + "ttl.txt"; + if (Poco::File(path).exists()) + { + ReadBufferFromFile in = openForReading(path); + assertString("ttl format version: ", in); + size_t format_version; + readText(format_version, in); + assertChar('\n', in); + + if (format_version == 1) + { + try + { + ttl_infos.read(in); + } + catch (const JSONException &) + { + throw Exception("Error while parsing file ttl.txt in part: " + name, ErrorCodes::BAD_TTL_FILE); + } + } + else + throw Exception("Unknown ttl format version: " + toString(format_version), ErrorCodes::BAD_TTL_FILE); + } +} + +void IMergeTreeDataPart::loadColumns(bool require) +{ + String path = getFullPath() + "columns.txt"; + Poco::File poco_file_path{path}; + if (!poco_file_path.exists()) + { + if (require) + throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + /// If there is no file with a list of columns, write it down. + for (const NameAndTypePair & column : storage.getColumns().getAllPhysical()) + if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) + columns.push_back(column); + + if (columns.empty()) + throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + { + WriteBufferFromFile out(path + ".tmp", 4096); + columns.writeText(out); + } + Poco::File(path + ".tmp").renameTo(path); + + return; + } + + is_frozen = !poco_file_path.canWrite(); + + ReadBufferFromFile file = openForReading(path); + columns.readText(file); +} + UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) { @@ -370,6 +622,8 @@ UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_new_dir_if_exists) const { + assertOnDisk(); + String from = getFullPath(); String to = storage.getFullPathOnDisk(disk) + new_relative_path + "/"; @@ -401,6 +655,120 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ relative_path = new_relative_path; } +void IMergeTreeDataPart::remove() const +{ + if (!isStoredOnDisk()) + return; + + if (relative_path.empty()) + throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); + + /** Atomic directory removal: + * - rename directory to temporary name; + * - remove it recursive. + * + * For temporary name we use "delete_tmp_" prefix. + * + * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, + * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. + * But when we removing data part, it can be old enough. And rename doesn't change mtime. + * And a race condition can happen that will lead to "File not found" error here. + */ + + String full_path = storage.getFullPathOnDisk(disk); + String from = full_path + relative_path; + String to = full_path + "delete_tmp_" + name; + // TODO directory delete_tmp_ is never removed if server crashes before returning from this function + + Poco::File from_dir{from}; + Poco::File to_dir{to}; + + if (to_dir.exists()) + { + LOG_WARNING(storage.log, "Directory " << to << " (to which part must be renamed before removing) already exists." + " Most likely this is due to unclean restart. Removing it."); + + try + { + to_dir.remove(true); + } + catch (...) + { + LOG_ERROR(storage.log, "Cannot remove directory " << to << ". Check owner and access rights."); + throw; + } + } + + try + { + from_dir.renameTo(to); + } + catch (const Poco::FileNotFoundException &) + { + LOG_ERROR(storage.log, "Directory " << from << " (part to remove) doesn't exist or one of nested files has gone." + " Most likely this is due to manual removing. This should be discouraged. Ignoring."); + + return; + } + + try + { + /// Remove each expected file in directory, then remove directory itself. + +#if !__clang__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#endif + std::shared_lock lock(columns_lock); + + for (const auto & [file, _] : checksums.files) + { + String path_to_remove = to + "/" + file; + if (0 != unlink(path_to_remove.c_str())) + throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, + ErrorCodes::CANNOT_UNLINK); + } +#if !__clang__ +#pragma GCC diagnostic pop +#endif + + for (const auto & file : {"checksums.txt", "columns.txt"}) + { + String path_to_remove = to + "/" + file; + if (0 != unlink(path_to_remove.c_str())) + throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, + ErrorCodes::CANNOT_UNLINK); + } + + if (0 != rmdir(to.c_str())) + throwFromErrnoWithPath("Cannot rmdir file " + to, to, ErrorCodes::CANNOT_UNLINK); + } + catch (...) + { + /// Recursive directory removal does many excessive "stat" syscalls under the hood. + + LOG_ERROR(storage.log, "Cannot quickly remove directory " << to << " by removing files; fallback to recursive removal. Reason: " + << getCurrentExceptionMessage(false)); + + to_dir.remove(true); + } +} + + +String IMergeTreeDataPart::typeToString(Type type) +{ + switch(type) + { + case Type::WIDE: + return "Wide"; + case Type::COMPACT: + return "Striped"; + case Type::IN_MEMORY: + return "InMemory"; + } + + __builtin_unreachable(); +} String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) const { @@ -449,7 +817,6 @@ void IMergeTreeDataPart::makeCloneInDetached(const String & prefix) const void IMergeTreeDataPart::makeCloneOnDiskDetached(const DiskSpace::ReservationPtr & reservation) const { assertOnDisk(); - auto & reserved_disk = reservation->getDisk(); if (reserved_disk->getName() == disk->getName()) throw Exception("Can not clone data part " + name + " to same disk " + disk->getName(), ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index cd64c0d26a0..6815e82890b 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -52,11 +52,6 @@ public: using MergeTreeReaderPtr = std::unique_ptr; using MergeTreeWriterPtr = std::unique_ptr; - // virtual BlockInputStreamPtr readAll() = 0; - // virtual BlockInputStreamPtr read() = 0; - // virtual BlockInputStreamPtr readWithThreadPool() = 0; - // virtual BlockInputStreamPtr readReverse() = 0; - virtual MergeTreeReaderPtr getReader( const NamesAndTypesList & columns_, const MarkRanges & mark_ranges, @@ -73,7 +68,7 @@ public: virtual bool isStoredOnDisk() const = 0; - virtual void remove() const = 0; + void remove() const; virtual bool supportsVerticalMerge() const { return false; } @@ -84,14 +79,12 @@ public: /// Initialize columns (from columns.txt if exists, or create from column files if not). /// Load checksums from checksums.txt if exists. Load index if required. - virtual void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) = 0; + void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency); /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. virtual String getColumnNameWithMinumumCompressedSize() const = 0; - // virtual void detach() = 0; - // virtual Checksums check( // bool require_checksums, // const DataTypes & primary_key_data_types, /// Check the primary key. If it is not necessary, pass an empty array. @@ -103,7 +96,6 @@ public: using ColumnToSize = std::map; - // void accumulateColumnSizes(ColumnToSize & column_to_size) const // { // throw Exception("Method 'accumulateColumnSizes' is not supported for data part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMETED); @@ -120,20 +112,7 @@ public: // virtual void renameTo() = 0; - static String typeToString(Type type) - { - switch(type) - { - case Type::WIDE: - return "Wide"; - case Type::COMPACT: - return "Striped"; - case Type::IN_MEMORY: - return "InMemory"; - } - - __builtin_unreachable(); - } + static String typeToString(Type type); String getTypeName() { return typeToString(getType()); } @@ -166,6 +145,9 @@ public: /// This is useful when you want to change e.g. block numbers or the mutation version of the part. String getNewName(const MergeTreePartInfo & new_part_info) const; + // Block sample_block; + size_t getColumnPosition(const String & name) const; + bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } /// If the partition key includes date column (a common case), these functions will return min and max values for this column. @@ -344,6 +326,27 @@ public: static UInt64 calculateTotalSizeOnDisk(const String & from); private: + /// Reads columns names and types from columns.txt + void loadColumns(bool require); + + /// If checksums.txt exists, reads files' checksums (and sizes) from it + void loadChecksums(bool require); + + /// Loads marks index granularity into memory + virtual void loadIndexGranularity(); + + /// Loads index file. + void loadIndex(); + + /// Load rows count for this part from disk (for the newer storage format version). + /// For the older format version calculates rows count from the size of a column with a fixed size. + void loadRowsCount(); + + /// Loads ttl infos in json format from file ttl.txt. If file doesn`t exists assigns ttl infos with all zeros + void loadTTLInfos(); + + void loadPartitionAndMinMaxIndex(); + String getRelativePathForDetachedPart(const String & prefix) const; void checkConsistency(bool require_part_metadata); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index acab159e4ba..1f89943e32f 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -68,7 +68,7 @@ public: virtual MarkWithOffset write( const Block & block, const IColumn::Permutation * permutation, - size_t from_mark, size_t offset, const MergeTreeIndexGranularity & index_granularity, + size_t from_mark, size_t offset, MergeTreeIndexGranularity & index_granularity, /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) = 0; diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 59b90639ba0..59ac49122e2 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -76,8 +76,11 @@ void fillIndexGranularityImpl( /// We should be less or equal than fixed index granularity index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); - for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) - index_granularity.appendMark(index_granularity_for_block); + UNUSED(index_offset); + UNUSED(index_granularity); + /// FIXME + // for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) + // index_granularity.appendMark(index_granularity_for_block); } void IMergedBlockOutputStream::fillIndexGranularity(const Block & block) diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp index 83614e7bf34..c89f87da881 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp @@ -149,7 +149,7 @@ Block MergeTreeBaseSelectBlockInputStream::readFromPartImpl() UInt64 recommended_rows = estimateNumRows(*task, task->range_reader); UInt64 rows_to_read = std::max(UInt64(1), std::min(current_max_block_size_rows, recommended_rows)); - std::cerr << "(readFromPartImpl) rows_to_read: " << rows_to_read << "\n"; + // std::cerr << "(readFromPartImpl) rows_to_read: " << rows_to_read << "\n"; auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index e0be40c8c99..0addc3c94a0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -811,6 +811,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) { const auto & part_name = part_names_with_disks[i].first; const auto part_disk_ptr = part_names_with_disks[i].second; + + std::cerr << "(loadDataParts) loading part: " << part_name << "\n"; + MergeTreePartInfo part_info; if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 040d77a100f..3678b74ef92 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1090,6 +1090,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor merge_entry->columns_written = all_columns.size() - updated_header.columns(); + new_data_part->index_granularity = source_part->index_granularity; IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( new_data_part, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 9bf6e98fdbf..759f01b31d9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -47,10 +47,10 @@ namespace ErrorCodes } -static ReadBufferFromFile openForReading(const String & path) -{ - return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); -} +// static ReadBufferFromFile openForReading(const String & path) +// { +// return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); +// } MergeTreeDataPartCompact::MergeTreeDataPartCompact( MergeTreeData & storage_, @@ -159,170 +159,6 @@ String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const return *minimum_size_column; } -MergeTreeDataPartCompact::~MergeTreeDataPartCompact() -{ - if (state == State::DeleteOnDestroy || is_temp) - { - try - { - std::string path = getFullPath(); - - Poco::File dir(path); - if (!dir.exists()) - return; - - if (is_temp) - { - String file_name = Poco::Path(relative_path).getFileName(); - - if (file_name.empty()) - throw Exception("relative_path " + relative_path + " of part " + name + " is invalid or not set", ErrorCodes::LOGICAL_ERROR); - - if (!startsWith(file_name, "tmp")) - { - LOG_ERROR(storage.log, "~DataPart() should remove part " << path - << " but its name doesn't start with tmp. Too suspicious, keeping the part."); - return; - } - } - - dir.remove(true); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } -} - -UInt64 MergeTreeDataPartCompact::calculateTotalSizeOnDisk(const String & from) -{ - Poco::File cur(from); - if (cur.isFile()) - return cur.getSize(); - std::vector files; - cur.list(files); - UInt64 res = 0; - for (const auto & file : files) - res += calculateTotalSizeOnDisk(from + file); - return res; -} - -void MergeTreeDataPartCompact::remove() const -{ - if (relative_path.empty()) - throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); - - /** Atomic directory removal: - * - rename directory to temporary name; - * - remove it recursive. - * - * For temporary name we use "delete_tmp_" prefix. - * - * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, - * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. - * But when we removing data part, it can be old enough. And rename doesn't change mtime. - * And a race condition can happen that will lead to "File not found" error here. - */ - - String full_path = storage.getFullPathOnDisk(disk); - String from = full_path + relative_path; - String to = full_path + "delete_tmp_" + name; - // TODO directory delete_tmp_ is never removed if server crashes before returning from this function - - - Poco::File from_dir{from}; - Poco::File to_dir{to}; - - if (to_dir.exists()) - { - LOG_WARNING(storage.log, "Directory " << to << " (to which part must be renamed before removing) already exists." - " Most likely this is due to unclean restart. Removing it."); - - try - { - to_dir.remove(true); - } - catch (...) - { - LOG_ERROR(storage.log, "Cannot remove directory " << to << ". Check owner and access rights."); - throw; - } - } - - try - { - from_dir.renameTo(to); - } - catch (const Poco::FileNotFoundException &) - { - LOG_ERROR(storage.log, "Directory " << from << " (part to remove) doesn't exist or one of nested files has gone." - " Most likely this is due to manual removing. This should be discouraged. Ignoring."); - - return; - } - - try - { - /// Remove each expected file in directory, then remove directory itself. - -#if !__clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" -#endif - std::shared_lock lock(columns_lock); - - for (const auto & [file, _] : checksums.files) - { - String path_to_remove = to + "/" + file; - if (0 != unlink(path_to_remove.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, - ErrorCodes::CANNOT_UNLINK); - } -#if !__clang__ -#pragma GCC diagnostic pop -#endif - - for (const auto & file : {"checksums.txt", "columns.txt"}) - { - String path_to_remove = to + "/" + file; - if (0 != unlink(path_to_remove.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, - ErrorCodes::CANNOT_UNLINK); - } - - if (0 != rmdir(to.c_str())) - throwFromErrnoWithPath("Cannot rmdir file " + to, to, ErrorCodes::CANNOT_UNLINK); - } - catch (...) - { - /// Recursive directory removal does many excessive "stat" syscalls under the hood. - - LOG_ERROR(storage.log, "Cannot quickly remove directory " << to << " by removing files; fallback to recursive removal. Reason: " - << getCurrentExceptionMessage(false)); - - to_dir.remove(true); - } -} - -void MergeTreeDataPartCompact::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool /* check_consistency */) -{ - /// Memory should not be limited during ATTACH TABLE query. - /// This is already true at the server startup but must be also ensured for manual table ATTACH. - /// Motivation: memory for index is shared between queries - not belong to the query itself. - auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - - loadColumns(require_columns_checksums); - loadChecksums(require_columns_checksums); - loadIndexGranularity(); - loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` - loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. - loadPartitionAndMinMaxIndex(); - loadTTLInfos(); - // if (check_consistency) - // checkConsistency(require_columns_checksums); -} - void MergeTreeDataPartCompact::loadIndexGranularity() { String full_path = getFullPath(); @@ -330,200 +166,29 @@ void MergeTreeDataPartCompact::loadIndexGranularity() if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - /// We can use any column, it doesn't matter + if (!index_granularity_info.is_adaptive) + throw Exception("MergeTreeDataPartCompact cannot be created with non-adaptive granulary. TODO: help message", ErrorCodes::NOT_IMPLEMENTED); + std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + "data"); if (!Poco::File(marks_file_path).exists()) throw Exception("Marks file '" + marks_file_path + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); size_t marks_file_size = Poco::File(marks_file_path).getSize(); - /// old version of marks with static index granularity - ReadBufferFromFile buffer(marks_file_path, marks_file_size, -1); + ReadBufferFromFile buffer(marks_file_path, marks_file_size); while (!buffer.eof()) { size_t granularity; readIntBinary(granularity, buffer); index_granularity.appendMark(granularity); /// Skip offsets for columns - buffer.seek(sizeof(size_t) * 2 * columns.size()); + buffer.seek(index_granularity_info.mark_size_in_bytes, SEEK_CUR); } + if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); -} -void MergeTreeDataPartCompact::loadIndex() -{ - /// It can be empty in case of mutations - if (!index_granularity.isInitialized()) - throw Exception("Index granularity is not loaded before index loading", ErrorCodes::LOGICAL_ERROR); - - size_t key_size = storage.primary_key_columns.size(); - - if (key_size) - { - MutableColumns loaded_index; - loaded_index.resize(key_size); - - for (size_t i = 0; i < key_size; ++i) - { - loaded_index[i] = storage.primary_key_data_types[i]->createColumn(); - loaded_index[i]->reserve(index_granularity.getMarksCount()); - } - - String index_path = getFullPath() + "primary.idx"; - ReadBufferFromFile index_file = openForReading(index_path); - - for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 - for (size_t j = 0; j < key_size; ++j) - storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], index_file); - - for (size_t i = 0; i < key_size; ++i) - { - loaded_index[i]->protect(); - if (loaded_index[i]->size() != index_granularity.getMarksCount()) - throw Exception("Cannot read all data from index file " + index_path - + "(expected size: " + toString(index_granularity.getMarksCount()) + ", read: " + toString(loaded_index[i]->size()) + ")", - ErrorCodes::CANNOT_READ_ALL_DATA); - } - - if (!index_file.eof()) - throw Exception("Index file " + index_path + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); - - index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); - } -} - -void MergeTreeDataPartCompact::loadPartitionAndMinMaxIndex() -{ - if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - DayNum min_date; - DayNum max_date; - MergeTreePartInfo::parseMinMaxDatesFromPartName(name, min_date, max_date); - - const auto & date_lut = DateLUT::instance(); - partition = MergeTreePartition(date_lut.toNumYYYYMM(min_date)); - minmax_idx = MinMaxIndex(min_date, max_date); - } - else - { - String path = getFullPath(); - partition.load(storage, path); - if (!isEmpty()) - minmax_idx.load(storage, path); - } - - String calculated_partition_id = partition.getID(storage.partition_key_sample); - if (calculated_partition_id != info.partition_id) - throw Exception( - "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id - + " differs from partition ID in part name: " + info.partition_id, - ErrorCodes::CORRUPTED_DATA); -} - -void MergeTreeDataPartCompact::loadChecksums(bool require) -{ - String path = getFullPath() + "checksums.txt"; - Poco::File checksums_file(path); - if (checksums_file.exists()) - { - ReadBufferFromFile file = openForReading(path); - if (checksums.read(file)) - { - assertEOF(file); - bytes_on_disk = checksums.getTotalSizeOnDisk(); - } - else - bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); - } - else - { - if (require) - throw Exception("No checksums.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); - } -} - -void MergeTreeDataPartCompact::loadRowsCount() -{ - if (index_granularity.empty()) - { - rows_count = 0; - } - else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - String path = getFullPath() + "count.txt"; - if (!Poco::File(path).exists()) - throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - ReadBufferFromFile file = openForReading(path); - readIntText(rows_count, file); - assertEOF(file); - } - else - { - for (const NameAndTypePair & column : columns) - { - ColumnPtr column_col = column.type->createColumn(); - if (!column_col->isFixedAndContiguous() || column_col->lowCardinality()) - continue; - - size_t column_size = getColumnSizeImpl(column.name, *column.type, nullptr).data_uncompressed; - if (!column_size) - continue; - - size_t sizeof_field = column_col->sizeOfValueIfFixed(); - rows_count = column_size / sizeof_field; - - if (column_size % sizeof_field != 0) - { - throw Exception( - "Uncompressed size of column " + column.name + "(" + toString(column_size) - + ") is not divisible by the size of value (" + toString(sizeof_field) + ")", - ErrorCodes::LOGICAL_ERROR); - } - - size_t last_mark_index_granularity = index_granularity.getLastNonFinalMarkRows(); - size_t rows_approx = index_granularity.getTotalRows(); - if (!(rows_count <= rows_approx && rows_approx < rows_count + last_mark_index_granularity)) - throw Exception( - "Unexpected size of column " + column.name + ": " + toString(rows_count) + " rows, expected " - + toString(rows_approx) + "+-" + toString(last_mark_index_granularity) + " rows according to the index", - ErrorCodes::LOGICAL_ERROR); - - return; - } - - throw Exception("Data part doesn't contain fixed size column (even Date column)", ErrorCodes::LOGICAL_ERROR); - } -} - -void MergeTreeDataPartCompact::loadTTLInfos() -{ - String path = getFullPath() + "ttl.txt"; - if (Poco::File(path).exists()) - { - ReadBufferFromFile in = openForReading(path); - assertString("ttl format version: ", in); - size_t format_version; - readText(format_version, in); - assertChar('\n', in); - - if (format_version == 1) - { - try - { - ttl_infos.read(in); - } - catch (const JSONException &) - { - throw Exception("Error while parsing file ttl.txt in part: " + name, ErrorCodes::BAD_TTL_FILE); - } - } - else - throw Exception("Unknown ttl format version: " + toString(format_version), ErrorCodes::BAD_TTL_FILE); - } + index_granularity.setInitialized(); } // void MergeTreeDataPartCompact::accumulateColumnSizes(ColumnToSize & column_to_size) const @@ -542,38 +207,6 @@ void MergeTreeDataPartCompact::loadTTLInfos() // } // } -void MergeTreeDataPartCompact::loadColumns(bool require) -{ - String path = getFullPath() + "columns.txt"; - Poco::File poco_file_path{path}; - if (!poco_file_path.exists()) - { - if (require) - throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - /// If there is no file with a list of columns, write it down. - for (const NameAndTypePair & column : storage.getColumns().getAllPhysical()) - if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) - columns.push_back(column); - - if (columns.empty()) - throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - { - WriteBufferFromFile out(path + ".tmp", 4096); - columns.writeText(out); - } - Poco::File(path + ".tmp").renameTo(path); - - return; - } - - is_frozen = !poco_file_path.canWrite(); - - ReadBufferFromFile file = openForReading(path); - columns.readText(file); -} - void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { String path = getFullPath(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 7b9e9631a49..3fe40f1691d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -65,46 +65,15 @@ public: bool isStoredOnDisk() const override { return true; } - void remove() const override; - - - /// Initialize columns (from columns.txt if exists, or create from column files if not). - /// Load checksums from checksums.txt if exists. Load index if required. - void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) override; - /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; virtual Type getType() const override { return Type::COMPACT; } - ~MergeTreeDataPartCompact() override; - - /// Calculate the total size of the entire directory with all the files - static UInt64 calculateTotalSizeOnDisk(const String & from); - - private: - /// Reads columns names and types from columns.txt - void loadColumns(bool require); - - /// If checksums.txt exists, reads files' checksums (and sizes) from it - void loadChecksums(bool require); - /// Loads marks index granularity into memory - void loadIndexGranularity(); - - /// Loads index file. - void loadIndex(); - - /// Load rows count for this part from disk (for the newer storage format version). - /// For the older format version calculates rows count from the size of a column with a fixed size. - void loadRowsCount(); - - /// Loads ttl infos in json format from file ttl.txt. If file doesn`t exists assigns ttl infos with all zeros - void loadTTLInfos(); - - void loadPartitionAndMinMaxIndex(); + void loadIndexGranularity() override; ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp index 8f16b088ec0..15d259379c4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp @@ -8,22 +8,22 @@ namespace DB const MergeTreePartInfo & info, const String & relative_path) { // /// FIXME - // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - // return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); + size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); - return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); + // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); + // return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); } std::shared_ptr createPart(MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const String & relative_path) { - // /// FIXME - // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - // return std::make_shared(storage, name, index_granularity_info, disk, relative_path); - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); - return std::make_shared(storage, name, index_granularity_info, disk, relative_path); + /// FIXME + size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + return std::make_shared(storage, name, index_granularity_info, disk, relative_path); + // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); + // return std::make_shared(storage, name, index_granularity_info, disk, relative_path); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.cpp deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h deleted file mode 100644 index 70e799ba926..00000000000 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartOnDisk.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -namespace DB -{ - -class MergeTreeDataPartOnDisk : IMergeTreeDataPart -{ - - -}; - -} \ No newline at end of file diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index bf626b9d68a..7ef500cd98b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -48,10 +48,10 @@ namespace ErrorCodes } -static ReadBufferFromFile openForReading(const String & path) -{ - return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); -} +// static ReadBufferFromFile openForReading(const String & path) +// { +// return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); +// } MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, @@ -158,42 +158,6 @@ String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const return *minimum_size_column; } -MergeTreeDataPartWide::~MergeTreeDataPartWide() -{ - if (state == State::DeleteOnDestroy || is_temp) - { - try - { - std::string path = getFullPath(); - - Poco::File dir(path); - if (!dir.exists()) - return; - - if (is_temp) - { - String file_name = Poco::Path(relative_path).getFileName(); - - if (file_name.empty()) - throw Exception("relative_path " + relative_path + " of part " + name + " is invalid or not set", ErrorCodes::LOGICAL_ERROR); - - if (!startsWith(file_name, "tmp")) - { - LOG_ERROR(storage.log, "~DataPart() should remove part " << path - << " but its name doesn't start with tmp. Too suspicious, keeping the part."); - return; - } - } - - dir.remove(true); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } -} - UInt64 MergeTreeDataPartWide::calculateTotalSizeOnDisk(const String & from) { Poco::File cur(from); @@ -207,126 +171,6 @@ UInt64 MergeTreeDataPartWide::calculateTotalSizeOnDisk(const String & from) return res; } -void MergeTreeDataPartWide::remove() const -{ - if (relative_path.empty()) - throw Exception("Part relative_path cannot be empty. This is bug.", ErrorCodes::LOGICAL_ERROR); - - /** Atomic directory removal: - * - rename directory to temporary name; - * - remove it recursive. - * - * For temporary name we use "delete_tmp_" prefix. - * - * NOTE: We cannot use "tmp_delete_" prefix, because there is a second thread, - * that calls "clearOldTemporaryDirectories" and removes all directories, that begin with "tmp_" and are old enough. - * But when we removing data part, it can be old enough. And rename doesn't change mtime. - * And a race condition can happen that will lead to "File not found" error here. - */ - - String full_path = storage.getFullPathOnDisk(disk); - String from = full_path + relative_path; - String to = full_path + "delete_tmp_" + name; - // TODO directory delete_tmp_ is never removed if server crashes before returning from this function - - - Poco::File from_dir{from}; - Poco::File to_dir{to}; - - if (to_dir.exists()) - { - LOG_WARNING(storage.log, "Directory " << to << " (to which part must be renamed before removing) already exists." - " Most likely this is due to unclean restart. Removing it."); - - try - { - to_dir.remove(true); - } - catch (...) - { - LOG_ERROR(storage.log, "Cannot remove directory " << to << ". Check owner and access rights."); - throw; - } - } - - try - { - from_dir.renameTo(to); - } - catch (const Poco::FileNotFoundException &) - { - LOG_ERROR(storage.log, "Directory " << from << " (part to remove) doesn't exist or one of nested files has gone." - " Most likely this is due to manual removing. This should be discouraged. Ignoring."); - - return; - } - - try - { - /// Remove each expected file in directory, then remove directory itself. - -#if !__clang__ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunused-variable" -#endif - std::shared_lock lock(columns_lock); - - std::cerr << "removing: " << name << "\n"; - std::cerr << "checksums size: " << checksums.files.size() << "\n"; - - for (const auto & [file, _] : checksums.files) - { - std::cerr << "checksums file: " << file << "\n"; - - String path_to_remove = to + "/" + file; - if (0 != unlink(path_to_remove.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, - ErrorCodes::CANNOT_UNLINK); - } -#if !__clang__ -#pragma GCC diagnostic pop -#endif - - for (const auto & file : {"checksums.txt", "columns.txt"}) - { - String path_to_remove = to + "/" + file; - if (0 != unlink(path_to_remove.c_str())) - throwFromErrnoWithPath("Cannot unlink file " + path_to_remove, path_to_remove, - ErrorCodes::CANNOT_UNLINK); - } - - if (0 != rmdir(to.c_str())) - throwFromErrnoWithPath("Cannot rmdir file " + to, to, ErrorCodes::CANNOT_UNLINK); - } - catch (...) - { - /// Recursive directory removal does many excessive "stat" syscalls under the hood. - - LOG_ERROR(storage.log, "Cannot quickly remove directory " << to << " by removing files; fallback to recursive removal. Reason: " - << getCurrentExceptionMessage(false)); - - to_dir.remove(true); - } -} - -void MergeTreeDataPartWide::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool /* check_consistency */) -{ - /// Memory should not be limited during ATTACH TABLE query. - /// This is already true at the server startup but must be also ensured for manual table ATTACH. - /// Motivation: memory for index is shared between queries - not belong to the query itself. - auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - - loadColumns(require_columns_checksums); - loadChecksums(require_columns_checksums); - loadIndexGranularity(); - loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` - loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. - loadPartitionAndMinMaxIndex(); - loadTTLInfos(); - // if (check_consistency) - // checkConsistency(require_columns_checksums); -} - void MergeTreeDataPartWide::loadIndexGranularity() { String full_path = getFullPath(); @@ -357,187 +201,14 @@ void MergeTreeDataPartWide::loadIndexGranularity() readIntBinary(granularity, buffer); index_granularity.appendMark(granularity); } + if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); } + index_granularity.setInitialized(); } -void MergeTreeDataPartWide::loadIndex() -{ - /// It can be empty in case of mutations - if (!index_granularity.isInitialized()) - throw Exception("Index granularity is not loaded before index loading", ErrorCodes::LOGICAL_ERROR); - - size_t key_size = storage.primary_key_columns.size(); - - if (key_size) - { - MutableColumns loaded_index; - loaded_index.resize(key_size); - - for (size_t i = 0; i < key_size; ++i) - { - loaded_index[i] = storage.primary_key_data_types[i]->createColumn(); - loaded_index[i]->reserve(index_granularity.getMarksCount()); - } - - String index_path = getFullPath() + "primary.idx"; - ReadBufferFromFile index_file = openForReading(index_path); - - for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756 - for (size_t j = 0; j < key_size; ++j) - storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], index_file); - - for (size_t i = 0; i < key_size; ++i) - { - loaded_index[i]->protect(); - if (loaded_index[i]->size() != index_granularity.getMarksCount()) - throw Exception("Cannot read all data from index file " + index_path - + "(expected size: " + toString(index_granularity.getMarksCount()) + ", read: " + toString(loaded_index[i]->size()) + ")", - ErrorCodes::CANNOT_READ_ALL_DATA); - } - - if (!index_file.eof()) - throw Exception("Index file " + index_path + " is unexpectedly long", ErrorCodes::EXPECTED_END_OF_FILE); - - index.assign(std::make_move_iterator(loaded_index.begin()), std::make_move_iterator(loaded_index.end())); - } -} - -void MergeTreeDataPartWide::loadPartitionAndMinMaxIndex() -{ - if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - DayNum min_date; - DayNum max_date; - MergeTreePartInfo::parseMinMaxDatesFromPartName(name, min_date, max_date); - - const auto & date_lut = DateLUT::instance(); - partition = MergeTreePartition(date_lut.toNumYYYYMM(min_date)); - minmax_idx = MinMaxIndex(min_date, max_date); - } - else - { - String path = getFullPath(); - partition.load(storage, path); - if (!isEmpty()) - minmax_idx.load(storage, path); - } - - String calculated_partition_id = partition.getID(storage.partition_key_sample); - if (calculated_partition_id != info.partition_id) - throw Exception( - "While loading part " + getFullPath() + ": calculated partition ID: " + calculated_partition_id - + " differs from partition ID in part name: " + info.partition_id, - ErrorCodes::CORRUPTED_DATA); -} - -void MergeTreeDataPartWide::loadChecksums(bool require) -{ - String path = getFullPath() + "checksums.txt"; - Poco::File checksums_file(path); - if (checksums_file.exists()) - { - ReadBufferFromFile file = openForReading(path); - if (checksums.read(file)) - { - assertEOF(file); - bytes_on_disk = checksums.getTotalSizeOnDisk(); - } - else - bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); - } - else - { - if (require) - throw Exception("No checksums.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - bytes_on_disk = calculateTotalSizeOnDisk(getFullPath()); - } -} - -void MergeTreeDataPartWide::loadRowsCount() -{ - if (index_granularity.empty()) - { - rows_count = 0; - } - else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - String path = getFullPath() + "count.txt"; - if (!Poco::File(path).exists()) - throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - ReadBufferFromFile file = openForReading(path); - readIntText(rows_count, file); - assertEOF(file); - } - else - { - for (const NameAndTypePair & column : columns) - { - ColumnPtr column_col = column.type->createColumn(); - if (!column_col->isFixedAndContiguous() || column_col->lowCardinality()) - continue; - - size_t column_size = getColumnSizeImpl(column.name, *column.type, nullptr).data_uncompressed; - if (!column_size) - continue; - - size_t sizeof_field = column_col->sizeOfValueIfFixed(); - rows_count = column_size / sizeof_field; - - if (column_size % sizeof_field != 0) - { - throw Exception( - "Uncompressed size of column " + column.name + "(" + toString(column_size) - + ") is not divisible by the size of value (" + toString(sizeof_field) + ")", - ErrorCodes::LOGICAL_ERROR); - } - - size_t last_mark_index_granularity = index_granularity.getLastNonFinalMarkRows(); - size_t rows_approx = index_granularity.getTotalRows(); - if (!(rows_count <= rows_approx && rows_approx < rows_count + last_mark_index_granularity)) - throw Exception( - "Unexpected size of column " + column.name + ": " + toString(rows_count) + " rows, expected " - + toString(rows_approx) + "+-" + toString(last_mark_index_granularity) + " rows according to the index", - ErrorCodes::LOGICAL_ERROR); - - return; - } - - throw Exception("Data part doesn't contain fixed size column (even Date column)", ErrorCodes::LOGICAL_ERROR); - } -} - -void MergeTreeDataPartWide::loadTTLInfos() -{ - String path = getFullPath() + "ttl.txt"; - if (Poco::File(path).exists()) - { - ReadBufferFromFile in = openForReading(path); - assertString("ttl format version: ", in); - size_t format_version; - readText(format_version, in); - assertChar('\n', in); - - if (format_version == 1) - { - try - { - ttl_infos.read(in); - } - catch (const JSONException &) - { - throw Exception("Error while parsing file ttl.txt in part: " + name, ErrorCodes::BAD_TTL_FILE); - } - } - else - throw Exception("Unknown ttl format version: " + toString(format_version), ErrorCodes::BAD_TTL_FILE); - } -} - // void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) const // { // std::shared_lock part_lock(columns_lock); @@ -554,140 +225,7 @@ void MergeTreeDataPartWide::loadTTLInfos() // } // } -void MergeTreeDataPartWide::loadColumns(bool require) -{ - String path = getFullPath() + "columns.txt"; - Poco::File poco_file_path{path}; - if (!poco_file_path.exists()) - { - if (require) - throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - /// If there is no file with a list of columns, write it down. - for (const NameAndTypePair & column : storage.getColumns().getAllPhysical()) - if (Poco::File(getFullPath() + escapeForFileName(column.name) + ".bin").exists()) - columns.push_back(column); - - if (columns.empty()) - throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - { - WriteBufferFromFile out(path + ".tmp", 4096); - columns.writeText(out); - } - Poco::File(path + ".tmp").renameTo(path); - - return; - } - - is_frozen = !poco_file_path.canWrite(); - - ReadBufferFromFile file = openForReading(path); - columns.readText(file); -} - -void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) -{ - String path = getFullPath(); - - if (!checksums.empty()) - { - if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) - throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); - - if (require_part_metadata) - { - for (const NameAndTypePair & name_type : columns) - { - IDataType::SubstreamPath stream_path; - name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); - String mrk_file_name = file_name + index_granularity_info.marks_file_extension; - String bin_file_name = file_name + ".bin"; - if (!checksums.files.count(mrk_file_name)) - throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + path, - ErrorCodes::NO_FILE_IN_DATA_PART); - if (!checksums.files.count(bin_file_name)) - throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + path, - ErrorCodes::NO_FILE_IN_DATA_PART); - }, stream_path); - } - } - - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - if (!checksums.files.count("count.txt")) - throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); - - if (storage.partition_key_expr && !checksums.files.count("partition.dat")) - throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); - - if (!isEmpty()) - { - for (const String & col_name : storage.minmax_idx_columns) - { - if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) - throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); - } - } - } - - checksums.checkSizes(path); - } - else - { - auto check_file_not_empty = [&path](const String & file_path) - { - Poco::File file(file_path); - if (!file.exists() || file.getSize() == 0) - throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - return file.getSize(); - }; - - /// Check that the primary key index is not empty. - if (!storage.primary_key_columns.empty()) - check_file_not_empty(path + "primary.idx"); - - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - check_file_not_empty(path + "count.txt"); - - if (storage.partition_key_expr) - check_file_not_empty(path + "partition.dat"); - - for (const String & col_name : storage.minmax_idx_columns) - check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); - } - - /// Check that all marks are nonempty and have the same size. - - std::optional marks_size; - for (const NameAndTypePair & name_type : columns) - { - name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - Poco::File file(IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension); - - /// Missing file is Ok for case when new column was added. - if (file.exists()) - { - UInt64 file_size = file.getSize(); - - if (!file_size) - throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - - if (!marks_size) - marks_size = file_size; - else if (file_size != *marks_size) - throw Exception("Part " + path + " is broken: marks have different sizes.", - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - } - }); - } - } -} // bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDataType & type) const // { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 81c3bb776a5..9fd8953dcaa 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -65,47 +65,21 @@ public: bool isStoredOnDisk() const override { return true; } - void remove() const override; - bool supportsVerticalMerge() const override { return true; } - /// Initialize columns (from columns.txt if exists, or create from column files if not). - /// Load checksums from checksums.txt if exists. Load index if required. - void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) override; - /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; Type getType() const override { return Type::WIDE; } - ~MergeTreeDataPartWide() override; - /// Calculate the total size of the entire directory with all the files static UInt64 calculateTotalSizeOnDisk(const String & from); private: - /// Reads columns names and types from columns.txt - void loadColumns(bool require); - - /// If checksums.txt exists, reads files' checksums (and sizes) from it - void loadChecksums(bool require); - /// Loads marks index granularity into memory - void loadIndexGranularity(); - - /// Loads index file. - void loadIndex(); - - /// Load rows count for this part from disk (for the newer storage format version). - /// For the older format version calculates rows count from the size of a column with a fixed size. - void loadRowsCount(); - - /// Loads ttl infos in json format from file ttl.txt. If file doesn`t exists assigns ttl infos with all zeros - void loadTTLInfos(); - - void loadPartitionAndMinMaxIndex(); + void loadIndexGranularity() override; ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index ad1ec6993b2..f99dcb1747e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -35,7 +35,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, - const MergeTreeIndexGranularity & index_granularity, + MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block, bool skip_offsets, const WrittenOffsetColumns & already_written_offset_columns) { @@ -73,10 +73,11 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( while (current_row < total_rows) { std::cerr << "(MergeTreeDataPartWriterCompact::write) current_row: " << current_row << "\n"; - bool write_marks = true; - size_t rows_to_write = std::min(total_rows, index_granularity.getMarkRows(current_mark)); + // size_t rows_to_write = std::min(total_rows, index_granularity.getMarkRows(current_mark)); + size_t rows_to_write = total_rows; + index_granularity.appendMark(total_rows); // if (current_row == 0 && index_offset != 0) // { // rows_to_write = index_offset; @@ -87,11 +88,13 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( // rows_to_write = index_granularity.getMarkRows(current_mark); // } - std::cerr << "(MergeTreeDataPartWriterCompact::write) rows_to_write: " << rows_to_write << "\n"; + // std::cerr << "(MergeTreeDataPartWriterCompact::write) rows_to_write: " << rows_to_write << "\n"; /// There could already be enough data to compress into the new block. - if (stream->compressed.offset() >= settings.min_compress_block_size) + if (stream->compressed.offset() >= settings.min_compress_block_size) stream->compressed.next(); + + size_t next_row = 0; if (write_marks) { @@ -100,15 +103,16 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( { writeIntBinary(stream->plain_hashing.count(), stream->marks); writeIntBinary(stream->compressed.offset(), stream->marks); - current_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); + next_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); } ++current_mark; } else { for (size_t i = 0; i < columns_to_write.size(); ++i) - current_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); + next_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); } + current_row = next_row; } return {current_mark, total_rows - current_row}; @@ -116,6 +120,10 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) { + + std::cerr << "(writeColumnSingleGranule) writing column: " << column.name << "\n"; + std::cerr << "(writeColumnSingleGranule) from_row: " << from_row << "\n"; + std::cerr << "(writeColumnSingleGranule) number_of_rows: " << number_of_rows << "\n"; IDataType::SerializeBinaryBulkStatePtr state; IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -134,7 +142,7 @@ void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & ch { if (write_final_mark) { - writeIntBinary(0, stream->marks); + writeIntBinary(0ULL, stream->marks); for (size_t i = 0; i < columns_list.size(); ++i) { writeIntBinary(stream->plain_hashing.count(), stream->marks); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index e92b03a5d78..2aab9ca6558 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -15,7 +15,7 @@ public: const WriterSettings & settings); MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, - size_t from_mark, size_t index_offset, const MergeTreeIndexGranularity & index_granularity, + size_t from_mark, size_t index_offset, MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block, bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index c82b2de93cf..eae2c6ec15e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -92,7 +92,7 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, - const MergeTreeIndexGranularity & index_granularity, + MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block, const Block & skip_indexes_block, bool skip_offsets, const WrittenOffsetColumns & already_written_offset_columns) { @@ -221,13 +221,17 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( const String & name, const IDataType & type, const IColumn & column, - const MergeTreeIndexGranularity & index_granularity, + MergeTreeIndexGranularity & index_granularity, WrittenOffsetColumns & offset_columns, bool skip_offsets, IDataType::SerializeBinaryBulkStatePtr & serialization_state, size_t from_mark, size_t index_offset) { + std::cerr << "(writeColumn) table: " << storage.getTableName() << "\n"; + std::cerr << "(writeColumn) column: " << name << "\n"; + std::cerr << "(writeColumn) from_mark: " << from_mark << "\n"; + std::cerr << "(writeColumn) index_offset: " << index_offset << "\n"; auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.getter = createStreamGetter(name, offset_columns, skip_offsets); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 18ee533f4be..b184b3cdab0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -20,7 +20,7 @@ public: MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, - const MergeTreeIndexGranularity & index_granularity, + MergeTreeIndexGranularity & index_granularity, const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) override; @@ -35,7 +35,7 @@ public: const String & name, const IDataType & type, const IColumn & column, - const MergeTreeIndexGranularity & index_granularity, + MergeTreeIndexGranularity & index_granularity, WrittenOffsetColumns & offset_columns, bool skip_offsets, IDataType::SerializeBinaryBulkStatePtr & serialization_state, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 8d73c4a5809..08c59b695d3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -168,7 +168,7 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPartition & block_with_partition) { Block & block = block_with_partition.block; - std::cerr << "(MergeTreeDataWriter::writeTempPart) block.rows(): " << block.rows() << "\n"; + // std::cerr << "(MergeTreeDataWriter::writeTempPart) block.rows(): " << block.rows() << "\n"; static const String TMP_PREFIX = "tmp_insert_"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 84f9ee04299..2eb53f9280d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -231,6 +231,9 @@ void MergeTreeRangeReader::ReadResult::addGranule(size_t num_rows) void MergeTreeRangeReader::ReadResult::adjustLastGranule() { + std::cerr << "(adjustLastGranule) num_read_rows: " << num_read_rows << "\n"; + std::cerr << "(adjustLastGranule) total_rows_per_granule: " << total_rows_per_granule << "\n"; + size_t num_rows_to_subtract = total_rows_per_granule - num_read_rows; if (rows_per_granule.empty()) @@ -588,6 +591,9 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t } auto rows_to_read = std::min(space_left, stream.numPendingRowsInCurrentGranule()); + + std::cerr << "(startReadingChain) rows_to_read: " << rows_to_read << "\n"; + bool last = rows_to_read == space_left; result.addRows(stream.read(result.block, rows_to_read, !last)); result.addGranule(rows_to_read); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index b2130b9704e..7abfa76a88d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -24,7 +24,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr if (uncompressed_cache) { auto buffer = std::make_unique( - "data.bin", uncompressed_cache, 0, settings.min_bytes_to_use_direct_io, buffer_size); + path + "data.bin", uncompressed_cache, 0, settings.min_bytes_to_use_direct_io, buffer_size); // if (profile_callback) // buffer->setProfileCallback(profile_callback, clock_type); @@ -35,7 +35,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr else { auto buffer = std::make_unique( - "data.bin", 0, settings.min_bytes_to_use_direct_io, buffer_size); + path + "data.bin", 0, settings.min_bytes_to_use_direct_io, buffer_size); // if (profile_callback) // buffer->setProfileCallback(profile_callback, clock_type); @@ -55,7 +55,6 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); size_t read_rows = 0; - size_t ind = 0; for (const auto & it : columns) { bool append = res.has(it.name); @@ -68,8 +67,9 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, try { size_t column_size_before_reading = column->size(); + size_t column_position = data_part->getColumnPosition(it.name); - readData(it.name, *it.type, *column, from_mark, ind++, rows_to_read); + readData(it.name, *it.type, *column, from_mark, column_position, rows_to_read); /// For elements of Nested, column_size_before_reading may be greater than column size /// if offsets are not empty and were already read, but elements are empty. @@ -97,6 +97,11 @@ void MergeTreeReaderCompact::readData( const String & name, const IDataType & type, IColumn & column, size_t from_mark, size_t column_position, size_t rows_to_read) { + std::cerr << "(MergeTreeReaderCompact::readData) from_mark: " << from_mark << "\n"; + std::cerr << "(MergeTreeReaderCompact::readData) column_position: " << column_position << "\n"; + std::cerr << "(MergeTreeReaderCompact::readData) rows_to_read: " << rows_to_read << "\n"; + std::cerr << "(MergeTreeReaderCompact::readData) start reading column: " << name << "\n"; + seekToMark(from_mark, column_position); IDataType::DeserializeBinaryBulkSettings deserialize_settings; @@ -108,9 +113,11 @@ void MergeTreeReaderCompact::readData( type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); type.deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state); - if (column.size() != rows_to_read) - throw Exception("Cannot read all data in NativeBlockInputStream. Rows read: " + toString(column.size()) + ". Rows expected: "+ toString(rows_to_read) + ".", - ErrorCodes::CANNOT_READ_ALL_DATA); + std::cerr << "(MergeTreeReaderCompact::readData) end reading column: " << name << "\n"; + + // if (column.size() != rows_to_read) + // throw Exception("Cannot read all data in NativeBlockInputStream. Rows read: " + toString(column.size()) + ". Rows expected: "+ toString(rows_to_read) + ".", + // ErrorCodes::CANNOT_READ_ALL_DATA); } @@ -118,11 +125,12 @@ void MergeTreeReaderCompact::loadMarks() { const auto & index_granularity_info = data_part->index_granularity_info; size_t marks_count = data_part->getMarksCount(); - std::string mrk_path = index_granularity_info.getMarksFilePath(NAME_OF_FILE_WITH_DATA); + std::string mrk_path = index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); auto load_func = [&]() -> MarkCache::MappedPtr { size_t file_size = Poco::File(mrk_path).getSize(); + size_t expected_file_size = index_granularity_info.mark_size_in_bytes * marks_count; if (expected_file_size != file_size) throw Exception( @@ -132,18 +140,27 @@ void MergeTreeReaderCompact::loadMarks() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - auto res = std::make_shared(marks_count * columns.size()); + size_t columns_num = data_part->columns.size(); + + auto res = std::make_shared(marks_count * columns_num); + + std::cerr << "(MergeTreeReaderCompact::loadMarks) marks_count: " << marks_count << "\n"; ReadBufferFromFile buffer(mrk_path, file_size); size_t i = 0; while (!buffer.eof()) { - buffer.seek(sizeof(size_t)); - buffer.read(marks.getRowAddress(i), marks.getRowSize()); + buffer.seek(sizeof(size_t), SEEK_CUR); + buffer.readStrict(reinterpret_cast(res->data() + i * columns_num), sizeof(MarkInCompressedFile) * columns_num); + std::cerr << "(MergeTreeReaderCompact::loadMarks) i: " << i << "\n"; + std::cerr << "(MergeTreeReaderCompact::loadMarks) buffer pos in file: " << buffer.getPositionInFile() << "\n"; ++i; } + std::cerr << "(MergeTreeReaderCompact::loadMarks) file_size: " << file_size << "\n"; + std::cerr << "(MergeTreeReaderCompact::loadMarks) correct file size: " << i * index_granularity_info.mark_size_in_bytes << "\n"; + if (i * index_granularity_info.mark_size_in_bytes != file_size) throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); @@ -151,8 +168,13 @@ void MergeTreeReaderCompact::loadMarks() return res; }; + std::cerr << "(MergeTreeReaderCompact::loadMarks) table: " << storage.getTableName() << ", part: " << path << "\n"; + std::cerr << "(MergeTreeReaderCompact::loadMarks) start marks load..." << "\n"; + auto marks_array = IMergeTreeReader::loadMarks(mrk_path, load_func); marks = MarksInCompressedFileCompact(marks_array, columns.size()); + + std::cerr << "(MergeTreeReaderCompact::loadMarks) end marks load..." << "\n"; } const MarkInCompressedFile & MergeTreeReaderCompact::getMark(size_t row, size_t col) @@ -166,6 +188,8 @@ void MergeTreeReaderCompact::seekToMark(size_t row, size_t col) { MarkInCompressedFile mark = getMark(row, col); + std::cerr << "(MergeTreeReaderCompact::seekToMark) mark: (" << mark.offset_in_compressed_file << ", " << mark.offset_in_decompressed_block << "\n"; + try { if (cached_buffer) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 46013172846..a7ac5d48883 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -134,7 +134,7 @@ size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, si throw; } - std::cerr << "(MergeTreeReaderWide::readRows) read_rows: " << read_rows << "\n"; + // std::cerr << "(MergeTreeReaderWide::readRows) read_rows: " << read_rows << "\n"; return read_rows; } @@ -175,7 +175,7 @@ void MergeTreeReaderWide::readData( size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) { - std::cerr << "(MergeTreeReaderWide::readData) max_rows_to_read: " << max_rows_to_read << "\n"; + // std::cerr << "(MergeTreeReaderWide::readData) max_rows_to_read: " << max_rows_to_read << "\n"; auto get_stream_getter = [&](bool stream_for_prefix) -> IDataType::InputStreamGetter { return [&, stream_for_prefix](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index ac4b820d9fb..d0d0da0c234 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -29,6 +29,8 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; writer = data_part_->getWriter(header.getNamesAndTypesList(), default_codec_, writer_settings); + + std::cerr << "(MergedColumnOnlyOutputStream) writer: " << !!writer << "\n"; initSkipIndices(); } @@ -57,14 +59,14 @@ void MergedColumnOnlyOutputStream::write(const Block & block) UNUSED(skip_offsets); UNUSED(already_written_offset_columns); - auto [new_index_offset, new_current_mark] = writer->write(block, nullptr, current_mark, index_offset, index_granularity); + auto [new_current_mark, new_index_offset] = writer->write(block, nullptr, current_mark, index_offset, index_granularity); /// Should be written before index offset update, because we calculate, /// indices of currently written granules calculateAndSerializeSkipIndices(skip_indexes_columns, rows); - index_offset = new_index_offset; current_mark = new_current_mark; + index_offset = new_index_offset; } void MergedColumnOnlyOutputStream::writeSuffix() From 7293841003fe66e704f2ea84c89dc6e6e1eacd9e Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 5 Nov 2019 14:53:22 +0300 Subject: [PATCH 0050/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 3 +- .../Storages/MergeTree/IMergeTreeDataPart.h | 4 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 289 +++++++++++++++++- .../MergeTree/IMergeTreeDataPartWriter.h | 49 ++- .../src/Storages/MergeTree/IMergeTreeReader.h | 4 +- .../MergeTree/IMergedBlockOutputStream.cpp | 154 ---------- .../MergeTree/IMergedBlockOutputStream.h | 38 +-- .../MergeTree/MergeTreeDataMergerMutator.cpp | 6 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 3 +- .../MergeTree/MergeTreeDataPartWide.cpp | 14 - .../MergeTree/MergeTreeDataPartWide.h | 9 +- .../MergeTreeDataPartWriterCompact.cpp | 8 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 26 +- .../MergeTree/MergeTreeDataPartWriterWide.h | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 3 +- .../MergeTree/MergeTreeReaderSettings.h | 43 ++- .../MergeTree/MergedBlockOutputStream.cpp | 132 ++------ .../MergeTree/MergedBlockOutputStream.h | 12 - .../MergedColumnOnlyOutputStream.cpp | 5 +- 19 files changed, 431 insertions(+), 373 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index ed9beac9521..a26d5de6200 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -344,7 +344,8 @@ void IMergeTreeDataPart::assertState(const std::initializer_list & indices_to_recalc, const CompressionCodecPtr & default_codec_, - const WriterSettings & writer_settings) const = 0; + const WriterSettings & writer_settings, + const MergeTreeIndexGranularity & computed_index_granularity = {}) const = 0; virtual bool isStoredOnDisk() const = 0; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index f1bd137be6b..90e389e6954 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -1,23 +1,15 @@ #include #include +#include namespace DB { -IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( - const String & part_path_, - const MergeTreeData & storage_, - const NamesAndTypesList & columns_list_, - const String & marks_file_extension_, - const CompressionCodecPtr & default_codec_, - const WriterSettings & settings_) -: part_path(part_path_) -, storage(storage_) -, columns_list(columns_list_) -, marks_file_extension(marks_file_extension_) -, default_codec(default_codec_) -, settings(settings_) {} -IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; +namespace +{ + // constexpr auto DATA_FILE_EXTENSION = ".bin"; + constexpr auto INDEX_FILE_EXTENSION = ".idx"; +} void IMergeTreeDataPartWriter::ColumnStream::finalize() { @@ -65,4 +57,273 @@ void IMergeTreeDataPartWriter::ColumnStream::addToChecksums(MergeTreeData::DataP checksums.files[name + marks_file_extension].file_hash = marks.getHash(); } + +IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( + const String & part_path_, + const MergeTreeData & storage_, + const NamesAndTypesList & columns_list_, + const std::vector & indices_to_recalc_, + const String & marks_file_extension_, + const CompressionCodecPtr & default_codec_, + const WriterSettings & settings_, + const MergeTreeIndexGranularity & index_granularity_) + : part_path(part_path_) + , storage(storage_) + , columns_list(columns_list_) + , skip_indices(indices_to_recalc_) + , marks_file_extension(marks_file_extension_) + , default_codec(default_codec_) + , settings(settings_) + , index_granularity(index_granularity_) + , compute_granularity(index_granularity.empty) + , with_final_mark(storage.getSettings()->write_final_mark && can_use_adaptive_granularity) +{ + if (settings.blocks_are_granules_size && !index_granularity.empty()) + throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); + + Poco::File(part_path).createDirectories(); + initPrimaryIndex(); + initSkipIndices(); +} + +IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; + +void fillIndexGranularityImpl( + const Block & block, + size_t index_granularity_bytes, + size_t fixed_index_granularity_rows, + bool blocks_are_granules, + size_t index_offset, + MergeTreeIndexGranularity & index_granularity, + bool can_use_adaptive_index_granularity) +{ + size_t rows_in_block = block.rows(); + size_t index_granularity_for_block; + if (!can_use_adaptive_index_granularity) + index_granularity_for_block = fixed_index_granularity_rows; + else + { + size_t block_size_in_memory = block.bytes(); + if (blocks_are_granules) + index_granularity_for_block = rows_in_block; + else if (block_size_in_memory >= index_granularity_bytes) + { + size_t granules_in_block = block_size_in_memory / index_granularity_bytes; + index_granularity_for_block = rows_in_block / granules_in_block; + } + else + { + size_t size_of_row_in_bytes = block_size_in_memory / rows_in_block; + index_granularity_for_block = index_granularity_bytes / size_of_row_in_bytes; + } + } + if (index_granularity_for_block == 0) /// very rare case when index granularity bytes less then single row + index_granularity_for_block = 1; + + /// We should be less or equal than fixed index granularity + index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); + + /// FIXME + for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) + index_granularity.appendMark(index_granularity_for_block); +} + +void IMergeTreeDataPartWriter::fillIndexGranularity(const Block & block) +{ + const auto storage_settings = storage.getSettings(); + fillIndexGranularityImpl( + block, + storage_settings->index_granularity_bytes, + storage_settings->index_granularity, + settings.blocks_are_granules_size, + index_offset, + index_granularity, + settings.can_use_adaptive_granularity); +} + +void IMergeTreeDataPartWriter::initPrimaryIndex() +{ + if (storage.hasPrimaryKey()) + { + index_file_stream = std::make_unique( + part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, O_TRUNC | O_CREAT | O_WRONLY); + index_stream = std::make_unique(*index_file_stream); + } +} + +void IMergeTreeDataPartWriter::initSkipIndices() +{ + for (const auto & index : skip_indices) + { + String stream_name = index->getFileName(); + skip_indices_streams.emplace_back( + std::make_unique( + stream_name, + part_path + stream_name, INDEX_FILE_EXTENSION, + part_path + stream_name, marks_file_extension, + default_codec, settings.max_compress_block_size, + 0, settings.aio_threshold)); + skip_indices_aggregators.push_back(index->createIndexAggregator()); + skip_index_filling.push_back(0); + } +} + +void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & primary_keys_block, size_t rows) +{ + size_t primary_columns_num = primary_keys_block.columns(); + if (index_columns.empty()) + { + index_types = primary_keys_block.getDataTypes(); + index_columns.resize(primary_columns_num); + last_index_row.resize(primary_columns_num); + for (size_t i = 0; i < primary_columns_num; ++i) + index_columns[i] = primary_keys_block.getByPosition(i).column->cloneEmpty(); + } + + /** While filling index (index_columns), disable memory tracker. + * Because memory is allocated here (maybe in context of INSERT query), + * but then freed in completely different place (while merging parts), where query memory_tracker is not available. + * And otherwise it will look like excessively growing memory consumption in context of query. + * (observed in long INSERT SELECTs) + */ + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); + + /// Write index. The index contains Primary Key value for each `index_granularity` row. + for (size_t i = index_offset; i < rows;) + { + if (storage.hasPrimaryKey()) + { + for (size_t j = 0; j < primary_columns_num; ++j) + { + const auto & primary_column = primary_keys_block.getByPosition(j); + index_columns[j]->insertFrom(*primary_column.column, i); + primary_column.type->serializeBinary(*primary_column.column, i, *index_stream); + } + } + + ++current_mark; + if (current_mark < index_granularity.getMarksCount()) + i += index_granularity.getMarkRows(current_mark); + else + break; + } + + /// store last index row to write final mark at the end of column + for (size_t j = 0; j < primary_columns_num; ++j) + { + const IColumn & primary_column = *primary_keys_block.getByPosition(j).column.get(); + primary_column.get(rows - 1, last_index_row[j]); + } +} + +void IMergeTreeDataPartWriter::calculateAndSerializeSkipIndices( + const Block & skip_indexes_block, size_t rows) +{ + size_t skip_index_current_data_mark = 0; + + /// Filling and writing skip indices like in IMergeTreeDataPartWriter::writeColumn + for (size_t i = 0; i < skip_indices.size(); ++i) + { + const auto index = skip_indices[i]; + auto & stream = *skip_indices_streams[i]; + size_t prev_pos = 0; + skip_index_current_data_mark = skip_index_data_mark; + while (prev_pos < rows) + { + UInt64 limit = 0; + if (prev_pos == 0 && index_offset != 0) + { + limit = index_offset; + } + else + { + limit = index_granularity.getMarkRows(skip_index_current_data_mark); + if (skip_indices_aggregators[i]->empty()) + { + skip_indices_aggregators[i] = index->createIndexAggregator(); + skip_index_filling[i] = 0; + + if (stream.compressed.offset() >= settings.min_compress_block_size) + stream.compressed.next(); + + writeIntBinary(stream.plain_hashing.count(), stream.marks); + writeIntBinary(stream.compressed.offset(), stream.marks); + /// Actually this numbers is redundant, but we have to store them + /// to be compatible with normal .mrk2 file format + if (settings.can_use_adaptive_granularity) + writeIntBinary(1UL, stream.marks); + } + /// this mark is aggregated, go to the next one + skip_index_current_data_mark++; + } + + size_t pos = prev_pos; + skip_indices_aggregators[i]->update(skip_indexes_block, &pos, limit); + + if (pos == prev_pos + limit) + { + ++skip_index_filling[i]; + + /// write index if it is filled + if (skip_index_filling[i] == index->granularity) + { + skip_indices_aggregators[i]->getGranuleAndReset()->serializeBinary(stream.compressed); + skip_index_filling[i] = 0; + } + } + prev_pos = pos; + } + } + skip_index_data_mark = skip_index_current_data_mark; +} + +void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums, bool write_final_mark) +{ + if (index_stream) + { + if (write_final_mark) + { + for (size_t j = 0; j < index_columns.size(); ++j) + { + index_columns[j]->insert(last_index_row[j]); + index_types[j]->serializeBinary(last_index_row[j], *index_stream); + } + last_index_row.clear(); + index_granularity.appendMark(0); + } + + index_stream->next(); + checksums.files["primary.idx"].file_size = index_stream->count(); + checksums.files["primary.idx"].file_hash = index_stream->getHash(); + index_stream = nullptr; + } +} + +void IMergeTreeDataPartWriter::finishSkipIndicesSerialization( + MergeTreeData::DataPart::Checksums & checksums) +{ + for (size_t i = 0; i < skip_indices.size(); ++i) + { + auto & stream = *skip_indices_streams[i]; + if (!skip_indices_aggregators[i]->empty()) + skip_indices_aggregators[i]->getGranuleAndReset()->serializeBinary(stream.compressed); + } + + for (auto & stream : skip_indices_streams) + { + stream->finalize(); + stream->addToChecksums(checksums); + } + skip_indices_streams.clear(); + + skip_indices_aggregators.clear(); + skip_index_filling.clear(); +} + +void IMergeTreeDataPartWriter::next() +{ + current_mark = next_mark; + index_offset = next_index_offset; +} + } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 1f89943e32f..71221f11d79 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -9,7 +9,7 @@ #include #include // #include -// #include +#include namespace DB @@ -62,9 +62,11 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, + const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, - const WriterSettings & settings); + const WriterSettings & settings, + const MergeTreeIndexGranularity & index_granularity); virtual MarkWithOffset write( const Block & block, const IColumn::Permutation * permutation, @@ -73,10 +75,21 @@ public: const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) = 0; - virtual void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) = 0; + virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) = 0; virtual ~IMergeTreeDataPartWriter(); + /// Count index_granularity for block and store in `index_granularity` + void fillIndexGranularity(const Block & block); + + void initSkipIndices(); + void initPrimaryIndex(); + void calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows); + void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, size_t rows); + void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums, bool write_final_mark); + void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); + void next(); + protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; using SerializationStates = std::vector; @@ -86,8 +99,38 @@ protected: NamesAndTypesList columns_list; const String marks_file_extension; + MergeTreeIndexGranularity index_granularity; + CompressionCodecPtr default_codec; + bool can_use_adaptive_granularity; + bool blocks_are_granules_size; + bool compute_granularity; + bool with_final_mark; + + size_t current_mark = 0; + size_t index_offset = 0; + + size_t next_mark = 0; + size_t next_index_offset = 0; + + /// Number of mark in data from which skip indices have to start + /// aggregation. I.e. it's data mark number, not skip indices mark. + size_t skip_index_data_mark = 0; + + std::vector skip_indices; + std::vector> skip_indices_streams; + MergeTreeIndexAggregators skip_indices_aggregators; + std::vector skip_index_filling; + + std::unique_ptr index_file_stream; + std::unique_ptr index_stream; + MutableColumns index_columns; + DataTypes index_types; + /// Index columns values from the last row from the last block + /// It's written to index file in the `writeSuffixAndFinalizePart` method + Row last_index_row; + WriterSettings settings; }; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 7f76d6eac88..5d0569b8ca9 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -48,8 +48,6 @@ public: const NamesAndTypesList & getColumns() const { return columns; } - MergeTreeData::DataPartPtr data_part; - size_t getFirstMarkToRead() const { return all_mark_ranges.back().begin; @@ -62,6 +60,8 @@ protected: using LoadFunc = std::function; MarksPtr loadMarks(const String & mrk_path, const LoadFunc & load_func); + MergeTreeData::DataPartPtr data_part; + /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 59ac49122e2..a81921b5cdb 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -41,160 +41,6 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); } -void fillIndexGranularityImpl( - const Block & block, - size_t index_granularity_bytes, - size_t fixed_index_granularity_rows, - bool blocks_are_granules, - size_t index_offset, - MergeTreeIndexGranularity & index_granularity, - bool can_use_adaptive_index_granularity) -{ - size_t rows_in_block = block.rows(); - size_t index_granularity_for_block; - if (!can_use_adaptive_index_granularity) - index_granularity_for_block = fixed_index_granularity_rows; - else - { - size_t block_size_in_memory = block.bytes(); - if (blocks_are_granules) - index_granularity_for_block = rows_in_block; - else if (block_size_in_memory >= index_granularity_bytes) - { - size_t granules_in_block = block_size_in_memory / index_granularity_bytes; - index_granularity_for_block = rows_in_block / granules_in_block; - } - else - { - size_t size_of_row_in_bytes = block_size_in_memory / rows_in_block; - index_granularity_for_block = index_granularity_bytes / size_of_row_in_bytes; - } - } - if (index_granularity_for_block == 0) /// very rare case when index granularity bytes less then single row - index_granularity_for_block = 1; - - /// We should be less or equal than fixed index granularity - index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); - - UNUSED(index_offset); - UNUSED(index_granularity); - /// FIXME - // for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) - // index_granularity.appendMark(index_granularity_for_block); -} - -void IMergedBlockOutputStream::fillIndexGranularity(const Block & block) -{ - const auto storage_settings = storage.getSettings(); - fillIndexGranularityImpl( - block, - storage_settings->index_granularity_bytes, - storage_settings->index_granularity, - blocks_are_granules_size, - index_offset, - index_granularity, - can_use_adaptive_granularity); -} - -void IMergedBlockOutputStream::initSkipIndices() -{ - for (const auto & index : skip_indices) - { - String stream_name = index->getFileName(); - skip_indices_streams.emplace_back( - std::make_unique( - stream_name, - part_path + stream_name, INDEX_FILE_EXTENSION, - part_path + stream_name, marks_file_extension, - codec, writer_settings.max_compress_block_size, - 0, writer_settings.aio_threshold)); - skip_indices_aggregators.push_back(index->createIndexAggregator()); - skip_index_filling.push_back(0); - } -} - -void IMergedBlockOutputStream::calculateAndSerializeSkipIndices( - const Block & skip_indexes_block, size_t rows) -{ - size_t skip_index_current_data_mark = 0; - - /// Filling and writing skip indices like in IMergedBlockOutputStream::writeColumn - for (size_t i = 0; i < skip_indices.size(); ++i) - { - const auto index = skip_indices[i]; - auto & stream = *skip_indices_streams[i]; - size_t prev_pos = 0; - skip_index_current_data_mark = skip_index_data_mark; - while (prev_pos < rows) - { - UInt64 limit = 0; - if (prev_pos == 0 && index_offset != 0) - { - limit = index_offset; - } - else - { - limit = index_granularity.getMarkRows(skip_index_current_data_mark); - if (skip_indices_aggregators[i]->empty()) - { - skip_indices_aggregators[i] = index->createIndexAggregator(); - skip_index_filling[i] = 0; - - if (stream.compressed.offset() >= writer_settings.min_compress_block_size) - stream.compressed.next(); - - writeIntBinary(stream.plain_hashing.count(), stream.marks); - writeIntBinary(stream.compressed.offset(), stream.marks); - /// Actually this numbers is redundant, but we have to store them - /// to be compatible with normal .mrk2 file format - if (can_use_adaptive_granularity) - writeIntBinary(1UL, stream.marks); - } - /// this mark is aggregated, go to the next one - skip_index_current_data_mark++; - } - - size_t pos = prev_pos; - skip_indices_aggregators[i]->update(skip_indexes_block, &pos, limit); - - if (pos == prev_pos + limit) - { - ++skip_index_filling[i]; - - /// write index if it is filled - if (skip_index_filling[i] == index->granularity) - { - skip_indices_aggregators[i]->getGranuleAndReset()->serializeBinary(stream.compressed); - skip_index_filling[i] = 0; - } - } - prev_pos = pos; - } - } - skip_index_data_mark = skip_index_current_data_mark; -} - -void IMergedBlockOutputStream::finishSkipIndicesSerialization( - MergeTreeData::DataPart::Checksums & checksums) -{ - for (size_t i = 0; i < skip_indices.size(); ++i) - { - auto & stream = *skip_indices_streams[i]; - if (!skip_indices_aggregators[i]->empty()) - skip_indices_aggregators[i]->getGranuleAndReset()->serializeBinary(stream.compressed); - } - - for (auto & stream : skip_indices_streams) - { - stream->finalize(); - stream->addToChecksums(checksums); - } - - skip_indices_streams.clear(); - skip_indices_aggregators.clear(); - skip_index_filling.clear(); -} - /// Implementation of IMergedBlockOutputStream::ColumnStream. } diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index fe8ad5cee14..f52a288a784 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -32,20 +32,6 @@ protected: IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); - /// Count index_granularity for block and store in `index_granularity` - void fillIndexGranularity(const Block & block); - - /// Write final mark to the end of column - void writeFinalMark( - const std::string & column_name, - const DataTypePtr column_type, - WrittenOffsetColumns & offset_columns, - bool skip_offsets, - DB::IDataType::SubstreamPath & path); - - void initSkipIndices(); - void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, size_t rows); - void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); protected: const MergeTreeData & storage; @@ -53,29 +39,29 @@ protected: String part_path; /// The offset to the first row of the block for which you want to write the index. - size_t index_offset = 0; + // size_t index_offset = 0; WriterSettings writer_settings; - size_t current_mark = 0; + // size_t current_mark = 0; /// Number of mark in data from which skip indices have to start /// aggregation. I.e. it's data mark number, not skip indices mark. - size_t skip_index_data_mark = 0; + // size_t skip_index_data_mark = 0; - const bool can_use_adaptive_granularity; - const std::string marks_file_extension; - const bool blocks_are_granules_size; + // const bool can_use_adaptive_granularity; + // const std::string marks_file_extension; + // const bool blocks_are_granules_size; - MergeTreeIndexGranularity index_granularity; + // MergeTreeIndexGranularity index_granularity; - const bool compute_granularity; + // const bool compute_granularity; CompressionCodecPtr codec; - std::vector skip_indices; - std::vector> skip_indices_streams; - MergeTreeIndexAggregators skip_indices_aggregators; - std::vector skip_index_filling; + // std::vector skip_indices; + // std::vector> skip_indices_streams; + // MergeTreeIndexAggregators skip_indices_aggregators; + // std::vector skip_index_filling; MergeTreeWriterPtr writer; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 3678b74ef92..da9884f5649 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -989,7 +989,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor in = std::make_shared( std::make_shared(in, data.primary_key_and_skip_indices_expr)); - IMergeTreeDataPart::MinMaxIndex minmax_idx; + IMergeTreeDataPart::MinMaxIndex minmax_idx; + + WriterSettings writer_settings; + auto part_writer = new_data_part->getWriter(all_columns, compression_codec, blocks_are_granules_size, ) MergedBlockOutputStream out(new_data_part, all_columns, compression_codec); @@ -1092,6 +1095,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor new_data_part->index_granularity = source_part->index_granularity; IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; + MergedColumnOnlyOutputStream out( new_data_part, updated_header, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 759f01b31d9..089e1ffd16a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -82,7 +82,8 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( const ValueSizeMap & avg_value_size_hints, const ReadBufferFromFileBase::ProfileCallback & /* profile_callback */) const { - return std::make_unique(shared_from_this(), columns_to_read, uncompressed_cache, + return std::make_unique( + shared_from_this(), columns_to_read, uncompressed_cache, mark_cache, mark_ranges, reader_settings, avg_value_size_hints); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 7ef500cd98b..2f43b722ca5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -158,19 +158,6 @@ String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const return *minimum_size_column; } -UInt64 MergeTreeDataPartWide::calculateTotalSizeOnDisk(const String & from) -{ - Poco::File cur(from); - if (cur.isFile()) - return cur.getSize(); - std::vector files; - cur.list(files); - UInt64 res = 0; - for (const auto & file : files) - res += calculateTotalSizeOnDisk(from + file); - return res; -} - void MergeTreeDataPartWide::loadIndexGranularity() { String full_path = getFullPath(); @@ -226,7 +213,6 @@ void MergeTreeDataPartWide::loadIndexGranularity() // } - // bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDataType & type) const // { // bool res = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 9fd8953dcaa..1e95cf3890a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -60,8 +60,10 @@ public: MergeTreeWriterPtr getWriter( const NamesAndTypesList & columns_list, + const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec_, - const WriterSettings & writer_settings) const override; + const WriterSettings & writer_settings, + const MergeTreeIndexGranularity & computed_index_granularity = {}) const override; bool isStoredOnDisk() const override { return true; } @@ -73,10 +75,6 @@ public: Type getType() const override { return Type::WIDE; } - /// Calculate the total size of the entire directory with all the files - static UInt64 calculateTotalSizeOnDisk(const String & from); - - private: /// Loads marks index granularity into memory void loadIndexGranularity() override; @@ -86,7 +84,6 @@ private: void checkConsistency(bool require_part_metadata); }; - // using MergeTreeDataPartState =IMergeTreeDataPart::State; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index f99dcb1747e..35da3d82261 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -28,7 +28,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( part_path + DATA_FILE_NAME, marks_file_extension, default_codec, settings.max_compress_block_size, - 0, + settings.estimated_size, settings.aio_threshold); } @@ -78,6 +78,7 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( // size_t rows_to_write = std::min(total_rows, index_granularity.getMarkRows(current_mark)); size_t rows_to_write = total_rows; index_granularity.appendMark(total_rows); + // if (current_row == 0 && index_offset != 0) // { // rows_to_write = index_offset; @@ -112,6 +113,7 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( for (size_t i = 0; i < columns_to_write.size(); ++i) next_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); } + current_row = next_row; } @@ -120,10 +122,10 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) { - std::cerr << "(writeColumnSingleGranule) writing column: " << column.name << "\n"; std::cerr << "(writeColumnSingleGranule) from_row: " << from_row << "\n"; std::cerr << "(writeColumnSingleGranule) number_of_rows: " << number_of_rows << "\n"; + IDataType::SerializeBinaryBulkStatePtr state; IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -148,6 +150,8 @@ void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & ch writeIntBinary(stream->plain_hashing.count(), stream->marks); writeIntBinary(stream->compressed.offset(), stream->marks); } + if (compute_granularity) + index_granularity.appendMark(0); } stream->finalize(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index eae2c6ec15e..38d0a55f08f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -22,20 +22,9 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( default_codec_, settings_) , can_use_adaptive_granularity(storage_.canUseAdaptiveGranularity()) { - size_t total_size = 0; - if (settings.aio_threshold > 0 && !merged_column_to_size.empty()) - { - for (const auto & it : columns_list) - { - auto it2 = merged_column_to_size.find(it.name); - if (it2 != merged_column_to_size.end()) - total_size += it2->second; - } - } - const auto & columns = storage.getColumns(); for (const auto & it : columns_list) - addStreams(it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec), total_size, false); + addStreams(it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec), settings.estimated_size, false); } void MergeTreeDataPartWriterWide::addStreams( @@ -100,13 +89,13 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(cons { serialization_states.reserve(columns_list.size()); WrittenOffsetColumns tmp_offset_columns; - IDataType::SerializeBinaryBulkSettings settings; + IDataType::SerializeBinaryBulkSettings serialize_settings; for (const auto & col : columns_list) { - settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); + serialize_settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); serialization_states.emplace_back(nullptr); - col.type->serializeBinaryBulkStatePrefix(settings, serialization_states.back()); + col.type->serializeBinaryBulkStatePrefix(serialize_settings, serialization_states.back()); } } @@ -174,7 +163,7 @@ void MergeTreeDataPartWriterWide::writeSingleMark( writeIntBinary(stream.plain_hashing.count(), stream.marks); writeIntBinary(stream.compressed.offset(), stream.marks); - if (can_use_adaptive_granularity) + if (settings.can_use_adaptive_granularity) writeIntBinary(number_of_rows, stream.marks); }, path); } @@ -290,10 +279,13 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( } }, serialize_settings.path); + next_mark = current_column_mark; + next_index_offset = current_row - total_rows; + return std::make_pair(current_column_mark, current_row - total_rows); } -void MergeTreeDataPartWriterWide::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) +void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) { const auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index b184b3cdab0..eca137cbcb2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -24,7 +24,7 @@ public: const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) override; - void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; + void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 08c59b695d3..15612f074a5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -175,7 +175,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// This will generate unique name in scope of current server process. Int64 temp_index = data.insert_increment.get(); - IMergeTreeDataPart::MinMaxIndex minmax_idx; + IMergeTreeDataPart::MinMaxIndex minmax_idx; minmax_idx.update(block, data.minmax_idx_columns); MergeTreePartition partition(std::move(block_with_partition.partition)); @@ -263,6 +263,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa auto compression_codec = data.global_context.chooseCompressionCodec(0, 0); NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); + MergedBlockOutputStream out(new_data_part, columns, compression_codec); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h index 5859328eef3..0ca5dd1ad01 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h @@ -3,19 +3,38 @@ namespace DB { - struct ReaderSettings +struct ReaderSettings +{ + size_t min_bytes_to_use_direct_io = 0; + size_t max_read_buffer_size = 0; + bool save_marks_in_cache = false; +}; + +struct WriterSettings +{ + WriterSettings(const Settings & settings, bool can_use_adaptive_granularity_, bool blocks_are_granules_size_ = false) + : min_compress_block_size(settings.min_compress_block_size) + , max_compress_block_size(settings.max_compress_block_size) + , aio_threshold(settings.min_bytes_to_use_direct_io) + , can_use_adaptive_granularity(can_use_adaptive_granularity_) + , blocks_are_granules_size(blocks_are_granules_size_) {} + + WriterSettings & setAdaptive(bool value) { - size_t min_bytes_to_use_direct_io = 0; - size_t max_read_buffer_size = 0; - bool save_marks_in_cache = false; - }; - - struct WriterSettings + can_use_adaptive_granularity = value; + } + + WriterSettings & setAioThreshHold(size_t value) { - size_t min_compress_block_size; - size_t max_compress_block_size; - size_t aio_threshold; - // String marks_file_extension; - }; + + } + + size_t min_compress_block_size; + size_t max_compress_block_size; + size_t aio_threshold; + bool can_use_adaptive_granularity; + bool blocks_are_granules_size; + size_t estimated_size = 0; +}; } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index c40c357c22c..a7d6e524a74 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -35,31 +35,43 @@ MergedBlockOutputStream::MergedBlockOutputStream( std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), data_part_->storage.canUseAdaptiveGranularity()) , columns_list(columns_list_) -{ - init(); - writer = data_part_->getWriter(columns_list_, default_codec_, writer_settings); +{ + const auto & global_settings = data_part->storage.global_context.getSettings(); + writer = data_part_->getWriter(columns_list_, default_codec_, WriterSettings(global_settings)); } +can_use_adaptive_granularity +index_granularity +skip_indices +blocks_are_granule_size + MergedBlockOutputStream::MergedBlockOutputStream( const MergeTreeDataPartPtr & data_part_, const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec_, - const MergeTreeData::DataPart::ColumnToSize & /* merged_column_to_size_ */, // FIXME - size_t aio_threshold_, + const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size, + size_t aio_threshold, bool blocks_are_granules_size_) : IMergedBlockOutputStream( data_part_, default_codec_, - { - data_part_->storage.global_context.getSettings().min_compress_block_size, - data_part_->storage.global_context.getSettings().max_compress_block_size, - aio_threshold_ - }, blocks_are_granules_size_, std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), data_part_->storage.canUseAdaptiveGranularity()) , columns_list(columns_list_) { - init(); + WriterSettings writer_settings(data_part->storage.global_context.getSettings()); + writer_settings.aio_threshold = aio_threshold; + + if (aio_threshold > 0 && !merged_column_to_size.empty()) + { + for (const auto & it : columns_list) + { + auto it2 = merged_column_to_size.find(it.name); + if (it2 != merged_column_to_size.end()) + writer_settings.estimated_size += it2->second; + } + } + writer = data_part_->getWriter(columns_list_, default_codec_, writer_settings); } @@ -101,34 +113,12 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( /// Finish columns serialization. bool write_final_mark = (with_final_mark && rows_count != 0); writer->finalize(checksums, write_final_mark); - - if (write_final_mark) - index_granularity.appendMark(0); /// last mark + writer->finishPrimaryIndexSerialization(checksums, write_final_mark); + writer->finishSkipIndicesSerialization(checksums); if (!total_column_list) total_column_list = &columns_list; - if (index_stream) - { - if (with_final_mark && rows_count != 0) - { - for (size_t j = 0; j < index_columns.size(); ++j) - { - auto & column = *last_index_row[j].column; - index_columns[j]->insertFrom(column, 0); /// it has only one element - last_index_row[j].type->serializeBinary(column, 0, *index_stream); - } - last_index_row.clear(); - } - - index_stream->next(); - checksums.files["primary.idx"].file_size = index_stream->count(); - checksums.files["primary.idx"].file_hash = index_stream->getHash(); - index_stream = nullptr; - } - - finishSkipIndicesSerialization(checksums); - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { new_part->partition.store(storage, part_path, checksums); @@ -179,16 +169,6 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( void MergedBlockOutputStream::init() { - Poco::File(part_path).createDirectories(); - - if (storage.hasPrimaryKey()) - { - index_file_stream = std::make_unique( - part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, O_TRUNC | O_CREAT | O_WRONLY); - index_stream = std::make_unique(*index_file_stream); - } - - initSkipIndices(); } @@ -242,66 +222,14 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } - if (index_columns.empty()) - { - index_columns.resize(primary_key_column_names.size()); - last_index_row.resize(primary_key_column_names.size()); - for (size_t i = 0, size = primary_key_column_names.size(); i < size; ++i) - { - last_index_row[i] = primary_key_block.getByPosition(i).cloneEmpty(); - index_columns[i] = last_index_row[i].column->cloneEmpty(); - } - } + writer->write(block, permutation, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block); + writer->calculateAndSerializeSkipIndices(skip_indexes_block, rows); + writer->calculateAndSerializePrimaryIndex(primary_index_block); + writer->next(); - size_t new_index_offset = writer->write(block, permutation, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block).second; rows_count += rows; - /// Should be written before index offset update, because we calculate, - /// indices of currently written granules - calculateAndSerializeSkipIndices(skip_indexes_block, rows); - - { - /** While filling index (index_columns), disable memory tracker. - * Because memory is allocated here (maybe in context of INSERT query), - * but then freed in completely different place (while merging parts), where query memory_tracker is not available. - * And otherwise it will look like excessively growing memory consumption in context of query. - * (observed in long INSERT SELECTs) - */ - auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - - /// Write index. The index contains Primary Key value for each `index_granularity` row. - for (size_t i = index_offset; i < rows;) - { - if (storage.hasPrimaryKey()) - { - for (size_t j = 0, size = primary_key_block.columns(); j < size; ++j) - { - const auto & primary_column = primary_key_block.getByPosition(j); - index_columns[j]->insertFrom(*primary_column.column, i); - primary_column.type->serializeBinary(*primary_column.column, i, *index_stream); - } - } - - ++current_mark; - if (current_mark < index_granularity.getMarksCount()) - i += index_granularity.getMarkRows(current_mark); - else - break; - } - } - - /// store last index row to write final mark at the end of column - for (size_t j = 0, size = primary_key_block.columns(); j < size; ++j) - { - const IColumn & primary_column = *primary_key_block.getByPosition(j).column.get(); - auto mutable_column = std::move(*last_index_row[j].column).mutate(); - if (!mutable_column->empty()) - mutable_column->popBack(1); - mutable_column->insertFrom(primary_column, rows - 1); - last_index_row[j].column = std::move(mutable_column); - } - - index_offset = new_index_offset; + // index_offset = new_index_offset; } } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h index 0701ceaa8c9..1bacd06ab1c 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -47,11 +47,6 @@ public: const NamesAndTypesList * total_columns_list = nullptr, MergeTreeData::DataPart::Checksums * additional_column_checksums = nullptr); - const MergeTreeIndexGranularity & getIndexGranularity() const - { - return index_granularity; - } - private: void init(); @@ -64,13 +59,6 @@ private: NamesAndTypesList columns_list; size_t rows_count = 0; - - std::unique_ptr index_file_stream; - std::unique_ptr index_stream; - MutableColumns index_columns; - /// Index columns values from the last row from the last block - /// It's written to index file in the `writeSuffixAndFinalizePart` method - ColumnsWithTypeAndName last_index_row; }; } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index d0d0da0c234..1f7b92ff262 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -59,7 +59,8 @@ void MergedColumnOnlyOutputStream::write(const Block & block) UNUSED(skip_offsets); UNUSED(already_written_offset_columns); - auto [new_current_mark, new_index_offset] = writer->write(block, nullptr, current_mark, index_offset, index_granularity); + auto [new_current_mark, new_index_offset] = writer->write(block, nullptr, current_mark, index_offset, + index_granularity, {}, {}, skip_offsets, already_written_offset_columns); /// Should be written before index offset update, because we calculate, /// indices of currently written granules @@ -81,8 +82,6 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG bool write_final_mark = with_final_mark && (index_offset != 0 || current_mark != 0); writer->finalize(checksums, write_final_mark, sync); - finishSkipIndicesSerialization(checksums); - return checksums; } From 8cf62369365d4df9d6325c1b3572dbd4aafa8aa4 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 7 Nov 2019 14:11:38 +0300 Subject: [PATCH 0051/2007] polymorphic parts (development) --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 12 +-- .../MergeTree/IMergeTreeDataPartWriter.h | 27 ++++-- .../src/Storages/MergeTree/IMergeTreeReader.h | 4 +- .../MergeTree/IMergedBlockOutputStream.cpp | 20 +---- .../MergeTree/IMergedBlockOutputStream.h | 18 ++-- .../MergeTree/MergeTreeBlockReadUtils.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.h | 5 ++ .../MergeTree/MergeTreeDataMergerMutator.cpp | 8 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 8 +- .../MergeTree/MergeTreeDataPartCompact.h | 6 +- .../MergeTree/MergeTreeDataPartWide.cpp | 8 +- .../MergeTreeDataPartWriterCompact.cpp | 42 ++++++---- .../MergeTreeDataPartWriterCompact.h | 12 +-- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 83 ++++++++++--------- .../MergeTree/MergeTreeDataPartWriterWide.h | 16 ++-- .../MergeTree/MergeTreeReaderSettings.h | 22 ++--- .../MergeTree/MergedBlockOutputStream.cpp | 63 +++++--------- .../MergedColumnOnlyOutputStream.cpp | 64 +++++++------- .../MergeTree/MergedColumnOnlyOutputStream.h | 3 + 19 files changed, 200 insertions(+), 223 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 90e389e6954..1cd44024e82 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -70,13 +70,13 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( : part_path(part_path_) , storage(storage_) , columns_list(columns_list_) - , skip_indices(indices_to_recalc_) , marks_file_extension(marks_file_extension_) - , default_codec(default_codec_) - , settings(settings_) , index_granularity(index_granularity_) - , compute_granularity(index_granularity.empty) - , with_final_mark(storage.getSettings()->write_final_mark && can_use_adaptive_granularity) + , default_codec(default_codec_) + , skip_indices(indices_to_recalc_) + , settings(settings_) + , compute_granularity(index_granularity.empty()) + , with_final_mark(storage.getSettings()->write_final_mark && settings.can_use_adaptive_granularity) { if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); @@ -314,8 +314,8 @@ void IMergeTreeDataPartWriter::finishSkipIndicesSerialization( stream->finalize(); stream->addToChecksums(checksums); } - skip_indices_streams.clear(); + skip_indices_streams.clear(); skip_indices_aggregators.clear(); skip_index_filling.clear(); } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 71221f11d79..ff01c369737 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -68,12 +68,10 @@ public: const WriterSettings & settings, const MergeTreeIndexGranularity & index_granularity); - virtual MarkWithOffset write( + virtual void write( const Block & block, const IColumn::Permutation * permutation, - size_t from_mark, size_t offset, MergeTreeIndexGranularity & index_granularity, /* Blocks with already sorted index columns */ - const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, - bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) = 0; + const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) = 0; @@ -82,6 +80,14 @@ public: /// Count index_granularity for block and store in `index_granularity` void fillIndexGranularity(const Block & block); + const MergeTreeIndexGranularity & getIndexGranularity() const { return index_granularity; } + + /// FIXME + MutableColumns && getIndexColumns() + { + return std::move(index_columns); + } + void initSkipIndices(); void initPrimaryIndex(); void calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows); @@ -92,7 +98,7 @@ public: protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; - using SerializationStates = std::vector; + using SerializationStates = std::unordered_map; String part_path; const MergeTreeData & storage; @@ -103,8 +109,11 @@ protected: CompressionCodecPtr default_codec; - bool can_use_adaptive_granularity; - bool blocks_are_granules_size; + std::vector skip_indices; + + WriterSettings settings; + + bool compute_granularity; bool with_final_mark; @@ -118,7 +127,6 @@ protected: /// aggregation. I.e. it's data mark number, not skip indices mark. size_t skip_index_data_mark = 0; - std::vector skip_indices; std::vector> skip_indices_streams; MergeTreeIndexAggregators skip_indices_aggregators; std::vector skip_index_filling; @@ -131,7 +139,8 @@ protected: /// It's written to index file in the `writeSuffixAndFinalizePart` method Row last_index_row; - WriterSettings settings; + bool data_written = false; + }; using MergeTreeWriterPtr = std::unique_ptr; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 5d0569b8ca9..95ef82461e0 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -55,13 +55,13 @@ public: using MarksPtr = MarkCache::MappedPtr; + MergeTreeData::DataPartPtr data_part; + protected: using LoadFunc = std::function; MarksPtr loadMarks(const String & mrk_path, const LoadFunc & load_func); - MergeTreeData::DataPartPtr data_part; - /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index a81921b5cdb..245c016cb0f 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -14,31 +14,15 @@ namespace ErrorCodes namespace { // constexpr auto DATA_FILE_EXTENSION = ".bin"; - constexpr auto INDEX_FILE_EXTENSION = ".idx"; + // constexpr auto INDEX_FILE_EXTENSION = ".idx"; } IMergedBlockOutputStream::IMergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part, - CompressionCodecPtr codec_, - const WriterSettings & writer_settings_, - bool blocks_are_granules_size_, - const std::vector & indices_to_recalc, - bool can_use_adaptive_granularity_) + const MergeTreeDataPartPtr & data_part) : storage(data_part->storage) , part_path(data_part->getFullPath()) - , writer_settings(writer_settings_) - , can_use_adaptive_granularity(can_use_adaptive_granularity_) - , marks_file_extension(data_part->getMarksFileExtension()) - , blocks_are_granules_size(blocks_are_granules_size_) - , index_granularity(data_part->index_granularity) - , compute_granularity(index_granularity.empty()) - , codec(std::move(codec_)) - , skip_indices(indices_to_recalc) - , with_final_mark(storage.getSettings()->write_final_mark && can_use_adaptive_granularity) { - if (blocks_are_granules_size && !index_granularity.empty()) - throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); } /// Implementation of IMergedBlockOutputStream::ColumnStream. diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index f52a288a784..8cb7bea9764 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -17,15 +17,15 @@ class IMergedBlockOutputStream : public IBlockOutputStream { public: IMergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part, - CompressionCodecPtr codec_, - const WriterSettings & writer_settings_, - bool blocks_are_granules_size_, - const std::vector & indices_to_recalc, - bool can_use_adaptive_granularity_); + const MergeTreeDataPartPtr & data_part); using WrittenOffsetColumns = std::set; + const MergeTreeIndexGranularity & getIndexGranularity() + { + return writer->getIndexGranularity(); + } + protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; using SerializationStates = std::vector; @@ -35,14 +35,11 @@ protected: protected: const MergeTreeData & storage; - SerializationStates serialization_states; String part_path; /// The offset to the first row of the block for which you want to write the index. // size_t index_offset = 0; - WriterSettings writer_settings; - // size_t current_mark = 0; /// Number of mark in data from which skip indices have to start @@ -56,7 +53,6 @@ protected: // MergeTreeIndexGranularity index_granularity; // const bool compute_granularity; - CompressionCodecPtr codec; // std::vector skip_indices; // std::vector> skip_indices_streams; @@ -65,7 +61,7 @@ protected: MergeTreeWriterPtr writer; - const bool with_final_mark; + // const bool with_final_mark; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index fc3c466f467..077ae84824a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -4,7 +4,7 @@ #include #include #include -#include +// #include namespace DB diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 4f06e651808..d3bad9b3939 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -450,6 +450,11 @@ public: DataPartPtr getPartIfExists(const String & part_name, const DataPartStates & valid_states); DataPartPtr getPartIfExists(const MergeTreePartInfo & part_info, const DataPartStates & valid_states); + std::vector getSkipIndices() const + { + return std::vector(std::begin(skip_indices), std::end(skip_indices)); + } + /// Total size of active parts in bytes. size_t getTotalActiveSizeInBytes() const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index da9884f5649..6b47234edc9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -991,10 +991,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor IMergeTreeDataPart::MinMaxIndex minmax_idx; - WriterSettings writer_settings; - auto part_writer = new_data_part->getWriter(all_columns, compression_codec, blocks_are_granules_size, ) - - MergedBlockOutputStream out(new_data_part, all_columns, compression_codec); + MergedBlockOutputStream out{ + new_data_part, + all_columns, + compression_codec}; in->readPrefix(); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 089e1ffd16a..ebdf4858a95 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -89,13 +89,15 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( const NamesAndTypesList & columns_list, + const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec, - const WriterSettings & writer_settings) const + const WriterSettings & writer_settings, + const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( - getFullPath(), storage, columns_list, + getFullPath(), storage, columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, - default_codec, writer_settings); + default_codec, writer_settings, computed_index_granularity); } /// Takes into account the fact that several columns can e.g. share their .size substreams. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 3fe40f1691d..13226d60705 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -60,8 +60,10 @@ public: MergeTreeWriterPtr getWriter( const NamesAndTypesList & columns_list, - const CompressionCodecPtr & default_codec, - const WriterSettings & writer_settings) const override; + const std::vector & indices_to_recalc, + const CompressionCodecPtr & default_codec_, + const WriterSettings & writer_settings, + const MergeTreeIndexGranularity & computed_index_granularity = {}) const override; bool isStoredOnDisk() const override { return true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 2f43b722ca5..fd07ae1fd56 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -89,13 +89,15 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader( IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const NamesAndTypesList & columns_list, + const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec, - const WriterSettings & writer_settings) const + const WriterSettings & writer_settings, + const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( - getFullPath(), storage, columns_list, + getFullPath(), storage, columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, - default_codec, writer_settings); + default_codec, writer_settings, computed_index_granularity); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 35da3d82261..9e6080d391e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -14,13 +14,15 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, + const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, - const WriterSettings & settings_) + const WriterSettings & settings_, + const MergeTreeIndexGranularity & index_granularity_) : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, - marks_file_extension_, - default_codec_, settings_) + indices_to_recalc_, marks_file_extension_, + default_codec_, settings_, index_granularity_) { stream = std::make_unique( DATA_FILE_NAME, @@ -32,20 +34,20 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( settings.aio_threshold); } -IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( +void MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, - size_t from_mark, size_t index_offset, - MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block, - bool skip_offsets, const WrittenOffsetColumns & already_written_offset_columns) + const Block & primary_key_block, const Block & skip_indexes_block) { - UNUSED(skip_offsets); - UNUSED(already_written_offset_columns); - size_t total_rows = block.rows(); - size_t current_mark = from_mark; + size_t from_mark = current_mark; size_t current_row = 0; + /// Fill index granularity for this block + /// if it's unknown (in case of insert data or horizontal merge, + /// but not in case of vertical merge) + if (compute_granularity) + fillIndexGranularity(block); + ColumnsWithTypeAndName columns_to_write(columns_list.size()); auto it = columns_list.begin(); for (size_t i = 0; i < columns_list.size(); ++i, ++it) @@ -68,8 +70,6 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( std::cerr << "(MergeTreeDataPartWriterCompact::write) total_rows: " << total_rows << "\n"; - UNUSED(index_offset); - while (current_row < total_rows) { std::cerr << "(MergeTreeDataPartWriterCompact::write) current_row: " << current_row << "\n"; @@ -79,6 +79,9 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( size_t rows_to_write = total_rows; index_granularity.appendMark(total_rows); + if (rows_to_write) + data_written = true; + // if (current_row == 0 && index_offset != 0) // { // rows_to_write = index_offset; @@ -106,7 +109,7 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( writeIntBinary(stream->compressed.offset(), stream->marks); next_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); } - ++current_mark; + ++from_mark; } else { @@ -117,7 +120,8 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterCompact::write( current_row = next_row; } - return {current_mark, total_rows - current_row}; + next_mark = from_mark; + next_index_offset = total_rows - current_row; } size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) @@ -140,9 +144,11 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith return from_row + number_of_rows; } -void MergeTreeDataPartWriterCompact::finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) +void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) { - if (write_final_mark) + UNUSED(write_final_mark); + + if (with_final_mark && data_written) { writeIntBinary(0ULL, stream->marks); for (size_t i = 0; i < columns_list.size(); ++i) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 2aab9ca6558..4cb4fc38f75 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -10,16 +10,16 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, + const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, - const WriterSettings & settings); + const WriterSettings & settings, + const MergeTreeIndexGranularity & index_granularity); - MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, - size_t from_mark, size_t index_offset, MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block, - bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) override; + void write(const Block & block, const IColumn::Permutation * permutation, + const Block & primary_key_block, const Block & skip_indexes_block) override; - void finalize(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; + void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; private: /// Write single granule of one column (rows between 2 marks) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 38d0a55f08f..446b5006a4d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -12,14 +12,14 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, + const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const WriterSettings & settings_, - const ColumnToSize & merged_column_to_size) + const MergeTreeIndexGranularity & index_granularity_) : IMergeTreeDataPartWriter(part_path_, - storage_, columns_list_, - marks_file_extension_, - default_codec_, settings_) + storage_, columns_list_, indices_to_recalc_, + marks_file_extension_, default_codec_, settings_, index_granularity_) , can_use_adaptive_granularity(storage_.canUseAdaptiveGranularity()) { const auto & columns = storage.getColumns(); @@ -79,27 +79,31 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( }; } -IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(const Block & block, - const IColumn::Permutation * permutation, size_t from_mark, size_t index_offset, - MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block, const Block & skip_indexes_block, - bool skip_offsets, const WrittenOffsetColumns & already_written_offset_columns) +void MergeTreeDataPartWriterWide::write(const Block & block, + const IColumn::Permutation * permutation, + const Block & primary_key_block, const Block & skip_indexes_block) { - if (serialization_states.empty()) - { - serialization_states.reserve(columns_list.size()); - WrittenOffsetColumns tmp_offset_columns; - IDataType::SerializeBinaryBulkSettings serialize_settings; + // if (serialization_states.empty()) + // { + // serialization_states.reserve(columns_list.size()); + // WrittenOffsetColumns tmp_offset_columns; + // IDataType::SerializeBinaryBulkSettings serialize_settings; - for (const auto & col : columns_list) - { - serialize_settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); - serialization_states.emplace_back(nullptr); - col.type->serializeBinaryBulkStatePrefix(serialize_settings, serialization_states.back()); - } - } + // for (const auto & col : columns_list) + // { + // serialize_settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); + // serialization_states.emplace_back(nullptr); + // col.type->serializeBinaryBulkStatePrefix(serialize_settings, serialization_states.back()); + // } + // } - WrittenOffsetColumns offset_columns = already_written_offset_columns; + /// Fill index granularity for this block + /// if it's unknown (in case of insert data or horizontal merge, + /// but not in case of vertical merge) + if (compute_granularity) + fillIndexGranularity(block); + + WrittenOffsetColumns offset_columns; MarkWithOffset result; auto it = columns_list.begin(); @@ -112,27 +116,25 @@ IMergeTreeDataPartWriter::MarkWithOffset MergeTreeDataPartWriterWide::write(cons if (primary_key_block.has(it->name)) { const auto & primary_column = *primary_key_block.getByName(it->name).column; - result = writeColumn(column.name, *column.type, primary_column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, primary_column, offset_columns); } else if (skip_indexes_block.has(it->name)) { const auto & index_column = *skip_indexes_block.getByName(it->name).column; - result = writeColumn(column.name, *column.type, index_column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, index_column, offset_columns); } else { /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. ColumnPtr permuted_column = column.column->permute(*permutation, 0); - result = writeColumn(column.name, *column.type, *permuted_column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, *permuted_column, offset_columns); } } else { - result = writeColumn(column.name, *column.type, *column.column, index_granularity, offset_columns, skip_offsets, serialization_states[i], from_mark, index_offset); + result = writeColumn(column.name, *column.type, *column.column, offset_columns); } } - - return result; } void MergeTreeDataPartWriterWide::writeSingleMark( @@ -210,16 +212,19 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( const String & name, const IDataType & type, const IColumn & column, - MergeTreeIndexGranularity & index_granularity, WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - size_t from_mark, - size_t index_offset) + bool skip_offsets) { + auto [it, inserted] = serialization_states.emplace(name, nullptr); + if (inserted) + { + IDataType::SerializeBinaryBulkSettings serialize_settings; + serialize_settings.getter = createStreamGetter(name, offset_columns, false); + type.serializeBinaryBulkStatePrefix(serialize_settings, it->second); + } + std::cerr << "(writeColumn) table: " << storage.getTableName() << "\n"; std::cerr << "(writeColumn) column: " << name << "\n"; - std::cerr << "(writeColumn) from_mark: " << from_mark << "\n"; std::cerr << "(writeColumn) index_offset: " << index_offset << "\n"; auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -229,7 +234,7 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( size_t total_rows = column.size(); size_t current_row = 0; - size_t current_column_mark = from_mark; + size_t current_column_mark = current_mark; while (current_row < total_rows) { size_t rows_to_write; @@ -251,13 +256,16 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( rows_to_write = index_granularity.getMarkRows(current_column_mark); } + if (rows_to_write != 0) + data_written = true; + current_row = writeSingleGranule( name, type, column, offset_columns, skip_offsets, - serialization_state, + it->second, serialize_settings, current_row, rows_to_write, @@ -268,7 +276,6 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( current_column_mark++; } - /// Memoize offsets for Nested types, that are already written. They will not be written again for next columns of Nested structure. type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; @@ -300,7 +307,7 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch if (!serialization_states.empty()) { serialize_settings.getter = createStreamGetter(it->name, offset_columns, false); - it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[i]); + it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[it->name]); } if (write_final_mark) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index eca137cbcb2..49914888a67 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -13,16 +13,14 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, + const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, const WriterSettings & settings, - const ColumnToSize & merged_column_to_size = {}); + const MergeTreeIndexGranularity & index_granularity); - MarkWithOffset write(const Block & block, const IColumn::Permutation * permutation, - size_t from_mark, size_t index_offset, - MergeTreeIndexGranularity & index_granularity, - const Block & primary_key_block = {}, const Block & skip_indexes_block = {}, - bool skip_offsets = false, const WrittenOffsetColumns & already_written_offset_columns = {}) override; + void write(const Block & block, const IColumn::Permutation * permutation, + const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) override; void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; @@ -35,12 +33,8 @@ public: const String & name, const IDataType & type, const IColumn & column, - MergeTreeIndexGranularity & index_granularity, WrittenOffsetColumns & offset_columns, - bool skip_offsets, - IDataType::SerializeBinaryBulkStatePtr & serialization_state, - size_t from_mark, - size_t index_offset); + bool skip_offsets = false); private: /// Write single granule of one column (rows between 2 marks) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h index 0ca5dd1ad01..877cfc6b7d9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h @@ -1,8 +1,10 @@ #pragma once -#include +#include +#include namespace DB { + struct ReaderSettings { size_t min_bytes_to_use_direct_io = 0; @@ -12,23 +14,13 @@ struct ReaderSettings struct WriterSettings { - WriterSettings(const Settings & settings, bool can_use_adaptive_granularity_, bool blocks_are_granules_size_ = false) - : min_compress_block_size(settings.min_compress_block_size) - , max_compress_block_size(settings.max_compress_block_size) - , aio_threshold(settings.min_bytes_to_use_direct_io) + WriterSettings(const Settings & global_settings, bool can_use_adaptive_granularity_, bool blocks_are_granules_size_ = false) + : min_compress_block_size(global_settings.min_compress_block_size) + , max_compress_block_size(global_settings.min_compress_block_size) + , aio_threshold(global_settings.min_bytes_to_use_direct_io) , can_use_adaptive_granularity(can_use_adaptive_granularity_) , blocks_are_granules_size(blocks_are_granules_size_) {} - WriterSettings & setAdaptive(bool value) - { - can_use_adaptive_granularity = value; - } - - WriterSettings & setAioThreshHold(size_t value) - { - - } - size_t min_compress_block_size; size_t max_compress_block_size; size_t aio_threshold; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index a7d6e524a74..13cb83e00ab 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -20,46 +20,30 @@ namespace ErrorCodes MergedBlockOutputStream::MergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part_, + const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, - CompressionCodecPtr default_codec_, - bool blocks_are_granules_size_) - : IMergedBlockOutputStream( - data_part_, default_codec_, - { - data_part_->storage.global_context.getSettings().min_compress_block_size, - data_part_->storage.global_context.getSettings().max_compress_block_size, - data_part_->storage.global_context.getSettings().min_bytes_to_use_direct_io - }, - blocks_are_granules_size_, - std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), - data_part_->storage.canUseAdaptiveGranularity()) + CompressionCodecPtr default_codec, + bool blocks_are_granules_size) + : IMergedBlockOutputStream(data_part) , columns_list(columns_list_) { - const auto & global_settings = data_part->storage.global_context.getSettings(); - writer = data_part_->getWriter(columns_list_, default_codec_, WriterSettings(global_settings)); + WriterSettings writer_settings(data_part->storage.global_context.getSettings(), + data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); + writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); } -can_use_adaptive_granularity -index_granularity -skip_indices -blocks_are_granule_size - MergedBlockOutputStream::MergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part_, + const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, - CompressionCodecPtr default_codec_, + CompressionCodecPtr default_codec, const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size, size_t aio_threshold, - bool blocks_are_granules_size_) - : IMergedBlockOutputStream( - data_part_, default_codec_, - blocks_are_granules_size_, - std::vector(std::begin(data_part_->storage.skip_indices), std::end(data_part_->storage.skip_indices)), - data_part_->storage.canUseAdaptiveGranularity()) + bool blocks_are_granules_size) + : IMergedBlockOutputStream(data_part) , columns_list(columns_list_) { - WriterSettings writer_settings(data_part->storage.global_context.getSettings()); + WriterSettings writer_settings(data_part->storage.global_context.getSettings(), + data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); writer_settings.aio_threshold = aio_threshold; if (aio_threshold > 0 && !merged_column_to_size.empty()) @@ -72,7 +56,8 @@ MergedBlockOutputStream::MergedBlockOutputStream( } } - writer = data_part_->getWriter(columns_list_, default_codec_, writer_settings); + writer = data_part->getWriter(columns_list, + data_part->storage.getSkipIndices(), default_codec, writer_settings); } std::string MergedBlockOutputStream::getPartPath() const @@ -111,8 +96,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( checksums = std::move(*additional_column_checksums); /// Finish columns serialization. - bool write_final_mark = (with_final_mark && rows_count != 0); - writer->finalize(checksums, write_final_mark); + bool write_final_mark = true; /// FIXME + writer->finishDataSerialization(checksums, write_final_mark); writer->finishPrimaryIndexSerialization(checksums, write_final_mark); writer->finishSkipIndicesSerialization(checksums); @@ -161,10 +146,12 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); new_part->columns = *total_column_list; + /// FIXME + auto index_columns = writer->getIndexColumns(); new_part->index.assign(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); new_part->checksums = checksums; new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); - new_part->index_granularity = index_granularity; + new_part->index_granularity = writer->getIndexGranularity(); } void MergedBlockOutputStream::init() @@ -179,12 +166,6 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm if (!rows) return; - /// Fill index granularity for this block - /// if it's unknown (in case of insert data or horizontal merge, - /// but not in case of vertical merge) - if (compute_granularity) - fillIndexGranularity(block); - Block primary_key_block; Block skip_indexes_block; @@ -222,9 +203,9 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm } } - writer->write(block, permutation, current_mark, index_offset, index_granularity, primary_key_block, skip_indexes_block); + writer->write(block, permutation, primary_key_block, skip_indexes_block); writer->calculateAndSerializeSkipIndices(skip_indexes_block, rows); - writer->calculateAndSerializePrimaryIndex(primary_index_block); + writer->calculateAndSerializePrimaryIndex(primary_key_block, rows); writer->next(); rows_count += rows; diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 1f7b92ff262..cbee4ed2e05 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -4,40 +4,36 @@ namespace DB { MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( - const MergeTreeDataPartPtr & data_part_, const Block & header_, bool sync_, - CompressionCodecPtr default_codec_, bool skip_offsets_, - const std::vector & indices_to_recalc_, + const MergeTreeDataPartPtr & data_part, const Block & header_, bool sync_, + CompressionCodecPtr default_codec, bool skip_offsets_, + const std::vector & indices_to_recalc, WrittenOffsetColumns & already_written_offset_columns_, - const MergeTreeIndexGranularityInfo * index_granularity_info_) - : IMergedBlockOutputStream( - data_part_, default_codec_, - { - data_part_->storage.global_context.getSettings().min_compress_block_size, - data_part_->storage.global_context.getSettings().max_compress_block_size, - data_part_->storage.global_context.getSettings().min_bytes_to_use_direct_io, - }, - false, - indices_to_recalc_, - index_granularity_info_ ? index_granularity_info_->is_adaptive : data_part_->storage.canUseAdaptiveGranularity()), + const MergeTreeIndexGranularityInfo * index_granularity_info) + : IMergedBlockOutputStream(data_part), header(header_), sync(sync_), skip_offsets(skip_offsets_), already_written_offset_columns(already_written_offset_columns_) { - std::cerr << "(MergedColumnOnlyOutputStream) storage: " << storage.getTableName() << "\n"; - std::cerr << "(MergedColumnOnlyOutputStream) can_use_adaptive_granularity: " << can_use_adaptive_granularity << "\n"; - std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info: " << !!index_granularity_info_ << "\n"; - if (index_granularity_info_) - std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; + // std::cerr << "(MergedColumnOnlyOutputStream) storage: " << storage.getTableName() << "\n"; + // std::cerr << "(MergedColumnOnlyOutputStream) can_use_adaptive_granularity: " << can_use_adaptive_granularity << "\n"; + // std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info: " << !!index_granularity_info_ << "\n"; + // if (index_granularity_info_) + // std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; - writer = data_part_->getWriter(header.getNamesAndTypesList(), default_codec_, writer_settings); + WriterSettings writer_settings(data_part->storage.global_context.getSettings(), false); + if (index_granularity_info && !index_granularity_info->is_adaptive) + writer_settings.can_use_adaptive_granularity = false; + writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, default_codec, writer_settings); + writer_wide = typeid_cast(writer.get()); + if (!writer_wide) + throw Exception("MergedColumnOnlyOutputStream can be used only for writing Wide parts", ErrorCodes::LOGICAL_ERROR); - std::cerr << "(MergedColumnOnlyOutputStream) writer: " << !!writer << "\n"; - initSkipIndices(); + /// FIXME unnessary init of primary idx } void MergedColumnOnlyOutputStream::write(const Block & block) { std::set skip_indexes_column_names_set; - for (const auto & index : skip_indices) + for (const auto & index : storage.skip_indices) /// FIXME save local indices std::copy(index->columns.cbegin(), index->columns.cend(), std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); @@ -56,18 +52,16 @@ void MergedColumnOnlyOutputStream::write(const Block & block) return; /// FIXME skip_offsets - UNUSED(skip_offsets); - UNUSED(already_written_offset_columns); - auto [new_current_mark, new_index_offset] = writer->write(block, nullptr, current_mark, index_offset, - index_granularity, {}, {}, skip_offsets, already_written_offset_columns); + WrittenOffsetColumns offset_columns = already_written_offset_columns; + for (size_t i = 0; i < header.columns(); ++i) + { + const auto & column = block.getByName(header.getByPosition(i).name); + writer_wide->writeColumn(column.name, *column.type, *column.column, offset_columns, skip_offsets); + } - /// Should be written before index offset update, because we calculate, - /// indices of currently written granules - calculateAndSerializeSkipIndices(skip_indexes_columns, rows); - - current_mark = new_current_mark; - index_offset = new_index_offset; + writer_wide->calculateAndSerializeSkipIndices(skip_indexes_columns, rows); + writer_wide->next(); } void MergedColumnOnlyOutputStream::writeSuffix() @@ -79,8 +73,8 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG { /// Finish columns serialization. MergeTreeData::DataPart::Checksums checksums; - bool write_final_mark = with_final_mark && (index_offset != 0 || current_mark != 0); - writer->finalize(checksums, write_final_mark, sync); + bool write_final_mark = true; /// FIXME + writer->finishDataSerialization(checksums, write_final_mark, sync); return checksums; } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index 6030977fc29..b843bcc5d3e 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -2,6 +2,7 @@ #include #include +#include namespace DB { @@ -35,6 +36,8 @@ private: /// To correctly write Nested elements column-by-column. WrittenOffsetColumns & already_written_offset_columns; + + MergeTreeDataPartWriterWide * writer_wide; }; From c0702542797ca4dd31ab3b9fd32a66d9f116c0a7 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Fri, 8 Nov 2019 17:36:10 +0300 Subject: [PATCH 0052/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 3 ++- .../MergeTree/IMergeTreeDataPartWriter.cpp | 23 +++++++++++++++---- .../MergeTree/IMergeTreeDataPartWriter.h | 2 ++ dbms/src/Storages/MergeTree/MergeTreeData.cpp | 1 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 6 ++--- .../MergeTree/MergeTreeDataPartFactory.cpp | 22 +++++++++--------- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 11 +++++++++ .../MergeTree/MergeTreeReaderWide.cpp | 5 +++- .../MergeTree/MergedBlockOutputStream.cpp | 7 ++++++ .../MergedColumnOnlyOutputStream.cpp | 15 +++++++----- .../MergeTree/MergedColumnOnlyOutputStream.h | 1 + 11 files changed, 70 insertions(+), 26 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index a26d5de6200..df36606fd6f 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -364,7 +364,8 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const String IMergeTreeDataPart::getFullPath() const { - assertOnDisk(); + /// FIXME + // assertOnDisk(); if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 1cd44024e82..d1177a7a347 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -80,10 +80,11 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( { if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); + + Poco::File part_dir(part_path); + if (!part_dir.exists()) + part_dir.createDirectories(); - Poco::File(part_path).createDirectories(); - initPrimaryIndex(); - initSkipIndices(); } IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; @@ -149,6 +150,8 @@ void IMergeTreeDataPartWriter::initPrimaryIndex() part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, O_TRUNC | O_CREAT | O_WRONLY); index_stream = std::make_unique(*index_file_stream); } + + primary_index_initialized = true; } void IMergeTreeDataPartWriter::initSkipIndices() @@ -166,10 +169,15 @@ void IMergeTreeDataPartWriter::initSkipIndices() skip_indices_aggregators.push_back(index->createIndexAggregator()); skip_index_filling.push_back(0); } + + skip_indices_initialized = true; } void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & primary_keys_block, size_t rows) { + if (!primary_index_initialized) + throw Exception("Primary index is not initialized", ErrorCodes::LOGICAL_ERROR); + size_t primary_columns_num = primary_keys_block.columns(); if (index_columns.empty()) { @@ -219,6 +227,9 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p void IMergeTreeDataPartWriter::calculateAndSerializeSkipIndices( const Block & skip_indexes_block, size_t rows) { + if (!skip_indices_initialized) + throw Exception("Skip indices are not initialized", ErrorCodes::LOGICAL_ERROR); + size_t skip_index_current_data_mark = 0; /// Filling and writing skip indices like in IMergeTreeDataPartWriter::writeColumn @@ -279,6 +290,7 @@ void IMergeTreeDataPartWriter::calculateAndSerializeSkipIndices( void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums, bool write_final_mark) { + std::cerr << "finishPrimaryIndexSerialization called...\n"; if (index_stream) { if (write_final_mark) @@ -288,10 +300,13 @@ void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::Da index_columns[j]->insert(last_index_row[j]); index_types[j]->serializeBinary(last_index_row[j], *index_stream); } + last_index_row.clear(); - index_granularity.appendMark(0); } + std::cerr << "(finishPrimaryIndexSerialization) marks_count: " << index_granularity.getMarksCount() << "\n"; + std::cerr << "(finishPrimaryIndexSerialization) write_final_mark: " << write_final_mark << "\n"; + index_stream->next(); checksums.files["primary.idx"].file_size = index_stream->count(); checksums.files["primary.idx"].file_hash = index_stream->getHash(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index ff01c369737..9ad18196d10 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -140,6 +140,8 @@ protected: Row last_index_row; bool data_written = false; + bool primary_index_initialized = false; + bool skip_indices_initialized = false; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 0addc3c94a0..c7dda1d0127 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1678,6 +1678,7 @@ void MergeTreeData::alterDataPart( /// Don't recalc indices because indices alter is restricted std::vector{}, unused_written_offsets, + part->index_granularity, &part->index_granularity_info); in.readPrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 6b47234edc9..bb36bc7ef7e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -831,7 +831,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor rows_sources_read_buf.seek(0, 0); ColumnGathererStream column_gathered_stream(column_name, column_part_streams, rows_sources_read_buf); - new_data_part->index_granularity = to.getIndexGranularity(); MergedColumnOnlyOutputStream column_to( new_data_part, column_gathered_stream.getHeader(), @@ -842,7 +841,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// because all of them were already recalculated and written /// as key part of vertical merge std::vector{}, - written_offset_columns); + written_offset_columns, + to.getIndexGranularity()); size_t column_elems_written = 0; @@ -1093,7 +1093,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor merge_entry->columns_written = all_columns.size() - updated_header.columns(); - new_data_part->index_granularity = source_part->index_granularity; IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( @@ -1104,6 +1103,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor /* skip_offsets = */ false, std::vector(indices_to_recalc.begin(), indices_to_recalc.end()), unused_written_offsets, + source_part->index_granularity, &source_part->index_granularity_info ); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp index 15d259379c4..8f16b088ec0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp @@ -8,22 +8,22 @@ namespace DB const MergeTreePartInfo & info, const String & relative_path) { // /// FIXME - size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); + // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + // return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); - // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); - // return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); + return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); } std::shared_ptr createPart(MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, const String & relative_path) { - /// FIXME - size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - return std::make_shared(storage, name, index_granularity_info, disk, relative_path); - // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); - // return std::make_shared(storage, name, index_granularity_info, disk, relative_path); + // /// FIXME + // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); + // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); + // return std::make_shared(storage, name, index_granularity_info, disk, relative_path); + MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); + return std::make_shared(storage, name, index_granularity_info, disk, relative_path); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 446b5006a4d..88100a4c66c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -55,6 +55,8 @@ void MergeTreeDataPartWriterWide::addStreams( settings.aio_threshold); }; + std::cerr << "(addStreams) name: " << name << "\n"; + IDataType::SubstreamPath stream_path; type.enumerateStreams(callback, stream_path); } @@ -102,6 +104,8 @@ void MergeTreeDataPartWriterWide::write(const Block & block, /// but not in case of vertical merge) if (compute_granularity) fillIndexGranularity(block); + + std::cerr << "(MergeTreeDataPartWriterWide::write) marks_count: " << index_granularity.getMarksCount() << "\n"; WrittenOffsetColumns offset_columns; MarkWithOffset result; @@ -226,6 +230,7 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( std::cerr << "(writeColumn) table: " << storage.getTableName() << "\n"; std::cerr << "(writeColumn) column: " << name << "\n"; std::cerr << "(writeColumn) index_offset: " << index_offset << "\n"; + auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.getter = createStreamGetter(name, offset_columns, skip_offsets); @@ -311,10 +316,16 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch } if (write_final_mark) + { writeFinalMark(it->name, it->type, offset_columns, false, serialize_settings.path); + } } } + /// FIXME ?? + if (compute_granularity && write_final_mark && data_written) + index_granularity.appendMark(0); + for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) { it->second->finalize(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index a7ac5d48883..b600b9e3bd2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -156,7 +156,10 @@ void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type */ if (!data_file_exists) return; - + + std::cerr << "(addStreams) part: " << path << '\n'; + std::cerr << "(addStreams) marks count: " << data_part->getMarksCount() << "\n"; + streams.emplace(stream_name, std::make_unique( path + stream_name, DATA_FILE_EXTENSION, data_part->getMarksCount(), all_mark_ranges, settings, mark_cache, diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 13cb83e00ab..39194f1eab6 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -30,6 +30,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( WriterSettings writer_settings(data_part->storage.global_context.getSettings(), data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); + init(); } MergedBlockOutputStream::MergedBlockOutputStream( @@ -58,6 +59,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); + init(); } std::string MergedBlockOutputStream::getPartPath() const @@ -152,10 +154,15 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->checksums = checksums; new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); new_part->index_granularity = writer->getIndexGranularity(); + std::cerr << "(writeSuffixAndFinalizePart) part: " << new_part->getFullPath() << "\n"; + std::cerr << "(writeSuffixAndFinalizePart) marks_count: " << new_part->index_granularity.getMarksCount() << "\n"; } void MergedBlockOutputStream::init() { + Poco::File(part_path).createDirectories(); + writer->initPrimaryIndex(); + writer->initSkipIndices(); } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index cbee4ed2e05..a4181d394f2 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -8,6 +8,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( CompressionCodecPtr default_codec, bool skip_offsets_, const std::vector & indices_to_recalc, WrittenOffsetColumns & already_written_offset_columns_, + const MergeTreeIndexGranularity & index_granularity, const MergeTreeIndexGranularityInfo * index_granularity_info) : IMergedBlockOutputStream(data_part), header(header_), sync(sync_), skip_offsets(skip_offsets_), @@ -19,15 +20,17 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( // if (index_granularity_info_) // std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; - WriterSettings writer_settings(data_part->storage.global_context.getSettings(), false); - if (index_granularity_info && !index_granularity_info->is_adaptive) - writer_settings.can_use_adaptive_granularity = false; - writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, default_codec, writer_settings); + + WriterSettings writer_settings( + data_part->storage.global_context.getSettings(), + index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity()); + + writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, default_codec, writer_settings, index_granularity); writer_wide = typeid_cast(writer.get()); if (!writer_wide) throw Exception("MergedColumnOnlyOutputStream can be used only for writing Wide parts", ErrorCodes::LOGICAL_ERROR); - /// FIXME unnessary init of primary idx + writer->initSkipIndices(); } void MergedColumnOnlyOutputStream::write(const Block & block) @@ -51,7 +54,7 @@ void MergedColumnOnlyOutputStream::write(const Block & block) if (!rows) return; - /// FIXME skip_offsets + std::cerr << "(MergedColumnOnlyOutputStream::write) writing rows: " << rows << "\n"; WrittenOffsetColumns offset_columns = already_written_offset_columns; for (size_t i = 0; i < header.columns(); ++i) diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index b843bcc5d3e..351969f5477 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -21,6 +21,7 @@ public: CompressionCodecPtr default_codec_, bool skip_offsets_, const std::vector & indices_to_recalc_, WrittenOffsetColumns & already_written_offset_columns_, + const MergeTreeIndexGranularity & index_granularity = {}, const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr); Block getHeader() const override { return header; } From f6b1fc53a88e4b1e8589e168b1eb333ead5a50b2 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 13 Nov 2019 04:57:45 +0300 Subject: [PATCH 0053/2007] polymorphic parts (development) --- dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp | 7 ++++++- .../MergeTree/MergeTreeSequentialBlockInputStream.cpp | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index b600b9e3bd2..348d6908847 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -50,6 +50,10 @@ MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) { + std::cerr << "(MergeTreeReaderWide::readRows) columns: " << columns.toString() << "\n"; + std::cerr << "(MergeTreeReaderWide::readRows) from_rows: " << from_mark << "\n"; + std::cerr << "(MergeTreeReaderWide::readRows) block: " << res.dumpStructure() << "\n"; + size_t read_rows = 0; try { @@ -178,7 +182,8 @@ void MergeTreeReaderWide::readData( size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) { - // std::cerr << "(MergeTreeReaderWide::readData) max_rows_to_read: " << max_rows_to_read << "\n"; + std::cerr << "(MergeTreeReaderWide::readData) name: " << name << "\n"; + std::cerr << "(MergeTreeReaderWide::readData) max_rows_to_read: " << max_rows_to_read << "\n"; auto get_stream_getter = [&](bool stream_for_prefix) -> IDataType::InputStreamGetter { return [&, stream_for_prefix](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 6051b1c36d0..1db67ee2e47 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -51,6 +51,10 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( columns_for_reader = data_part->columns.addTypes(columns_to_read); } + std::cerr << "(MergeTreeSequentialBlockInputStream) table: " << storage.getTableName() << "\n"; + std::cerr << "(MergeTreeSequentialBlockInputStream) part: " << data_part_->getFullPath() << "\n"; + std::cerr << "(MergeTreeSequentialBlockInputStream) columns_for_reader: " << columns_for_reader.toString() << "\n"; + ReaderSettings reader_settings = { /// This is hack @@ -119,6 +123,8 @@ try finish(); } + std::cerr << "(MergeTreeSequentialBlockInputStream::readImpl) block: " << res.dumpStructure() << "\n"; + return res; } catch (...) From 6cd6af7ec66429e576cf974d17924b1435d83bcb Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 18 Nov 2019 15:22:27 +0300 Subject: [PATCH 0054/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 24 +++++++++---------- .../Storages/MergeTree/IMergeTreeDataPart.h | 7 ++++++ dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 4 ++-- .../MergeTree/MergeTreeDataPartCompact.cpp | 5 ++++ .../MergeTree/MergeTreeDataPartCompact.h | 2 ++ .../MergeTree/MergeTreeDataPartWide.cpp | 5 ++++ .../MergeTree/MergeTreeDataPartWide.h | 2 ++ .../MergeTree/MergedBlockOutputStream.cpp | 2 +- 9 files changed, 37 insertions(+), 16 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index df36606fd6f..02e3cc8d30d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -203,15 +203,7 @@ String IMergeTreeDataPart::getNewName(const MergeTreePartInfo & new_part_info) c size_t IMergeTreeDataPart::getColumnPosition(const String & column_name) const { - /// FIXME - size_t i = 0; - for (const auto & it : columns) - { - if (it.name == column_name) - return i; - ++i; - } - return -1; + return sample_block.getPositionByName(column_name); } DayNum IMergeTreeDataPart::getMinDate() const @@ -248,7 +240,16 @@ time_t IMergeTreeDataPart::getMaxTime() const return 0; } -IMergeTreeDataPart::~IMergeTreeDataPart() +void IMergeTreeDataPart::setColumns(const NamesAndTypesList & columns_) +{ + columns = columns_; + for (const auto & column : columns) + sample_block.insert({column.type, column.name}); +} + +IMergeTreeDataPart::~IMergeTreeDataPart() = default; + +void IMergeTreeDataPart::removeIfNeeded() { if (state == State::DeleteOnDestroy || is_temp) { @@ -364,8 +365,7 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const String IMergeTreeDataPart::getFullPath() const { - /// FIXME - // assertOnDisk(); + assertOnDisk(); if (relative_path.empty()) throw Exception("Part relative_path cannot be empty. It's bug.", ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 86d2d8b2d47..9e1fa5b413e 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -160,6 +160,8 @@ public: time_t getMinTime() const; time_t getMaxTime() const; + void setColumns(const NamesAndTypesList & columns_); + bool isEmpty() const { return rows_count == 0; } const MergeTreeData & storage; @@ -327,7 +329,12 @@ public: static UInt64 calculateTotalSizeOnDisk(const String & from); +protected: + void removeIfNeeded(); + private: + Block sample_block; + /// Reads columns names and types from columns.txt void loadColumns(bool require); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index c7dda1d0127..94c67a3a746 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1822,7 +1822,7 @@ void MergeTreeData::AlterDataPartTransaction::commit() auto & mutable_part = const_cast(*data_part); mutable_part.checksums = new_checksums; - mutable_part.columns = new_columns; + mutable_part.setColumns(new_columns); /// 3) Delete the old files. for (const auto & from_to : rename_map) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index bb36bc7ef7e..3945d7e4795 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1131,16 +1131,16 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor } /// Write the columns list of the resulting part in the same order as all_columns. - new_data_part->columns = all_columns; Names source_column_names = source_part->columns.getNames(); NameSet source_columns_name_set(source_column_names.begin(), source_column_names.end()); - for (auto it = new_data_part->columns.begin(); it != new_data_part->columns.end();) + for (auto it = all_columns.begin(); it != all_columns.end();) { if (source_columns_name_set.count(it->name) || updated_header.has(it->name)) ++it; else it = new_data_part->columns.erase(it); } + new_data_part->setColumns(all_columns); { /// Write a file with a description of columns. WriteBufferFromFile out_columns(new_part_tmp_path + "columns.txt", 4096); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index ebdf4858a95..62a1801ed04 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -313,6 +313,11 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) } } +MergeTreeDataPartCompact::~MergeTreeDataPartCompact() +{ + removeIfNeeded(); +} + // bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const IDataType & type) const // { // bool res = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 13226d60705..39bf10dfde7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -73,6 +73,8 @@ public: virtual Type getType() const override { return Type::COMPACT; } + ~MergeTreeDataPartCompact() override; + private: /// Loads marks index granularity into memory void loadIndexGranularity() override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index fd07ae1fd56..98abec33ee4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -198,6 +198,11 @@ void MergeTreeDataPartWide::loadIndexGranularity() index_granularity.setInitialized(); } +MergeTreeDataPartWide::~MergeTreeDataPartWide() +{ + removeIfNeeded(); +} + // void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) const // { // std::shared_lock part_lock(columns_lock); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 1e95cf3890a..9370c64701a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -75,6 +75,8 @@ public: Type getType() const override { return Type::WIDE; } + ~MergeTreeDataPartWide() override; + private: /// Loads marks index granularity into memory void loadIndexGranularity() override; diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 39194f1eab6..9d9bccc8069 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -147,7 +147,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); - new_part->columns = *total_column_list; + new_part->setColumns(*total_column_list); /// FIXME auto index_columns = writer->getIndexColumns(); new_part->index.assign(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); From e1d13ea5b05b4c4c001b58b2fa8d0e5c5021ca26 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 18 Nov 2019 18:18:50 +0300 Subject: [PATCH 0055/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 17 ++- .../Storages/MergeTree/IMergeTreeDataPart.h | 10 +- .../MergeTree/IMergeTreeDataPartWriter.cpp | 11 +- .../MergeTree/IMergeTreeDataPartWriter.h | 9 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 20 +-- .../MergeTree/MergeTreeDataPartCompact.cpp | 16 --- .../MergeTree/MergeTreeDataPartCompact.h | 2 + .../MergeTree/MergeTreeDataPartWide.cpp | 132 ++++++++++++++++-- .../MergeTree/MergeTreeDataPartWide.h | 6 +- .../MergeTreeDataPartWriterCompact.cpp | 4 +- .../MergeTreeDataPartWriterCompact.h | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 8 +- .../MergeTree/MergeTreeDataPartWriterWide.h | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 9 +- .../MergedColumnOnlyOutputStream.cpp | 3 +- 15 files changed, 172 insertions(+), 79 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 02e3cc8d30d..670ae2a1069 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -35,6 +35,7 @@ namespace ErrorCodes extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; extern const int BAD_TTL_FILE; extern const int CANNOT_UNLINK; + extern const int NOT_IMPLEMENTED; } @@ -373,7 +374,7 @@ String IMergeTreeDataPart::getFullPath() const return storage.getFullPathOnDisk(disk) + relative_path + "/"; } -void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool /* check_consistency */) +void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency) { assertOnDisk(); @@ -390,9 +391,8 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadPartitionAndMinMaxIndex(); loadTTLInfos(); - /// FIXME - // if (check_consistency) - // checkConsistency(require_columns_checksums); + if (check_consistency) + checkConsistency(require_columns_checksums); } void IMergeTreeDataPart::loadIndexGranularity() @@ -756,6 +756,15 @@ void IMergeTreeDataPart::remove() const } } +void IMergeTreeDataPart::accumulateColumnSizes(ColumnToSize & /* column_to_size */) const +{ + throw Exception("Method 'accumulateColumnSizes' is not supported for data part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMENTED); +} + +void IMergeTreeDataPart::checkConsistency(bool /* require_part_metadata */) const +{ + throw Exception("Method 'checkConsistency' is not supported for data part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMENTED); +} String IMergeTreeDataPart::typeToString(Type type) { diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 9e1fa5b413e..d35c1429f64 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -98,10 +98,7 @@ public: using ColumnToSize = std::map; - // void accumulateColumnSizes(ColumnToSize & column_to_size) const - // { - // throw Exception("Method 'accumulateColumnSizes' is not supported for data part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMETED); - // } + virtual void accumulateColumnSizes(ColumnToSize & column_to_size) const; enum class Type { @@ -148,7 +145,7 @@ public: String getNewName(const MergeTreePartInfo & new_part_info) const; // Block sample_block; - size_t getColumnPosition(const String & name) const; + size_t getColumnPosition(const String & column_name) const; bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } @@ -331,6 +328,7 @@ public: protected: void removeIfNeeded(); + virtual void checkConsistency(bool require_part_metadata) const; private: Block sample_block; @@ -358,8 +356,6 @@ private: String getRelativePathForDetachedPart(const String & prefix) const; - void checkConsistency(bool require_part_metadata); - virtual ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const = 0; }; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index d1177a7a347..562121c578b 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -124,7 +124,7 @@ void fillIndexGranularityImpl( /// We should be less or equal than fixed index granularity index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); - /// FIXME + /// FIXME: split/join last mark for compact parts for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) index_granularity.appendMark(index_granularity_for_block); } @@ -288,12 +288,12 @@ void IMergeTreeDataPartWriter::calculateAndSerializeSkipIndices( skip_index_data_mark = skip_index_current_data_mark; } -void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums, bool write_final_mark) +void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums) { std::cerr << "finishPrimaryIndexSerialization called...\n"; if (index_stream) { - if (write_final_mark) + if (with_final_mark && data_written) { for (size_t j = 0; j < index_columns.size(); ++j) { @@ -301,11 +301,14 @@ void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::Da index_types[j]->serializeBinary(last_index_row[j], *index_stream); } + if (compute_granularity) + index_granularity.appendMark(0); + last_index_row.clear(); } + std::cerr << "(finishPrimaryIndexSerialization) marks_count: " << index_granularity.getMarksCount() << "\n"; - std::cerr << "(finishPrimaryIndexSerialization) write_final_mark: " << write_final_mark << "\n"; index_stream->next(); checksums.files["primary.idx"].file_size = index_stream->count(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 9ad18196d10..2eab44cba40 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -73,7 +73,7 @@ public: /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; - virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) = 0; + virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) = 0; virtual ~IMergeTreeDataPartWriter(); @@ -82,17 +82,16 @@ public: const MergeTreeIndexGranularity & getIndexGranularity() const { return index_granularity; } - /// FIXME - MutableColumns && getIndexColumns() + Columns releaseIndexColumns() { - return std::move(index_columns); + return Columns(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); } void initSkipIndices(); void initPrimaryIndex(); void calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows); void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, size_t rows); - void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums, bool write_final_mark); + void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums); void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); void next(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 3945d7e4795..599a1b846b7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -554,10 +554,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeTreeData::DataPart::ColumnToSize merged_column_to_size; - /// FIXME - // for (const MergeTreeData::DataPartPtr & part : parts) - // part->accumulateColumnSizes(merged_column_to_size); - Names all_column_names = data.getColumns().getNamesOfPhysical(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); const auto data_settings = data.getSettings(); @@ -607,6 +603,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor String rows_sources_file_path; std::unique_ptr rows_sources_uncompressed_write_buf; std::unique_ptr rows_sources_write_buf; + std::optional column_sizes; if (merge_alg == MergeAlgorithm::Vertical) { @@ -614,6 +611,11 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor rows_sources_file_path = new_part_tmp_path + "rows_sources"; rows_sources_uncompressed_write_buf = std::make_unique(rows_sources_file_path); rows_sources_write_buf = std::make_unique(*rows_sources_uncompressed_write_buf); + + for (const MergeTreeData::DataPartPtr & part : parts) + part->accumulateColumnSizes(merged_column_to_size); + + column_sizes = ColumnSizeEstimator(merged_column_to_size, merging_column_names, gathering_column_names); } else { @@ -623,8 +625,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor gathering_column_names.clear(); } - ColumnSizeEstimator column_sizes(merged_column_to_size, merging_column_names, gathering_column_names); - /** Read from all parts, merge and write into a new one. * In passing, we calculate expression for sorting. */ @@ -651,7 +651,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor } MergeStageProgress horizontal_stage_progress( - merge_alg == MergeAlgorithm::Horizontal ? 1.0 : column_sizes.keyColumnsWeight()); + column_sizes ? 1.0 : column_sizes->keyColumnsWeight()); for (const auto & part : parts) { @@ -787,7 +787,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor { size_t sum_input_rows_exact = merge_entry->rows_read; merge_entry->columns_written = merging_column_names.size(); - merge_entry->progress.store(column_sizes.keyColumnsWeight(), std::memory_order_relaxed); + merge_entry->progress.store(column_sizes->keyColumnsWeight(), std::memory_order_relaxed); BlockInputStreams column_part_streams(parts.size()); @@ -816,7 +816,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor Names column_names{column_name}; Float64 progress_before = merge_entry->progress.load(std::memory_order_relaxed); - MergeStageProgress column_progress(progress_before, column_sizes.columnWeight(column_name)); + MergeStageProgress column_progress(progress_before, column_sizes->columnWeight(column_name)); for (size_t part_num = 0; part_num < parts.size(); ++part_num) { auto column_part_stream = std::make_shared( @@ -869,7 +869,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor merge_entry->columns_written += 1; merge_entry->bytes_written_uncompressed += column_gathered_stream.getProfileInfo().bytes; - merge_entry->progress.store(progress_before + column_sizes.columnWeight(column_name), std::memory_order_relaxed); + merge_entry->progress.store(progress_before + column_sizes->columnWeight(column_name), std::memory_order_relaxed); } Poco::File(rows_sources_file_path).remove(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 62a1801ed04..1871500d4e0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -194,22 +194,6 @@ void MergeTreeDataPartCompact::loadIndexGranularity() index_granularity.setInitialized(); } -// void MergeTreeDataPartCompact::accumulateColumnSizes(ColumnToSize & column_to_size) const -// { -// std::shared_lock part_lock(columns_lock); - -// for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical()) -// { -// IDataType::SubstreamPath path; -// name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) -// { -// Poco::File bin_file(getFullPath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"); -// if (bin_file.exists()) -// column_to_size[name_type.name] += bin_file.getSize(); -// }, path); -// } -// } - void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { String path = getFullPath(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 39bf10dfde7..d454cab70b2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -73,6 +73,8 @@ public: virtual Type getType() const override { return Type::COMPACT; } + void checkConsistency(bool /* require_part_metadata */) const override {} + ~MergeTreeDataPartCompact() override; private: diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 98abec33ee4..cb2a9c47333 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -203,21 +203,125 @@ MergeTreeDataPartWide::~MergeTreeDataPartWide() removeIfNeeded(); } -// void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) const -// { -// std::shared_lock part_lock(columns_lock); +void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) const +{ + std::shared_lock part_lock(columns_lock); + + for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical()) + { + IDataType::SubstreamPath path; + name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + Poco::File bin_file(getFullPath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"); + if (bin_file.exists()) + column_to_size[name_type.name] += bin_file.getSize(); + }, path); + } +} + +void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const +{ + String path = getFullPath(); + + if (!checksums.empty()) + { + if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) + throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (require_part_metadata) + { + for (const NameAndTypePair & name_type : columns) + { + IDataType::SubstreamPath stream_path; + name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); + String mrk_file_name = file_name + index_granularity_info.marks_file_extension; + String bin_file_name = file_name + ".bin"; + if (!checksums.files.count(mrk_file_name)) + throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + path, + ErrorCodes::NO_FILE_IN_DATA_PART); + if (!checksums.files.count(bin_file_name)) + throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + path, + ErrorCodes::NO_FILE_IN_DATA_PART); + }, stream_path); + } + } + + if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + if (!checksums.files.count("count.txt")) + throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (storage.partition_key_expr && !checksums.files.count("partition.dat")) + throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (!isEmpty()) + { + for (const String & col_name : storage.minmax_idx_columns) + { + if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) + throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); + } + } + } + + checksums.checkSizes(path); + } + else + { + auto check_file_not_empty = [&path](const String & file_path) + { + Poco::File file(file_path); + if (!file.exists() || file.getSize() == 0) + throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + return file.getSize(); + }; + + /// Check that the primary key index is not empty. + if (!storage.primary_key_columns.empty()) + check_file_not_empty(path + "primary.idx"); + + if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + check_file_not_empty(path + "count.txt"); + + if (storage.partition_key_expr) + check_file_not_empty(path + "partition.dat"); + + for (const String & col_name : storage.minmax_idx_columns) + check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); + } + + /// Check that all marks are nonempty and have the same size. + + std::optional marks_size; + for (const NameAndTypePair & name_type : columns) + { + name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + Poco::File file(IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension); + + /// Missing file is Ok for case when new column was added. + if (file.exists()) + { + UInt64 file_size = file.getSize(); + + if (!file_size) + throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + + if (!marks_size) + marks_size = file_size; + else if (file_size != *marks_size) + throw Exception("Part " + path + " is broken: marks have different sizes.", + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + } + }); + } + } +} -// for (const NameAndTypePair & name_type : storage.getColumns().getAllPhysical()) -// { -// IDataType::SubstreamPath path; -// name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) -// { -// Poco::File bin_file(getFullPath() + IDataType::getFileNameForStream(name_type.name, substream_path) + ".bin"); -// if (bin_file.exists()) -// column_to_size[name_type.name] += bin_file.getSize(); -// }, path); -// } -// } // bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDataType & type) const diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 9370c64701a..fdd574f8278 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -69,6 +69,8 @@ public: bool supportsVerticalMerge() const override { return true; } + void accumulateColumnSizes(ColumnToSize & column_to_size) const override; + /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; @@ -77,13 +79,15 @@ public: ~MergeTreeDataPartWide() override; +protected: + void checkConsistency(bool require_part_metadata) const override; + private: /// Loads marks index granularity into memory void loadIndexGranularity() override; ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const override; - void checkConsistency(bool require_part_metadata); }; // using MergeTreeDataPartState =IMergeTreeDataPart::State; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 9e6080d391e..6d305e99ab7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -144,10 +144,8 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith return from_row + number_of_rows; } -void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) +void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) { - UNUSED(write_final_mark); - if (with_final_mark && data_written) { writeIntBinary(0ULL, stream->marks); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 4cb4fc38f75..efe84182640 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -19,7 +19,7 @@ public: void write(const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) override; - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; + void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) override; private: /// Write single granule of one column (rows between 2 marks) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 88100a4c66c..46e0fee2ac5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -297,7 +297,7 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( return std::make_pair(current_column_mark, current_row - total_rows); } -void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync) +void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) { const auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -305,6 +305,8 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; WrittenOffsetColumns offset_columns; + bool write_final_mark = (with_final_mark && data_written); + { auto it = columns_list.begin(); for (size_t i = 0; i < columns_list.size(); ++i, ++it) @@ -322,10 +324,6 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch } } - /// FIXME ?? - if (compute_granularity && write_final_mark && data_written) - index_granularity.appendMark(0); - for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it) { it->second->finalize(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 49914888a67..21749d97d88 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -22,7 +22,7 @@ public: void write(const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) override; - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool write_final_mark, bool sync = false) override; + void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) override; IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 9d9bccc8069..2cb650c177d 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -98,9 +98,8 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( checksums = std::move(*additional_column_checksums); /// Finish columns serialization. - bool write_final_mark = true; /// FIXME - writer->finishDataSerialization(checksums, write_final_mark); - writer->finishPrimaryIndexSerialization(checksums, write_final_mark); + writer->finishDataSerialization(checksums); + writer->finishPrimaryIndexSerialization(checksums); writer->finishSkipIndicesSerialization(checksums); if (!total_column_list) @@ -148,9 +147,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); new_part->setColumns(*total_column_list); - /// FIXME - auto index_columns = writer->getIndexColumns(); - new_part->index.assign(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); + new_part->index = writer->releaseIndexColumns(); new_part->checksums = checksums; new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); new_part->index_granularity = writer->getIndexGranularity(); diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index a4181d394f2..29a659193ca 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -76,8 +76,7 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG { /// Finish columns serialization. MergeTreeData::DataPart::Checksums checksums; - bool write_final_mark = true; /// FIXME - writer->finishDataSerialization(checksums, write_final_mark, sync); + writer->finishDataSerialization(checksums, sync); return checksums; } From 426c62aafefb41780a71f8721dd960dd9755d201 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 20 Nov 2019 16:33:41 +0300 Subject: [PATCH 0056/2007] polymorphic parts (development) --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 10 ++-- .../Storages/MergeTree/IMergeTreeReader.cpp | 26 ---------- .../src/Storages/MergeTree/IMergeTreeReader.h | 5 -- .../MergeTreeDataPartWriterCompact.cpp | 2 - .../MergeTree/MergeTreeMarksLoader.cpp | 51 +++++++++++++++++++ .../Storages/MergeTree/MergeTreeMarksLoader.h | 35 +++++++++++++ .../MergeTree/MergeTreeReaderCompact.cpp | 26 +++------- .../MergeTree/MergeTreeReaderCompact.h | 36 +------------ .../MergeTree/MergeTreeReaderStream.cpp | 45 +++++----------- .../MergeTree/MergeTreeReaderStream.h | 5 +- 10 files changed, 118 insertions(+), 123 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp create mode 100644 dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 562121c578b..9b70f4e6cef 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -291,9 +291,14 @@ void IMergeTreeDataPartWriter::calculateAndSerializeSkipIndices( void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums) { std::cerr << "finishPrimaryIndexSerialization called...\n"; + + bool write_final_mark = (with_final_mark && data_written); + if (write_final_mark && compute_granularity) + index_granularity.appendMark(0); + if (index_stream) { - if (with_final_mark && data_written) + if (write_final_mark) { for (size_t j = 0; j < index_columns.size(); ++j) { @@ -301,9 +306,6 @@ void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::Da index_types[j]->serializeBinary(last_index_row[j], *index_stream); } - if (compute_granularity) - index_granularity.appendMark(0); - last_index_row.clear(); } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp index 5c585b7ce60..75b01a22081 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -65,32 +65,6 @@ static bool arrayHasNoElementsRead(const IColumn & column) return last_offset != 0; } -IMergeTreeReader::MarksPtr IMergeTreeReader::loadMarks(const String & mrk_path, const LoadFunc & load_func) -{ - MarksPtr marks; - if (mark_cache) - { - auto key = mark_cache->hash(mrk_path); - if (settings.save_marks_in_cache) - { - marks = mark_cache->getOrSet(key, load_func); - } - else - { - marks = mark_cache->get(key); - if (!marks) - marks = load_func(); - } - } - else - marks = load_func(); - - if (!marks) - throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); - - return marks; -} - void IMergeTreeReader::fillMissingColumns(Block & res, bool & should_reorder, bool & should_evaluate_missing_defaults, size_t num_rows) { diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 95ef82461e0..92b5dcbae98 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -53,15 +53,10 @@ public: return all_mark_ranges.back().begin; } - using MarksPtr = MarkCache::MappedPtr; - MergeTreeData::DataPartPtr data_part; protected: - using LoadFunc = std::function; - MarksPtr loadMarks(const String & mrk_path, const LoadFunc & load_func); - /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 6d305e99ab7..8a97ca0a658 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -154,8 +154,6 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: writeIntBinary(stream->plain_hashing.count(), stream->marks); writeIntBinary(stream->compressed.offset(), stream->marks); } - if (compute_granularity) - index_granularity.appendMark(0); } stream->finalize(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp new file mode 100644 index 00000000000..81baabda7df --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -0,0 +1,51 @@ +#include + +namespace DB +{ + +MergeTreeMarksLoader::MergeTreeMarksLoader( + MarkCache * mark_cache_, + const String & mrk_path_, + const LoadFunc & load_func_, + bool save_marks_in_cache_, + size_t columns_num_) + : mark_cache(mark_cache_) + , mrk_path(mrk_path_) + , load_func(load_func_) + , save_marks_in_cache(save_marks_in_cache_) + , columns_num(columns_num_) {} + +const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, size_t column_index) +{ + if (!marks) + loadMarks(); + if (column_index >= columns_num) + throw Exception("", ErrorCodes::LOGICAL_ERROR); + + return (*marks)[row_index * columns_num + column_index]; +} + +void MergeTreeMarksLoader::loadMarks() +{ + if (mark_cache) + { + auto key = mark_cache->hash(mrk_path); + if (save_marks_in_cache) + { + marks = mark_cache->getOrSet(key, load_func); + } + else + { + marks = mark_cache->get(key); + if (!marks) + marks = load_func(); + } + } + else + marks = load_func(); + + if (!marks) + throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); +} + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h new file mode 100644 index 00000000000..8c3bc1d5dff --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -0,0 +1,35 @@ +#include + +namespace DB +{ + +class MergeTreeMarksLoader +{ +public: + using MarksPtr = MarkCache::MappedPtr; + using LoadFunc = std::function; + + MergeTreeMarksLoader() {} + + MergeTreeMarksLoader(MarkCache * mark_cache_, + const String & mrk_path_, + const LoadFunc & load_func_, + bool save_marks_in_cache_, + size_t columns_num_ = 1); + + const MarkInCompressedFile & getMark(size_t row_index, size_t column_index = 0); + + bool initialized() const { return marks != nullptr; } + +private: + MarkCache * mark_cache = nullptr; + String mrk_path; + LoadFunc load_func; + bool save_marks_in_cache = false; + size_t columns_num; + MarksPtr marks; + + void loadMarks(); +}; + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 7abfa76a88d..226360a5d15 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -19,6 +19,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr , uncompressed_cache_, mark_cache_, mark_ranges_ , settings_, avg_value_size_hints_) { + initMarksLoader(); size_t buffer_size = settings.max_read_buffer_size; if (uncompressed_cache) @@ -121,13 +122,14 @@ void MergeTreeReaderCompact::readData( } -void MergeTreeReaderCompact::loadMarks() +void MergeTreeReaderCompact::initMarksLoader() { const auto & index_granularity_info = data_part->index_granularity_info; size_t marks_count = data_part->getMarksCount(); std::string mrk_path = index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); + size_t columns_num = data_part->columns.size(); - auto load_func = [&]() -> MarkCache::MappedPtr + auto load = [&]() -> MarkCache::MappedPtr { size_t file_size = Poco::File(mrk_path).getSize(); @@ -140,7 +142,6 @@ void MergeTreeReaderCompact::loadMarks() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - size_t columns_num = data_part->columns.size(); auto res = std::make_shared(marks_count * columns_num); @@ -168,25 +169,14 @@ void MergeTreeReaderCompact::loadMarks() return res; }; - std::cerr << "(MergeTreeReaderCompact::loadMarks) table: " << storage.getTableName() << ", part: " << path << "\n"; - std::cerr << "(MergeTreeReaderCompact::loadMarks) start marks load..." << "\n"; - - auto marks_array = IMergeTreeReader::loadMarks(mrk_path, load_func); - marks = MarksInCompressedFileCompact(marks_array, columns.size()); + marks_loader = MergeTreeMarksLoader{mark_cache, mrk_path, load, settings.save_marks_in_cache, columns_num}; std::cerr << "(MergeTreeReaderCompact::loadMarks) end marks load..." << "\n"; } -const MarkInCompressedFile & MergeTreeReaderCompact::getMark(size_t row, size_t col) +void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) { - if (!marks.initialized()) - loadMarks(); - return marks.getMark(row, col); -} - -void MergeTreeReaderCompact::seekToMark(size_t row, size_t col) -{ - MarkInCompressedFile mark = getMark(row, col); + MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); std::cerr << "(MergeTreeReaderCompact::seekToMark) mark: (" << mark.offset_in_compressed_file << ", " << mark.offset_in_decompressed_block << "\n"; @@ -201,7 +191,7 @@ void MergeTreeReaderCompact::seekToMark(size_t row, size_t col) { /// Better diagnostics. if (e.code() == ErrorCodes::ARGUMENT_OUT_OF_BOUND) - e.addMessage("(while seeking to mark (" + toString(row) + ", " + toString(col) + ")"); + e.addMessage("(while seeking to mark (" + toString(row_index) + ", " + toString(column_index) + ")"); throw; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index f9d78fb22ab..49bc9e25c38 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -8,38 +8,6 @@ namespace DB { -class MarksInCompressedFileCompact -{ -public: - using MarksPtr = MarkCache::MappedPtr; - - MarksInCompressedFileCompact() = default; - - MarksInCompressedFileCompact(const MarksPtr & data_, size_t columns_num_) - : data(data_), columns_num(columns_num_) {} - - const MarkInCompressedFile & getMark(size_t index, size_t column) const - { - return (*data)[index * columns_num + column]; - } - - char * getRowAddress(size_t index) const - { - return reinterpret_cast(data->data() + index * columns_num); - } - - size_t getRowSize() const - { - return sizeof(MarkInCompressedFile) * columns_num; - } - - bool initialized() { return data != nullptr; } - -private: - MarksPtr data; - size_t columns_num; -}; - /// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. /// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. /// Avoids loading the marks file if it is not needed (e.g. when reading the whole part). @@ -63,9 +31,9 @@ private: std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; - MarksInCompressedFileCompact marks; + MergeTreeMarksLoader marks_loader; - void loadMarks(); + void initMarksLoader(); void seekToStart(); void seekToMark(size_t row, size_t col); const MarkInCompressedFile & getMark(size_t row, size_t col); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index f7f158f7d3e..2b0314645b3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -32,6 +32,8 @@ MergeTreeReaderStream::MergeTreeReaderStream( /// Care should be taken to not load marks when the part is empty (marks_count == 0). + initMarksLoader(); + for (const auto & mark_range : all_mark_ranges) { size_t left_mark = mark_range.begin; @@ -41,10 +43,10 @@ MergeTreeReaderStream::MergeTreeReaderStream( /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. /// If the end of range is inside the block, we will need to read it too. - if (right_mark < marks_count && getMark(right_mark).offset_in_decompressed_block > 0) + if (right_mark < marks_count && marks_loader.getMark(right_mark).offset_in_decompressed_block > 0) { while (right_mark < marks_count - && getMark(right_mark).offset_in_compressed_file == getMark(mark_range.end).offset_in_compressed_file) + && marks_loader.getMark(right_mark).offset_in_compressed_file == marks_loader.getMark(mark_range.end).offset_in_compressed_file) { ++right_mark; } @@ -55,13 +57,13 @@ MergeTreeReaderStream::MergeTreeReaderStream( /// If there are no marks after the end of range, just use file size if (right_mark >= marks_count || (right_mark + 1 == marks_count - && getMark(right_mark).offset_in_compressed_file == getMark(mark_range.end).offset_in_compressed_file)) + && marks_loader.getMark(right_mark).offset_in_compressed_file == marks_loader.getMark(mark_range.end).offset_in_compressed_file)) { - mark_range_bytes = file_size - (left_mark < marks_count ? getMark(left_mark).offset_in_compressed_file : 0); + mark_range_bytes = file_size - (left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0); } else { - mark_range_bytes = getMark(right_mark).offset_in_compressed_file - getMark(left_mark).offset_in_compressed_file; + mark_range_bytes = marks_loader.getMark(right_mark).offset_in_compressed_file - marks_loader.getMark(left_mark).offset_in_compressed_file; } max_mark_range_bytes = std::max(max_mark_range_bytes, mark_range_bytes); @@ -101,16 +103,11 @@ MergeTreeReaderStream::MergeTreeReaderStream( } -const MarkInCompressedFile & MergeTreeReaderStream::getMark(size_t index) +void MergeTreeReaderStream::initMarksLoader() { - if (!marks) - loadMarks(); - return (*marks)[index]; -} + if (marks_loader.initialized()) + return; - -void MergeTreeReaderStream::loadMarks() -{ std::string mrk_path = index_granularity_info->getMarksFilePath(path_prefix); auto load = [&]() -> MarkCache::MappedPtr @@ -153,31 +150,13 @@ void MergeTreeReaderStream::loadMarks() return res; }; - if (mark_cache) - { - auto key = mark_cache->hash(mrk_path); - if (save_marks_in_cache) - { - marks = mark_cache->getOrSet(key, load); - } - else - { - marks = mark_cache->get(key); - if (!marks) - marks = load(); - } - } - else - marks = load(); - - if (!marks) - throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); + marks_loader = MergeTreeMarksLoader{mark_cache, mrk_path, load, save_marks_in_cache}; } void MergeTreeReaderStream::seekToMark(size_t index) { - MarkInCompressedFile mark = getMark(index); + MarkInCompressedFile mark = marks_loader.getMark(index); try { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index 5476b37c866..818c385de91 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -33,7 +34,7 @@ private: /// NOTE: lazily loads marks from the marks cache. const MarkInCompressedFile & getMark(size_t index); - void loadMarks(); + void initMarksLoader(); std::string path_prefix; std::string data_file_extension; @@ -48,5 +49,7 @@ private: std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; + + MergeTreeMarksLoader marks_loader; }; } From 435060146b3df249ed34f351390b906b3e658fa0 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 20 Nov 2019 16:37:31 +0300 Subject: [PATCH 0057/2007] polymorphic parts (development) --- dbms/src/Storages/MergeTree/MergeTreeReaderStream.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index 818c385de91..d7961e38d3d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -31,9 +31,6 @@ public: ReadBuffer * data_buffer; private: - /// NOTE: lazily loads marks from the marks cache. - const MarkInCompressedFile & getMark(size_t index); - void initMarksLoader(); std::string path_prefix; @@ -43,7 +40,6 @@ private: MarkCache * mark_cache; bool save_marks_in_cache; - MarkCache::MappedPtr marks; const MergeTreeIndexGranularityInfo * index_granularity_info; From 94abf3691c03bc91be8fe65f042868fa8424586e Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 21 Nov 2019 19:10:22 +0300 Subject: [PATCH 0058/2007] polymorphic parts (development) --- .../Storages/MergeTree/DataPartsExchange.cpp | 4 +- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 7 +- .../Storages/MergeTree/IMergeTreeDataPart.h | 11 +- .../MergeTree/IMergeTreeDataPart_fwd.h | 22 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 2 + dbms/src/Storages/MergeTree/MergeTreeData.cpp | 112 ++++++++- dbms/src/Storages/MergeTree/MergeTreeData.h | 28 +++ .../MergeTree/MergeTreeDataMergerMutator.cpp | 24 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 215 +++++++++--------- .../MergeTree/MergeTreeDataPartCompact.h | 2 - .../MergeTree/MergeTreeDataPartFactory.cpp | 29 --- .../MergeTree/MergeTreeDataPartFactory.h | 10 - .../MergeTree/MergeTreeDataPartWide.cpp | 7 +- .../MergeTree/MergeTreeDataPartWide.h | 2 - .../MergeTree/MergeTreeDataWriter.cpp | 8 +- .../MergeTreeIndexGranularityInfo.cpp | 47 ++-- .../MergeTree/MergeTreeIndexGranularityInfo.h | 61 +++-- .../MergeTree/MergeTreeMarksLoader.cpp | 2 +- .../MergeTree/MergeTreePartsMover.cpp | 3 +- .../MergeTree/MergeTreeReaderCompact.cpp | 3 + .../Storages/MergeTree/MergeTreeSettings.h | 5 + 21 files changed, 374 insertions(+), 230 deletions(-) delete mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp delete mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 6cff1dfe9f5..fa1d684c640 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -269,10 +268,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( part_file.createDirectory(); - MergeTreeData::MutableDataPartPtr new_data_part = createPart(data, reservation->getDisk(), part_name, relative_part_path); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, reservation->getDisk(),relative_part_path); new_data_part->is_temp = true; - MergeTreeData::DataPart::Checksums checksums; for (size_t i = 0; i < files; ++i) { diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 670ae2a1069..f5a8588a6f7 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -138,13 +138,11 @@ void IMergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other) IMergeTreeDataPart::IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) : storage(storage_) , name(name_) , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) - , index_granularity_info(index_granularity_info_) , disk(disk_) , relative_path(relative_path_.value_or(name_)) { @@ -154,13 +152,11 @@ IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) : storage(storage_) , name(name_) , info(info_) - , index_granularity_info(index_granularity_info_) , disk(disk_) , relative_path(relative_path_.value_or(name_)) { @@ -246,6 +242,7 @@ void IMergeTreeDataPart::setColumns(const NamesAndTypesList & columns_) columns = columns_; for (const auto & column : columns) sample_block.insert({column.type, column.name}); + index_granularity_info.initialize(storage, getType(), columns.size()); } IMergeTreeDataPart::~IMergeTreeDataPart() = default; @@ -776,6 +773,8 @@ String IMergeTreeDataPart::typeToString(Type type) return "Striped"; case Type::IN_MEMORY: return "InMemory"; + case Type::UNKNOWN: + return "Unknown"; } __builtin_unreachable(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index d35c1429f64..a6af875a334 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -85,7 +85,7 @@ public: /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. - virtual String getColumnNameWithMinumumCompressedSize() const = 0; + virtual String getColumnNameWithMinumumCompressedSize() const { return columns.front().name; } // virtual Checksums check( // bool require_checksums, @@ -100,12 +100,7 @@ public: virtual void accumulateColumnSizes(ColumnToSize & column_to_size) const; - enum class Type - { - WIDE, - COMPACT, - IN_MEMORY, - }; + using Type = MergeTreeDataPartType; virtual Type getType() const = 0; @@ -121,14 +116,12 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk = {}, const std::optional & relative_path = {}); IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk = {}, const std::optional & relative_path = {}); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h index c5e10608e0f..6812357b196 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h @@ -3,12 +3,20 @@ namespace DB { - class IMergeTreeDataPart; - class IMergeTreeReader; - class IMergeTreeWriter; + // class IMergeTreeDataPart; + // class IMergeTreeReader; + // class IMergeTreeWriter; - using MergeTreeMutableDataPartPtr = std::shared_ptr; - using MergeTreeDataPartPtr = std::shared_ptr; - using MergeTreeReaderPtr = std::unique_ptr; - using MergeTreeWriterPtr = std::unique_ptr; + // using MergeTreeMutableDataPartPtr = std::shared_ptr; + // using MergeTreeDataPartPtr = std::shared_ptr; + // using MergeTreeReaderPtr = std::unique_ptr; + // using MergeTreeWriterPtr = std::unique_ptr; + + enum class MergeTreeDataPartType + { + WIDE, + COMPACT, + IN_MEMORY, + UNKNOWN, + }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 7dc9a40e89a..418b20e25c5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -8,6 +8,8 @@ namespace DB { + +/// FIXME: implement for compact parts NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData::DataPartPtr & part, Names & columns) { NameSet required_columns{std::begin(columns), std::end(columns)}; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 94c67a3a746..e0cc0990aee 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -4,8 +4,9 @@ #include #include #include +#include +#include #include -#include #include #include #include @@ -817,8 +818,8 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) MergeTreePartInfo part_info; if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return; - - MutableDataPartPtr part = createPart(*this, part_disk_ptr, part_name, part_info, part_name); + + auto part = createPart(part_name, part_info, part_disk_ptr, part_name); bool broken = false; try @@ -1411,6 +1412,8 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c getIndices().indices, new_indices.indices, unused_expression, unused_map, unused_bool); } + +/// FIXME implement alter for compact parts void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, const IndicesASTs & old_indices, const IndicesASTs & new_indices, ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const @@ -1423,7 +1426,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name if (part) part_mrk_file_extension = part->index_granularity_info.marks_file_extension; else - part_mrk_file_extension = settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(); + part_mrk_file_extension = settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE); /// FIXME support compact parts using NameToType = std::map; NameToType new_types; @@ -1575,6 +1578,99 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name } } +/// FIXME implement createPart without columns and with loadMetadata + +MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_on_disk, size_t rows_count) const +{ + const auto settings = getSettings(); + if (bytes_on_disk < settings->min_bytes_for_wide_part || rows_count < settings->min_rows_for_wide_part) + return MergeTreeDataPartType::COMPACT; + + return MergeTreeDataPartType::WIDE; +} + +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, + const DiskSpace::DiskPtr & disk, const String & relative_path) const +{ + return createPart(name, type, MergeTreePartInfo::fromPartName(name, format_version), disk, relative_path); +} + +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, + const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, const String & relative_path) const +{ + if (type == MergeTreeDataPartType::COMPACT) + return std::make_shared(*this, name, part_info, disk, relative_path); + else if (type == MergeTreeDataPartType::WIDE) + return std::make_shared(*this, name, part_info, disk, relative_path); + else + throw Exception("Unknown part type", ErrorCodes::LOGICAL_ERROR); +} + +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, + const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, + size_t bytes_on_disk, size_t rows_num, const String & relative_path) const +{ + return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), + disk, columns, bytes_on_disk, rows_num, relative_path); +} + +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, + size_t bytes_on_disk, size_t rows_count, const String & relative_path) const +{ + auto part = createPart(name, choosePartType(bytes_on_disk, rows_count), part_info, disk, relative_path); + + part->setColumns(columns); + part->bytes_on_disk = bytes_on_disk; + part->rows_count = rows_count; + + return part; +} + +static MergeTreeDataPartType getPartTypeFromMarkExtension(const String & mrk_ext) +{ + if (mrk_ext == getNonAdaptiveMrkExtension()) + return MergeTreeDataPartType::WIDE; + if (mrk_ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE)) + return MergeTreeDataPartType::WIDE; + if (mrk_ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) + return MergeTreeDataPartType::COMPACT; + + return MergeTreeDataPartType::UNKNOWN; +} + +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, + const DiskSpace::DiskPtr & disk, const String & relative_path) const +{ + return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), disk, relative_path); +} + +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, const String & relative_path) const +{ + auto type = MergeTreeDataPartType::UNKNOWN; + auto full_path = getFullPathOnDisk(disk) + relative_path + "/"; + auto mrk_ext = MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(full_path); + if (mrk_ext) + type = getPartTypeFromMarkExtension(*mrk_ext); + else + /// Didn't find any mark file, suppose that part is empty. + type = choosePartType(0, 0); + + MutableDataPartPtr part; + + /// FIXME do not pass emty granularity_info + if (type == MergeTreeDataPartType::COMPACT) + part = std::make_shared(*this, name, part_info, disk, relative_path); + else if (type == MergeTreeDataPartType::WIDE) + part = std::make_shared(*this, name, part_info, disk, relative_path); + else + throw Exception("Unknown part type", ErrorCodes::LOGICAL_ERROR); + + return part; +} + void MergeTreeData::alterDataPart( const NamesAndTypesList & new_columns, const IndicesASTs & new_indices, @@ -2589,7 +2685,7 @@ MergeTreeData::DataPartPtr MergeTreeData::getPartIfExists(const String & part_na MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const DiskSpace::DiskPtr & disk, const String & relative_path) { - MutableDataPartPtr part = createPart(*this, disk, Poco::Path(relative_path).getFileName(), relative_path); + MutableDataPartPtr part = createPart(Poco::Path(relative_path).getFileName(), disk, relative_path); loadPartAndFixMetadata(part); return part; } @@ -3027,8 +3123,7 @@ MergeTreeData::MutableDataPartsVector MergeTreeData::tryLoadPartsToAttach(const for (const auto & part_names : renamed_parts.old_and_new_names) { LOG_DEBUG(log, "Checking part " << part_names.second); - MutableDataPartPtr part = createPart( - *this, name_to_disk[part_names.first], part_names.first, source_dir + part_names.second); + MutableDataPartPtr part = createPart(part_names.first, name_to_disk[part_names.first], source_dir + part_names.second); loadPartAndFixMetadata(part); loaded_parts.push_back(part); } @@ -3240,8 +3335,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::cloneAndLoadDataPart(const Merg LOG_DEBUG(log, "Cloning part " << src_part_absolute_path.toString() << " to " << dst_part_absolute_path.toString()); localBackup(src_part_absolute_path, dst_part_absolute_path); - MergeTreeData::MutableDataPartPtr dst_data_part = createPart( - *this, reservation->getDisk(), dst_part_name, dst_part_info, tmp_dst_part_name); + auto dst_data_part = createPart(dst_part_name, dst_part_info, reservation->getDisk(), tmp_dst_part_name); dst_data_part->is_temp = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d3bad9b3939..9220c4d633f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -178,6 +178,34 @@ public: using DataPartsLock = std::unique_lock; DataPartsLock lockParts() const { return DataPartsLock(data_parts_mutex); } + MergeTreeDataPartType choosePartType(size_t bytes_on_disk, size_t rows_count) const; + + + /// FIXME remove version with columns. + + MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, + size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; + + MutableDataPartPtr createPart(const String & name, + const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, + size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; + + /// After this methods loadColumnsChecksumsIndexes must be called + /// FIXME make this inside this function + MutableDataPartPtr createPart(const String & name, + const DiskSpace::DiskPtr & disk, const String & relative_path) const; + + MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, const String & relative_path) const; + + MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, + const DiskSpace::DiskPtr & disk, const String & relative_path) const; + + MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, + const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, const String & relative_path) const; + /// Auxiliary object to add a set of parts into the working set in two steps: /// * First, as PreCommitted parts (the parts are ready, but not yet in the active set). /// * Next, if commit() is called, the parts are added to the active set and the parts that are diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 599a1b846b7..0a1d65a8ff7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -564,9 +563,20 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor extractMergingAndGatheringColumns( all_columns, data.sorting_key_expr, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); + + size_t total_bytes = 0; + size_t total_rows = 0; + for (const auto & part : future_part.parts) + { + total_bytes += part->bytes_on_disk; + total_rows += part->rows_count; + } - MergeTreeData::MutableDataPartPtr new_data_part = createPart( - data, space_reservation->getDisk(), future_part.name, future_part.part_info, TMP_PREFIX + future_part.name); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( + future_part.name, future_part.part_info, + space_reservation->getDisk(), + all_columns, total_bytes, total_rows, + TMP_PREFIX + future_part.name); new_data_part->partition.assign(future_part.getPartition()); new_data_part->is_temp = true; @@ -950,8 +960,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor else LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation); - MergeTreeData::MutableDataPartPtr new_data_part = createPart( - data, space_reservation->getDisk(), future_part.name, future_part.part_info, "tmp_mut_" + future_part.name); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( + future_part.name, source_part->getType(), + future_part.part_info, space_reservation->getDisk(), + "tmp_mut_" + future_part.name); new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; @@ -1060,7 +1072,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor NameSet files_to_skip = {"checksums.txt", "columns.txt"}; /// Don't change granularity type while mutating subset of columns - auto mrk_extension = source_part->index_granularity_info.is_adaptive ? getAdaptiveMrkExtension() : getNonAdaptiveMrkExtension(); + auto mrk_extension = source_part->index_granularity_info.is_adaptive ? getAdaptiveMrkExtension(new_data_part->getType()) : getNonAdaptiveMrkExtension(); for (const auto & entry : updated_header) { IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 1871500d4e0..396a3490c3d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -55,10 +55,9 @@ namespace ErrorCodes MergeTreeDataPartCompact::MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, index_granularity_info_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) { } @@ -66,10 +65,9 @@ MergeTreeDataPartCompact::MergeTreeDataPartCompact( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, info_, index_granularity_info_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) { } @@ -139,31 +137,34 @@ ColumnSize MergeTreeDataPartCompact::getColumnSizeImpl( */ String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const { - const auto & storage_columns = storage.getColumns().getAllPhysical(); - const std::string * minimum_size_column = nullptr; - UInt64 minimum_size = std::numeric_limits::max(); + /// FIXME: save column sizes - for (const auto & column : storage_columns) - { - if (!hasColumnFiles(column.name, *column.type)) - continue; + // const auto & storage_columns = storage.getColumns().getAllPhysical(); + // const std::string * minimum_size_column = nullptr; + // UInt64 minimum_size = std::numeric_limits::max(); - const auto size = getColumnSizeImpl(column.name, *column.type, nullptr).data_compressed; - if (size < minimum_size) - { - minimum_size = size; - minimum_size_column = &column.name; - } - } + // for (const auto & column : storage_columns) + // { + // if (!hasColumnFiles(column.name, *column.type)) + // continue; - if (!minimum_size_column) - throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); + // const auto size = getColumnSizeImpl(column.name, *column.type, nullptr).data_compressed; + // if (size < minimum_size) + // { + // minimum_size = size; + // minimum_size_column = &column.name; + // } + // } - return *minimum_size_column; + // if (!minimum_size_column) + // throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); + + return columns.front().name; } void MergeTreeDataPartCompact::loadIndexGranularity() { + index_granularity_info.initialize(storage, getType(), columns.size()); String full_path = getFullPath(); if (columns.empty()) @@ -196,105 +197,109 @@ void MergeTreeDataPartCompact::loadIndexGranularity() void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { - String path = getFullPath(); + UNUSED(require_part_metadata); + /// FIXME implement for compact parts - if (!checksums.empty()) - { - if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) - throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); - if (require_part_metadata) - { - for (const NameAndTypePair & name_type : columns) - { - IDataType::SubstreamPath stream_path; - name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); - String mrk_file_name = file_name + index_granularity_info.marks_file_extension; - String bin_file_name = file_name + ".bin"; - if (!checksums.files.count(mrk_file_name)) - throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + path, - ErrorCodes::NO_FILE_IN_DATA_PART); - if (!checksums.files.count(bin_file_name)) - throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + path, - ErrorCodes::NO_FILE_IN_DATA_PART); - }, stream_path); - } - } + // String path = getFullPath(); - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - if (!checksums.files.count("count.txt")) - throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); + // if (!checksums.empty()) + // { + // if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) + // throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); - if (storage.partition_key_expr && !checksums.files.count("partition.dat")) - throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); + // if (require_part_metadata) + // { + // for (const NameAndTypePair & name_type : columns) + // { + // IDataType::SubstreamPath stream_path; + // name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + // { + // String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); + // String mrk_file_name = file_name + index_granularity_info.marks_file_extension; + // String bin_file_name = file_name + ".bin"; + // if (!checksums.files.count(mrk_file_name)) + // throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + path, + // ErrorCodes::NO_FILE_IN_DATA_PART); + // if (!checksums.files.count(bin_file_name)) + // throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + path, + // ErrorCodes::NO_FILE_IN_DATA_PART); + // }, stream_path); + // } + // } - if (!isEmpty()) - { - for (const String & col_name : storage.minmax_idx_columns) - { - if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) - throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); - } - } - } + // if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + // { + // if (!checksums.files.count("count.txt")) + // throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); - checksums.checkSizes(path); - } - else - { - auto check_file_not_empty = [&path](const String & file_path) - { - Poco::File file(file_path); - if (!file.exists() || file.getSize() == 0) - throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - return file.getSize(); - }; + // if (storage.partition_key_expr && !checksums.files.count("partition.dat")) + // throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); - /// Check that the primary key index is not empty. - if (!storage.primary_key_columns.empty()) - check_file_not_empty(path + "primary.idx"); + // if (!isEmpty()) + // { + // for (const String & col_name : storage.minmax_idx_columns) + // { + // if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) + // throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); + // } + // } + // } - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - check_file_not_empty(path + "count.txt"); + // checksums.checkSizes(path); + // } + // else + // { + // auto check_file_not_empty = [&path](const String & file_path) + // { + // Poco::File file(file_path); + // if (!file.exists() || file.getSize() == 0) + // throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + // return file.getSize(); + // }; - if (storage.partition_key_expr) - check_file_not_empty(path + "partition.dat"); + // /// Check that the primary key index is not empty. + // if (!storage.primary_key_columns.empty()) + // check_file_not_empty(path + "primary.idx"); - for (const String & col_name : storage.minmax_idx_columns) - check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); - } + // if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + // { + // check_file_not_empty(path + "count.txt"); - /// Check that all marks are nonempty and have the same size. + // if (storage.partition_key_expr) + // check_file_not_empty(path + "partition.dat"); - std::optional marks_size; - for (const NameAndTypePair & name_type : columns) - { - name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - Poco::File file(IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension); + // for (const String & col_name : storage.minmax_idx_columns) + // check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); + // } - /// Missing file is Ok for case when new column was added. - if (file.exists()) - { - UInt64 file_size = file.getSize(); + // /// Check that all marks are nonempty and have the same size. - if (!file_size) - throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + // std::optional marks_size; + // for (const NameAndTypePair & name_type : columns) + // { + // name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + // { + // Poco::File file(IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension); - if (!marks_size) - marks_size = file_size; - else if (file_size != *marks_size) - throw Exception("Part " + path + " is broken: marks have different sizes.", - ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - } - }); - } - } + // /// Missing file is Ok for case when new column was added. + // if (file.exists()) + // { + // UInt64 file_size = file.getSize(); + + // if (!file_size) + // throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", + // ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + + // if (!marks_size) + // marks_size = file_size; + // else if (file_size != *marks_size) + // throw Exception("Part " + path + " is broken: marks have different sizes.", + // ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + // } + // }); + // } + // } } MergeTreeDataPartCompact::~MergeTreeDataPartCompact() diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index d454cab70b2..1aa030a194a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -38,14 +38,12 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_ = {}); MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_ = {}); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp deleted file mode 100644 index 8f16b088ec0..00000000000 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "MergeTreeDataPartFactory.h" -#include -#include - -namespace DB -{ - std::shared_ptr createPart(const MergeTreeData & storage, const DiskSpace::DiskPtr & disk, const String & name, - const MergeTreePartInfo & info, const String & relative_path) - { - // /// FIXME - // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - // return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); - - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); - return std::make_shared(storage, name, info, index_granularity_info, disk, relative_path); - } - - std::shared_ptr createPart(MergeTreeData & storage, const DiskSpace::DiskPtr & disk, - const String & name, const String & relative_path) - { - // /// FIXME - // size_t size_of_mark = sizeof(size_t) + sizeof(size_t) * 2 * storage.getColumns().getAllPhysical().size(); - // MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk3", size_of_mark); - // return std::make_shared(storage, name, index_granularity_info, disk, relative_path); - MergeTreeIndexGranularityInfo index_granularity_info(storage, ".mrk2", sizeof(size_t) * 3); - return std::make_shared(storage, name, index_granularity_info, disk, relative_path); - } -} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h deleted file mode 100644 index 98d808a252a..00000000000 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartFactory.h +++ /dev/null @@ -1,10 +0,0 @@ -#include -#include -#include - -namespace DB -{ - std::shared_ptr createPart(const MergeTreeData & storage_, const DiskSpace::DiskPtr & disk_, const String & name_, const MergeTreePartInfo & info_, const String & relative_path); - - std::shared_ptr createPart(MergeTreeData & storage_, const DiskSpace::DiskPtr & disk_, const String & name_, const String & relative_path); -} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index cb2a9c47333..4dc7c16bd8d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -56,10 +56,9 @@ namespace ErrorCodes MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, index_granularity_info_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) { } @@ -67,10 +66,9 @@ MergeTreeDataPartWide::MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, info_, index_granularity_info_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) { } @@ -162,6 +160,7 @@ String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const void MergeTreeDataPartWide::loadIndexGranularity() { + index_granularity_info.initialize(storage, getType(), columns.size()); String full_path = getFullPath(); index_granularity_info.changeGranularityIfRequired(full_path); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index fdd574f8278..b1a31e96b98 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -38,14 +38,12 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk, const std::optional & relative_path = {}); MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const MergeTreeIndexGranularityInfo & index_granularity_info_, const DiskSpace::DiskPtr & disk, const std::optional & relative_path = {}); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 15612f074a5..bf9dab4df25 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include #include @@ -205,8 +204,11 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa auto reservation = data.reserveSpace(expected_size); - MergeTreeData::MutableDataPartPtr new_data_part = - createPart(data, reservation->getDisk(), part_name, new_part_info, TMP_PREFIX + part_name); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( + part_name, new_part_info, + reservation->getDisk(), block.getNamesAndTypesList(), + expected_size, block.rows(), + TMP_PREFIX + part_name); new_data_part->partition = std::move(partition); new_data_part->minmax_idx = std::move(minmax_idx); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 83952e0c625..e8e3a12a73b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -7,14 +7,7 @@ namespace DB { - -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - - -std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(const std::string & path_to_part) const +std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(const std::string & path_to_part) { if (Poco::File(path_to_part).exists()) { @@ -22,44 +15,62 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( for (Poco::DirectoryIterator part_it(path_to_part); part_it != end; ++part_it) { const auto & ext = "." + part_it.path().getExtension(); - if (ext == getNonAdaptiveMrkExtension() || ext == getAdaptiveMrkExtension()) + if (ext == getNonAdaptiveMrkExtension() + || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE) + || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) return ext; } } return {}; } -MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeData & storage, - const String & marks_file_extension_, UInt8 mark_size_in_bytes_) - : marks_file_extension(marks_file_extension_), mark_size_in_bytes(mark_size_in_bytes_) +MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo( + const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num) { + initialize(storage, part_type, columns_num); +} + +void MergeTreeIndexGranularityInfo::initialize(const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num) +{ + if (initialized) + return; + const auto storage_settings = storage.getSettings(); fixed_index_granularity = storage_settings->index_granularity; + /// Granularity is fixed if (!storage.canUseAdaptiveGranularity()) + { + if (part_type != MergeTreeDataPartType::WIDE) + throw ""; /// FIXME normal exception setNonAdaptive(); + } else - setAdaptive(storage_settings->index_granularity_bytes); + setAdaptive(storage_settings->index_granularity_bytes, part_type, columns_num); } -void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const String & path) +void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const std::string & path_to_part) { - auto mrk_ext = getMrkExtensionFromFS(path); - if (mrk_ext && *mrk_ext == ".mrk") /// TODO + /// FIXME check when we cant create compact part + auto mrk_ext = getMrkExtensionFromFS(path_to_part); + if (mrk_ext && *mrk_ext == getNonAdaptiveMrkExtension()) setNonAdaptive(); } -void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_) +void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_, MergeTreeDataPartType part_type, size_t columns_num) { is_adaptive = true; + mark_size_in_bytes = getAdaptiveMrkSize(part_type, columns_num); + marks_file_extension = getAdaptiveMrkExtension(part_type); index_granularity_bytes = index_granularity_bytes_; - } void MergeTreeIndexGranularityInfo::setNonAdaptive() { is_adaptive = false; + mark_size_in_bytes = getNonAdaptiveMrkSize(); + marks_file_extension = getNonAdaptiveMrkExtension(); index_granularity_bytes = 0; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index da3bcad7dbc..b2333813886 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -2,54 +2,83 @@ #include #include -// #include +#include namespace DB { class MergeTreeData; -class IMergeTreeDataPart; /// Meta information about index granularity struct MergeTreeIndexGranularityInfo { public: - using MergeTreeDataPartPtr = std::shared_ptr; - /// Marks file extension '.mrk' or '.mrk2' String marks_file_extension; /// Size of one mark in file two or three size_t numbers - UInt8 mark_size_in_bytes; + UInt16 mark_size_in_bytes = 0; /// Is stride in rows between marks non fixed? - bool is_adaptive; + bool is_adaptive = false; /// Fixed size in rows of one granule if index_granularity_bytes is zero - size_t fixed_index_granularity; + size_t fixed_index_granularity = 0; /// Approximate bytes size of one granule - size_t index_granularity_bytes; + size_t index_granularity_bytes = 0; - MergeTreeIndexGranularityInfo(const MergeTreeData & storage, - const String & mark_file_extension_, UInt8 mark_size_in_bytes_); + bool initialized = false; - void changeGranularityIfRequired(const String & path); + MergeTreeIndexGranularityInfo() {} + + MergeTreeIndexGranularityInfo( + const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num); + + void initialize(const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num); + + void changeGranularityIfRequired(const std::string & path_to_part); String getMarksFilePath(const String & path_prefix) const { return path_prefix + marks_file_extension; } -private: - void setAdaptive(size_t index_granularity_bytes_); + static std::optional getMrkExtensionFromFS(const std::string & path_to_table); + +private: + void setAdaptive(size_t index_granularity_bytes_, MergeTreeDataPartType part_type, size_t columns_num); void setNonAdaptive(); - std::optional getMrkExtensionFromFS(const std::string & path_to_table) const; + void setCompactAdaptive(size_t index_granularity_bytes_, size_t columns_num); }; constexpr inline auto getNonAdaptiveMrkExtension() { return ".mrk"; } -constexpr inline auto getAdaptiveMrkExtension() { return ".mrk2"; } constexpr inline auto getNonAdaptiveMrkSize() { return sizeof(UInt64) * 2; } -constexpr inline auto getAdaptiveMrkSize() { return sizeof(UInt64) * 3; } + +inline std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) +{ + switch(part_type) + { + case MergeTreeDataPartType::WIDE: + return ".mrk2"; + case MergeTreeDataPartType::COMPACT: + return ".mrk3"; + default: + throw "unknown part type"; /// FIXME normal exception + } +} + +inline size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num) +{ + switch(part_type) + { + case MergeTreeDataPartType::WIDE: + return sizeof(UInt64) * 3; + case MergeTreeDataPartType::COMPACT: + return sizeof(UInt64) * (columns_num * 2 + 1); + default: + throw "unknown part type"; /// FIXME normal exception + } +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 81baabda7df..0ae72df739d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -20,7 +20,7 @@ const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, siz if (!marks) loadMarks(); if (column_index >= columns_num) - throw Exception("", ErrorCodes::LOGICAL_ERROR); + throw Exception("", ErrorCodes::LOGICAL_ERROR); /// FIXME better exception return (*marks)[row_index * columns_num + column_index]; } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp index 7234ac6743a..8476dc0794a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -1,6 +1,5 @@ #include #include -#include #include #include @@ -145,7 +144,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt moving_part.part->makeCloneOnDiskDetached(moving_part.reserved_space); MergeTreeData::MutableDataPartPtr cloned_part = - createPart(*data, moving_part.reserved_space->getDisk(), moving_part.part->name, "detached/" + moving_part.part->name); + data->createPart(moving_part.part->name, moving_part.reserved_space->getDisk(), "detached/" + moving_part.part->name); LOG_TRACE(log, "Part " << moving_part.part->name << " was cloned to " << cloned_part->getFullPath()); cloned_part->loadColumnsChecksumsIndexes(true, true); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 226360a5d15..cee729c1c05 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -124,6 +124,9 @@ void MergeTreeReaderCompact::readData( void MergeTreeReaderCompact::initMarksLoader() { + if (marks_loader.initialized()) + return; + const auto & index_granularity_info = data_part->index_granularity_info; size_t marks_count = data_part->getMarksCount(); std::string mrk_path = index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 3652718451f..e6f581b58e0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -25,9 +25,14 @@ class ASTStorage; struct MergeTreeSettings : public SettingsCollection { +/// FIXME description for settings #define LIST_OF_MERGE_TREE_SETTINGS(M) \ M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.") \ \ + /** Data storing format settigns. */ \ + M(SettingUInt64, min_bytes_for_wide_part, 0, "") \ + M(SettingUInt64, min_rows_for_wide_part, 0, "") \ + \ /** Merge settings. */ \ M(SettingUInt64, max_bytes_to_merge_at_max_space_in_pool, 150ULL * 1024 * 1024 * 1024, "Maximum in total size of parts to merge, when there are maximum free threads in background pool (or entries in replication queue).") \ M(SettingUInt64, max_bytes_to_merge_at_min_space_in_pool, 1024 * 1024, "Maximum in total size of parts to merge, when there are minimum free threads in background pool (or entries in replication queue).") \ From 43b4c4c5f44ea1531bc38c982191d52007b02f28 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 21 Nov 2019 20:05:03 +0300 Subject: [PATCH 0059/2007] polymorphic parts (development) --- dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 2b0314645b3..45f509db78f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -108,13 +108,13 @@ void MergeTreeReaderStream::initMarksLoader() if (marks_loader.initialized()) return; - std::string mrk_path = index_granularity_info->getMarksFilePath(path_prefix); - - auto load = [&]() -> MarkCache::MappedPtr + auto load = [this]() -> MarkCache::MappedPtr { /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); + std::string mrk_path = index_granularity_info->getMarksFilePath(path_prefix); + size_t file_size = Poco::File(mrk_path).getSize(); size_t expected_file_size = index_granularity_info->mark_size_in_bytes * marks_count; if (expected_file_size != file_size) @@ -150,6 +150,7 @@ void MergeTreeReaderStream::initMarksLoader() return res; }; + auto mrk_path = index_granularity_info->getMarksFilePath(path_prefix); marks_loader = MergeTreeMarksLoader{mark_cache, mrk_path, load, save_marks_in_cache}; } From 49982ad18c80408a56e1d6714c0e95b75c7d28b1 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Fri, 22 Nov 2019 15:51:00 +0300 Subject: [PATCH 0060/2007] polymorphic parts (development) --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 43 +++++-------------- dbms/src/Storages/MergeTree/MergeTreeData.h | 15 ++----- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- 4 files changed, 17 insertions(+), 45 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index e0cc0990aee..a833cb66f98 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1578,7 +1578,6 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name } } -/// FIXME implement createPart without columns and with loadMetadata MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_on_disk, size_t rows_count) const { @@ -1589,14 +1588,9 @@ MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_on_disk, size_t return MergeTreeDataPartType::WIDE; } -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, - const DiskSpace::DiskPtr & disk, const String & relative_path) const -{ - return createPart(name, type, MergeTreePartInfo::fromPartName(name, format_version), disk, relative_path); -} -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, - const MergeTreePartInfo & part_info, +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, + MergeTreeDataPartType type, const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, const String & relative_path) const { if (type == MergeTreeDataPartType::COMPACT) @@ -1608,20 +1602,11 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, - const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, - size_t bytes_on_disk, size_t rows_num, const String & relative_path) const -{ - return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), - disk, columns, bytes_on_disk, rows_num, relative_path); -} - -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, + const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, size_t bytes_on_disk, size_t rows_count, const String & relative_path) const { auto part = createPart(name, choosePartType(bytes_on_disk, rows_count), part_info, disk, relative_path); - part->setColumns(columns); part->bytes_on_disk = bytes_on_disk; part->rows_count = rows_count; @@ -1640,35 +1625,29 @@ static MergeTreeDataPartType getPartTypeFromMarkExtension(const String & mrk_ext return MergeTreeDataPartType::UNKNOWN; } -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, - const DiskSpace::DiskPtr & disk, const String & relative_path) const +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( + const String & name, const DiskSpace::DiskPtr & disk, const String & relative_path) const { return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), disk, relative_path); } -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, const MergeTreePartInfo & part_info, +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( + const String & name, const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, const String & relative_path) const { auto type = MergeTreeDataPartType::UNKNOWN; auto full_path = getFullPathOnDisk(disk) + relative_path + "/"; auto mrk_ext = MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(full_path); + if (mrk_ext) type = getPartTypeFromMarkExtension(*mrk_ext); else + { /// Didn't find any mark file, suppose that part is empty. type = choosePartType(0, 0); + } - MutableDataPartPtr part; - - /// FIXME do not pass emty granularity_info - if (type == MergeTreeDataPartType::COMPACT) - part = std::make_shared(*this, name, part_info, disk, relative_path); - else if (type == MergeTreeDataPartType::WIDE) - part = std::make_shared(*this, name, part_info, disk, relative_path); - else - throw Exception("Unknown part type", ErrorCodes::LOGICAL_ERROR); - - return part; + return createPart(name, type, part_info, disk, relative_path); } void MergeTreeData::alterDataPart( diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 9220c4d633f..e0b1eb6c8d9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -183,13 +183,13 @@ public: /// FIXME remove version with columns. - MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, + MutableDataPartPtr createPart(const String & name, + const MergeTreePartInfo & part_info,const DiskSpace::DiskPtr & disk, size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; MutableDataPartPtr createPart(const String & name, - const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, - size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; + MergeTreeDataPartType type, const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, const String & relative_path) const; /// After this methods loadColumnsChecksumsIndexes must be called /// FIXME make this inside this function @@ -199,13 +199,6 @@ public: MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, const String & relative_path) const; - MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, - const DiskSpace::DiskPtr & disk, const String & relative_path) const; - - MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, - const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, const String & relative_path) const; - /// Auxiliary object to add a set of parts into the working set in two steps: /// * First, as PreCommitted parts (the parts are ready, but not yet in the active set). /// * Next, if commit() is called, the parts are added to the active set and the parts that are diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 0a1d65a8ff7..bf813a96c26 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -575,7 +575,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( future_part.name, future_part.part_info, space_reservation->getDisk(), - all_columns, total_bytes, total_rows, + total_bytes, total_rows, TMP_PREFIX + future_part.name); new_data_part->partition.assign(future_part.getPartition()); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index bf9dab4df25..e74d446cfdf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -206,7 +206,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( part_name, new_part_info, - reservation->getDisk(), block.getNamesAndTypesList(), + reservation->getDisk(), expected_size, block.rows(), TMP_PREFIX + part_name); From 921d32485bb0ca1dea638b0b0741f4a023ce419d Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Fri, 22 Nov 2019 16:58:18 +0300 Subject: [PATCH 0061/2007] polymorphic parts (development) --- dbms/src/Storages/MergeTree/IMergeTreeReader.h | 1 - .../MergeTree/MergeTreeDataMergerMutator.cpp | 17 ++++++----------- .../Storages/MergeTree/MergeTreeDataWriter.cpp | 1 + 3 files changed, 7 insertions(+), 12 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 92b5dcbae98..df92f89b4e6 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -76,7 +76,6 @@ protected: const MergeTreeData & storage; MarkRanges all_mark_ranges; - friend class MergeTreeRangeReader::DelayedStream; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index bf813a96c26..3526a5ff1ea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -564,20 +564,15 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor all_columns, data.sorting_key_expr, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); - size_t total_bytes = 0; - size_t total_rows = 0; - for (const auto & part : future_part.parts) - { - total_bytes += part->bytes_on_disk; - total_rows += part->rows_count; - } - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( - future_part.name, future_part.part_info, + future_part.name, + future_part.part_info, space_reservation->getDisk(), - total_bytes, total_rows, + merge_entry->total_size_bytes_compressed, + merge_entry->total_rows_count, TMP_PREFIX + future_part.name); + new_data_part->setColumns(all_columns); new_data_part->partition.assign(future_part.getPartition()); new_data_part->is_temp = true; @@ -964,7 +959,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor future_part.name, source_part->getType(), future_part.part_info, space_reservation->getDisk(), "tmp_mut_" + future_part.name); - + new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; new_data_part->index_granularity_info = source_part->index_granularity_info; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index e74d446cfdf..c51bd3b4dda 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -265,6 +265,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa auto compression_codec = data.global_context.chooseCompressionCodec(0, 0); NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); + new_data_part->setColumns(columns); MergedBlockOutputStream out(new_data_part, columns, compression_codec); From b54f1629ab77fe29f3e58694b60d74abc6735cfc Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 25 Nov 2019 14:06:59 +0300 Subject: [PATCH 0062/2007] polymorphic parts (development) --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 6 +++--- .../MergeTree/IMergeTreeDataPartWriter.cpp | 4 ++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 1 - .../MergeTree/MergeTreeDataMergerMutator.cpp | 15 +++++++++----- .../MergeTree/MergeTreeDataPartWide.cpp | 1 - .../MergeTreeDataPartWriterCompact.cpp | 3 ++- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 2 +- .../MergeTree/MergeTreeDataWriter.cpp | 3 +-- .../MergeTree/MergeTreeReaderCompact.cpp | 20 ++++++++++++------- 9 files changed, 34 insertions(+), 21 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index f5a8588a6f7..61500997ddd 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -326,8 +326,6 @@ String IMergeTreeDataPart::stateString() const return stateToString(state); } - - void IMergeTreeDataPart::assertState(const std::initializer_list & affordable_states) const { if (!checkState(affordable_states)) @@ -599,9 +597,11 @@ void IMergeTreeDataPart::loadColumns(bool require) } is_frozen = !poco_file_path.canWrite(); - ReadBufferFromFile file = openForReading(path); columns.readText(file); + index_granularity_info.initialize(storage, getType(), columns.size()); + for (const auto & it : columns) + sample_block.insert({it.type, it.name}); } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 9b70f4e6cef..dbf12110987 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -98,6 +98,7 @@ void fillIndexGranularityImpl( MergeTreeIndexGranularity & index_granularity, bool can_use_adaptive_index_granularity) { + /// FIXME correct index granularity for compact size_t rows_in_block = block.rows(); size_t index_granularity_for_block; if (!can_use_adaptive_index_granularity) @@ -124,6 +125,9 @@ void fillIndexGranularityImpl( /// We should be less or equal than fixed index granularity index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); + /// FIXME correct index granularity for compact + index_granularity_for_block = rows_in_block; + /// FIXME: split/join last mark for compact parts for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) index_granularity.appendMark(index_granularity_for_block); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index e0b1eb6c8d9..183027a179d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -181,7 +181,6 @@ public: MergeTreeDataPartType choosePartType(size_t bytes_on_disk, size_t rows_count) const; - /// FIXME remove version with columns. MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info,const DiskSpace::DiskPtr & disk, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 3526a5ff1ea..185cf510301 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -563,23 +563,26 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor extractMergingAndGatheringColumns( all_columns, data.sorting_key_expr, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); + + size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; + size_t estimated_bytes_uncompressed = 0; + for (const auto & part : parts) + estimated_bytes_uncompressed += part->getTotalColumnsSize().data_uncompressed; MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( future_part.name, future_part.part_info, space_reservation->getDisk(), - merge_entry->total_size_bytes_compressed, - merge_entry->total_rows_count, + estimated_bytes_uncompressed, + sum_input_rows_upper_bound, TMP_PREFIX + future_part.name); new_data_part->setColumns(all_columns); new_data_part->partition.assign(future_part.getPartition()); new_data_part->is_temp = true; - size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; - bool need_remove_expired_values = force_ttl; - for (const MergeTreeData::DataPartPtr & part : parts) + for (const auto & part : parts) new_data_part->ttl_infos.update(part->ttl_infos); const auto & part_min_ttl = new_data_part->ttl_infos.part_min_ttl; @@ -954,6 +957,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor } else LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( future_part.name, source_part->getType(), @@ -1080,6 +1084,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor IDataType::SubstreamPath stream_path; entry.type->enumerateStreams(callback, stream_path); } + for (const auto & index : indices_to_recalc) { files_to_skip.insert(index->getFileName() + ".idx"); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 4dc7c16bd8d..d214939967c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -160,7 +160,6 @@ String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const void MergeTreeDataPartWide::loadIndexGranularity() { - index_granularity_info.initialize(storage, getType(), columns.size()); String full_path = getFullPath(); index_granularity_info.changeGranularityIfRequired(full_path); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 8a97ca0a658..2cf4d57e658 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -77,7 +77,8 @@ void MergeTreeDataPartWriterCompact::write( bool write_marks = true; // size_t rows_to_write = std::min(total_rows, index_granularity.getMarkRows(current_mark)); size_t rows_to_write = total_rows; - index_granularity.appendMark(total_rows); + // if (compute_granularity) + // index_granularity.appendMark(total_rows); if (rows_to_write) data_written = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 46e0fee2ac5..94aaba60284 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -243,7 +243,7 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( while (current_row < total_rows) { size_t rows_to_write; - bool write_marks = true; + bool write_marks = true; /// FIXME not always true /// If there is `index_offset`, then the first mark goes not immediately, but after this number of rows. if (current_row == 0 && index_offset != 0) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index c51bd3b4dda..81acbb5e600 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -203,8 +203,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa size_t expected_size = block.bytes(); auto reservation = data.reserveSpace(expected_size); - - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( + auto new_data_part = data.createPart( part_name, new_part_info, reservation->getDisk(), expected_size, block.rows(), diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index cee729c1c05..fa42ada3fed 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -53,6 +53,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, UNUSED(max_rows_to_read); UNUSED(res); + /// FIXME compute correct granularity size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); size_t read_rows = 0; @@ -114,6 +115,7 @@ void MergeTreeReaderCompact::readData( type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); type.deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state); + std::cerr << "(MergeTreeReaderCompact::readData) end reading column rows: " << column.size() << "\n"; std::cerr << "(MergeTreeReaderCompact::readData) end reading column: " << name << "\n"; // if (column.size() != rows_to_read) @@ -127,16 +129,20 @@ void MergeTreeReaderCompact::initMarksLoader() if (marks_loader.initialized()) return; - const auto & index_granularity_info = data_part->index_granularity_info; - size_t marks_count = data_part->getMarksCount(); - std::string mrk_path = index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); + std::string mrk_path = data_part->index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); size_t columns_num = data_part->columns.size(); - auto load = [&]() -> MarkCache::MappedPtr + /// FIXME pass mrk_path as argument + auto load = [this, columns_num, mrk_path]() -> MarkCache::MappedPtr { size_t file_size = Poco::File(mrk_path).getSize(); + size_t marks_count = data_part->getMarksCount(); + size_t mark_size_in_bytes = data_part->index_granularity_info.mark_size_in_bytes; - size_t expected_file_size = index_granularity_info.mark_size_in_bytes * marks_count; + std::cerr << "(initMarksLoader) marks_count: " << marks_count << "\n"; + std::cerr << "() mark_size_in_bytes: " << mark_size_in_bytes << "\n"; + + size_t expected_file_size = mark_size_in_bytes * marks_count; if (expected_file_size != file_size) throw Exception( "Bad size of marks file '" + mrk_path + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), @@ -163,9 +169,9 @@ void MergeTreeReaderCompact::initMarksLoader() } std::cerr << "(MergeTreeReaderCompact::loadMarks) file_size: " << file_size << "\n"; - std::cerr << "(MergeTreeReaderCompact::loadMarks) correct file size: " << i * index_granularity_info.mark_size_in_bytes << "\n"; + std::cerr << "(MergeTreeReaderCompact::loadMarks) correct file size: " << i * mark_size_in_bytes << "\n"; - if (i * index_granularity_info.mark_size_in_bytes != file_size) + if (i * mark_size_in_bytes != file_size) throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); res->protect(); From 9e7adf4cbe66aa63910ea4926b4a31df7d9298db Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 25 Nov 2019 23:19:43 +0300 Subject: [PATCH 0063/2007] polymorphic parts (development) --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 23 +++++----- dbms/src/Storages/MergeTree/MergeTreeData.h | 5 ++- .../MergeTree/MergeTreeDataMergerMutator.cpp | 43 ++++++++++--------- .../MergeTree/MergeTreeDataWriter.cpp | 9 ++-- .../MergeTree/MergeTreeIndexReader.cpp | 1 + .../MergeTree/MergeTreeMarksLoader.cpp | 7 +-- .../Storages/MergeTree/MergeTreeMarksLoader.h | 2 +- .../MergeTree/MergeTreeReaderCompact.cpp | 9 ++-- .../MergeTree/MergeTreeReaderStream.cpp | 8 ++-- .../MergeTree/MergedBlockOutputStream.cpp | 2 +- 11 files changed, 58 insertions(+), 53 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index dbf12110987..e44ace2e521 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -126,7 +126,7 @@ void fillIndexGranularityImpl( index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); /// FIXME correct index granularity for compact - index_granularity_for_block = rows_in_block; + // index_granularity_for_block = rows_in_block; /// FIXME: split/join last mark for compact parts for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index a833cb66f98..dba837a80df 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1579,10 +1579,10 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name } -MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_on_disk, size_t rows_count) const +MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_uncompressed, size_t rows_count) const { const auto settings = getSettings(); - if (bytes_on_disk < settings->min_bytes_for_wide_part || rows_count < settings->min_rows_for_wide_part) + if (bytes_uncompressed < settings->min_bytes_for_wide_part || rows_count < settings->min_rows_for_wide_part) return MergeTreeDataPartType::COMPACT; return MergeTreeDataPartType::WIDE; @@ -1601,15 +1601,18 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, throw Exception("Unknown part type", ErrorCodes::LOGICAL_ERROR); } -MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, - const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, - size_t bytes_on_disk, size_t rows_count, const String & relative_path) const +MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( + const String & name, + const MergeTreePartInfo & part_info, + const DiskSpace::DiskPtr & disk, + const NamesAndTypesList & columns, + size_t bytes_uncompressed, + size_t rows_count, + const String & relative_path) const { - auto part = createPart(name, choosePartType(bytes_on_disk, rows_count), part_info, disk, relative_path); - - part->bytes_on_disk = bytes_on_disk; - part->rows_count = rows_count; - + auto part = createPart(name, choosePartType(bytes_uncompressed, rows_count), part_info, disk, relative_path); + part->setColumns(columns); + /// Don't save rows_count count here as it can change later return part; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 183027a179d..be1fa069f1b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -180,10 +180,11 @@ public: MergeTreeDataPartType choosePartType(size_t bytes_on_disk, size_t rows_count) const; - - + /// After this methods setColumns must be called + /// FIXME make this inside this function MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info,const DiskSpace::DiskPtr & disk, + const NamesAndTypesList & columns, size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; MutableDataPartPtr createPart(const String & name, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 185cf510301..9f65a2fe964 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -573,11 +573,11 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor future_part.name, future_part.part_info, space_reservation->getDisk(), + all_columns, estimated_bytes_uncompressed, sum_input_rows_upper_bound, TMP_PREFIX + future_part.name); - new_data_part->setColumns(all_columns); new_data_part->partition.assign(future_part.getPartition()); new_data_part->is_temp = true; @@ -958,15 +958,32 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor else LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation); + auto in = mutations_interpreter.execute(table_lock_holder); + const auto & updated_header = mutations_interpreter.getUpdatedHeader(); - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( - future_part.name, source_part->getType(), - future_part.part_info, space_reservation->getDisk(), + NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); + + const auto & source_column_names = source_part->columns.getNames(); + const auto & updated_column_names = updated_header.getNames(); + + NameSet new_columns_set(source_column_names.begin(), source_column_names.end()); + new_columns_set.insert(updated_column_names.begin(), updated_column_names.end()); + auto new_columns = all_columns.filter(new_columns_set); + + auto new_data_part = data.createPart( + future_part.name, + future_part.part_info, + space_reservation->getDisk(), + std::move(new_columns), + source_part->bytes_on_disk, + source_part->rows_count, "tmp_mut_" + future_part.name); new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; - new_data_part->index_granularity_info = source_part->index_granularity_info; + + /// FIXME Now it's wrong code. Check if nothing will break + // new_data_part->index_granularity_info = source_part->index_granularity_info; String new_part_tmp_path = new_data_part->getFullPath(); @@ -981,10 +998,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor Poco::File(new_part_tmp_path).createDirectories(); - auto in = mutations_interpreter.execute(table_lock_holder); - const auto & updated_header = mutations_interpreter.getUpdatedHeader(); - - NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); const auto data_settings = data.getSettings(); Block in_header = in->getHeader(); @@ -1141,18 +1154,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor WriteBufferFromFile out_checksums(new_part_tmp_path + "checksums.txt", 4096); new_data_part->checksums.write(out_checksums); } - - /// Write the columns list of the resulting part in the same order as all_columns. - Names source_column_names = source_part->columns.getNames(); - NameSet source_columns_name_set(source_column_names.begin(), source_column_names.end()); - for (auto it = all_columns.begin(); it != all_columns.end();) - { - if (source_columns_name_set.count(it->name) || updated_header.has(it->name)) - ++it; - else - it = new_data_part->columns.erase(it); - } - new_data_part->setColumns(all_columns); { /// Write a file with a description of columns. WriteBufferFromFile out_columns(new_part_tmp_path + "columns.txt", 4096); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 81acbb5e600..5eea3f38c46 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -203,10 +203,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa size_t expected_size = block.bytes(); auto reservation = data.reserveSpace(expected_size); + NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); + auto new_data_part = data.createPart( part_name, new_part_info, reservation->getDisk(), - expected_size, block.rows(), + columns, + expected_size, + block.rows(), TMP_PREFIX + part_name); new_data_part->partition = std::move(partition); @@ -263,9 +267,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_codec = data.global_context.chooseCompressionCodec(0, 0); - NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); - new_data_part->setColumns(columns); - MergedBlockOutputStream out(new_data_part, columns, compression_codec); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp index ce852d1fa46..af899bba402 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -26,6 +26,7 @@ MergeTreeIndexGranulePtr MergeTreeIndexReader::read() { auto granule = index->createIndexGranule(); granule->deserializeBinary(*stream.data_buffer); + std::cerr << "(MergeTreeIndexReader) granule.empty(): " << granule->empty() << "\n"; return granule; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 0ae72df739d..2dbdc6d7ad6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -27,22 +27,23 @@ const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, siz void MergeTreeMarksLoader::loadMarks() { + auto load = std::bind(load_func, mrk_path); if (mark_cache) { auto key = mark_cache->hash(mrk_path); if (save_marks_in_cache) { - marks = mark_cache->getOrSet(key, load_func); + marks = mark_cache->getOrSet(key, load); } else { marks = mark_cache->get(key); if (!marks) - marks = load_func(); + marks = load(); } } else - marks = load_func(); + marks = load(); if (!marks) throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h index 8c3bc1d5dff..aefcaff7aa1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -7,7 +7,7 @@ class MergeTreeMarksLoader { public: using MarksPtr = MarkCache::MappedPtr; - using LoadFunc = std::function; + using LoadFunc = std::function; MergeTreeMarksLoader() {} diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index fa42ada3fed..517c6212b1b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -129,11 +129,9 @@ void MergeTreeReaderCompact::initMarksLoader() if (marks_loader.initialized()) return; - std::string mrk_path = data_part->index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); size_t columns_num = data_part->columns.size(); - /// FIXME pass mrk_path as argument - auto load = [this, columns_num, mrk_path]() -> MarkCache::MappedPtr + auto load = [this, columns_num](const String & mrk_path) -> MarkCache::MappedPtr { size_t file_size = Poco::File(mrk_path).getSize(); size_t marks_count = data_part->getMarksCount(); @@ -178,9 +176,8 @@ void MergeTreeReaderCompact::initMarksLoader() return res; }; - marks_loader = MergeTreeMarksLoader{mark_cache, mrk_path, load, settings.save_marks_in_cache, columns_num}; - - std::cerr << "(MergeTreeReaderCompact::loadMarks) end marks load..." << "\n"; + auto mrk_path = data_part->index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); + marks_loader = MergeTreeMarksLoader{mark_cache, std::move(mrk_path), load, settings.save_marks_in_cache, columns_num}; } void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 45f509db78f..8fde48c6791 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -108,13 +108,13 @@ void MergeTreeReaderStream::initMarksLoader() if (marks_loader.initialized()) return; - auto load = [this]() -> MarkCache::MappedPtr + auto load = [this](const String & mrk_path) -> MarkCache::MappedPtr { + std::cerr << "reading marks from path: " << mrk_path << "\n"; + std::cerr << "marks: " << marks_count << "\n"; /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - std::string mrk_path = index_granularity_info->getMarksFilePath(path_prefix); - size_t file_size = Poco::File(mrk_path).getSize(); size_t expected_file_size = index_granularity_info->mark_size_in_bytes * marks_count; if (expected_file_size != file_size) @@ -151,7 +151,7 @@ void MergeTreeReaderStream::initMarksLoader() }; auto mrk_path = index_granularity_info->getMarksFilePath(path_prefix); - marks_loader = MergeTreeMarksLoader{mark_cache, mrk_path, load, save_marks_in_cache}; + marks_loader = MergeTreeMarksLoader{mark_cache, std::move(mrk_path), load, save_marks_in_cache}; } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 2cb650c177d..9cfc920c8e7 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -146,7 +146,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); - new_part->setColumns(*total_column_list); + // new_part->setColumns(*total_column_list); new_part->index = writer->releaseIndexColumns(); new_part->checksums = checksums; new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); From 49e465d6e0caf1f08956568e0d714b990efde114 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 26 Nov 2019 12:48:22 +0300 Subject: [PATCH 0064/2007] polymorphic parts (development) --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 27 ++++++++++++++++--- .../MergeTree/IMergeTreeDataPartWriter.h | 5 ++-- .../MergeTreeDataPartWriterCompact.cpp | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 2 +- .../MergeTree/MergeTreeIndexGranularity.cpp | 6 +++++ .../MergeTree/MergeTreeIndexGranularity.h | 2 ++ 6 files changed, 36 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index e44ace2e521..955164dc7ac 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -66,7 +66,8 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const WriterSettings & settings_, - const MergeTreeIndexGranularity & index_granularity_) + const MergeTreeIndexGranularity & index_granularity_, + bool need_finish_last_granule_) : part_path(part_path_) , storage(storage_) , columns_list(columns_list_) @@ -77,6 +78,7 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( , settings(settings_) , compute_granularity(index_granularity.empty()) , with_final_mark(storage.getSettings()->write_final_mark && settings.can_use_adaptive_granularity) + , need_finish_last_granule(need_finish_last_granule_) { if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); @@ -96,7 +98,8 @@ void fillIndexGranularityImpl( bool blocks_are_granules, size_t index_offset, MergeTreeIndexGranularity & index_granularity, - bool can_use_adaptive_index_granularity) + bool can_use_adaptive_index_granularity, + bool need_finish_last_granule) { /// FIXME correct index granularity for compact size_t rows_in_block = block.rows(); @@ -129,8 +132,23 @@ void fillIndexGranularityImpl( // index_granularity_for_block = rows_in_block; /// FIXME: split/join last mark for compact parts - for (size_t current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) + size_t current_row; + for (current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) index_granularity.appendMark(index_granularity_for_block); + + size_t rows_rest_in_block = rows_in_block - (current_row - index_granularity_for_block); + if (need_finish_last_granule && rows_rest_in_block) + { + /// If enough rows are left, create a new granule. Otherwise, extend previous granule. + /// So,real size of granule differs from index_granularity_for_block not more than 50%. + if (rows_rest_in_block * 2 >= index_granularity_for_block) + index_granularity.appendMark(rows_rest_in_block); + else + { + index_granularity.popMark(); + index_granularity.appendMark(index_granularity_for_block + rows_rest_in_block); + } + } } void IMergeTreeDataPartWriter::fillIndexGranularity(const Block & block) @@ -143,7 +161,8 @@ void IMergeTreeDataPartWriter::fillIndexGranularity(const Block & block) settings.blocks_are_granules_size, index_offset, index_granularity, - settings.can_use_adaptive_granularity); + settings.can_use_adaptive_granularity, + need_finish_last_granule); } void IMergeTreeDataPartWriter::initPrimaryIndex() diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 2eab44cba40..c182ca072fe 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -66,7 +66,8 @@ public: const String & marks_file_extension, const CompressionCodecPtr & default_codec, const WriterSettings & settings, - const MergeTreeIndexGranularity & index_granularity); + const MergeTreeIndexGranularity & index_granularity, + bool need_finish_last_granule); virtual void write( const Block & block, const IColumn::Permutation * permutation, @@ -115,6 +116,7 @@ protected: bool compute_granularity; bool with_final_mark; + bool need_finish_last_granule; size_t current_mark = 0; size_t index_offset = 0; @@ -141,7 +143,6 @@ protected: bool data_written = false; bool primary_index_initialized = false; bool skip_indices_initialized = false; - }; using MergeTreeWriterPtr = std::unique_ptr; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 2cf4d57e658..9c6ac028ac1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -22,7 +22,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, indices_to_recalc_, marks_file_extension_, - default_codec_, settings_, index_granularity_) + default_codec_, settings_, index_granularity_, true) { stream = std::make_unique( DATA_FILE_NAME, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 94aaba60284..7b17bcb2ec8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -19,7 +19,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const MergeTreeIndexGranularity & index_granularity_) : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, indices_to_recalc_, - marks_file_extension_, default_codec_, settings_, index_granularity_) + marks_file_extension_, default_codec_, settings_, index_granularity_, false) , can_use_adaptive_granularity(storage_.canUseAdaptiveGranularity()) { const auto & columns = storage.getColumns(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp index bfa60d653c0..b2dd5141df3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp @@ -48,6 +48,12 @@ void MergeTreeIndexGranularity::appendMark(size_t rows_count) marks_rows_partial_sums.push_back(marks_rows_partial_sums.back() + rows_count); } +void MergeTreeIndexGranularity::popMark() +{ + if (!marks_rows_partial_sums.empty()) + marks_rows_partial_sums.pop_back(); +} + size_t MergeTreeIndexGranularity::getRowsCountInRange(size_t begin, size_t end) const { size_t subtrahend = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h index 5a006855632..a2d78596279 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -92,6 +92,8 @@ public: /// Add new mark with rows_count void appendMark(size_t rows_count); + void popMark(); + /// Add `size` of marks with `fixed_granularity` rows void resizeWithFixedGranularity(size_t size, size_t fixed_granularity); }; From d1ddfbb415e01148779b2f2d5e81e95ec57fd02d Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 27 Nov 2019 14:35:27 +0300 Subject: [PATCH 0065/2007] polymorphic parts (development) --- .../AsynchronousBlockInputStream.h | 1 - dbms/src/DataStreams/SquashingTransform.h | 2 + .../MergeTree/IMergeTreeDataPartWriter.cpp | 32 +++--- .../MergeTree/IMergeTreeDataPartWriter.h | 1 + .../MergeTreeDataPartWriterCompact.cpp | 100 +++++++++--------- .../MergeTreeDataPartWriterCompact.h | 6 ++ .../MergeTree/MergeTreeDataPartWriterWide.cpp | 2 +- .../MergeTree/MergeTreeIndexGranularity.cpp | 8 ++ .../MergeTree/MergeTreeIndexGranularity.h | 2 + .../MergeTree/MergeTreeRangeReader.cpp | 5 + .../MergeTree/MergeTreeReaderCompact.cpp | 72 ++++++++----- 11 files changed, 134 insertions(+), 97 deletions(-) diff --git a/dbms/src/DataStreams/AsynchronousBlockInputStream.h b/dbms/src/DataStreams/AsynchronousBlockInputStream.h index 93c695f20c9..a32d3049995 100644 --- a/dbms/src/DataStreams/AsynchronousBlockInputStream.h +++ b/dbms/src/DataStreams/AsynchronousBlockInputStream.h @@ -70,7 +70,6 @@ public: return ready.tryWait(milliseconds); } - Block getHeader() const override { return children.at(0)->getHeader(); } void cancel(bool kill) override diff --git a/dbms/src/DataStreams/SquashingTransform.h b/dbms/src/DataStreams/SquashingTransform.h index f1681c57c8c..de6f1cc8383 100644 --- a/dbms/src/DataStreams/SquashingTransform.h +++ b/dbms/src/DataStreams/SquashingTransform.h @@ -40,6 +40,8 @@ public: */ Result add(MutableColumns && columns); + bool hasPendingData() { return !accumulated_columns.empty(); } + private: size_t min_block_size_rows; size_t min_block_size_bytes; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 955164dc7ac..5826fb0b1f8 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -127,28 +127,30 @@ void fillIndexGranularityImpl( /// We should be less or equal than fixed index granularity index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); - - /// FIXME correct index granularity for compact - // index_granularity_for_block = rows_in_block; - - /// FIXME: split/join last mark for compact parts + size_t current_row; for (current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) - index_granularity.appendMark(index_granularity_for_block); - - size_t rows_rest_in_block = rows_in_block - (current_row - index_granularity_for_block); - if (need_finish_last_granule && rows_rest_in_block) { - /// If enough rows are left, create a new granule. Otherwise, extend previous granule. - /// So,real size of granule differs from index_granularity_for_block not more than 50%. - if (rows_rest_in_block * 2 >= index_granularity_for_block) - index_granularity.appendMark(rows_rest_in_block); + size_t rows_rest_in_block = rows_in_block - current_row; + std::cerr << "rows_rest_in_block: " << rows_rest_in_block << "\n"; + std::cerr << "rows_rest_in_block: " << index_granularity_for_block << "\n"; + + /// FIXME may be remove need_finish_last_granule and do it always + if (need_finish_last_granule && rows_rest_in_block < index_granularity_for_block) + { + if (rows_rest_in_block * 2 >= index_granularity_for_block) + index_granularity.appendMark(rows_rest_in_block); + else + index_granularity.addRowsToLastMark(rows_rest_in_block); + } else { - index_granularity.popMark(); - index_granularity.appendMark(index_granularity_for_block + rows_rest_in_block); + index_granularity.appendMark(index_granularity_for_block); } } + + for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) + std::cerr << "marks: " << index_granularity.getMarkRows(i) << "\n"; } void IMergeTreeDataPartWriter::fillIndexGranularity(const Block & block) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index c182ca072fe..e3deeea859e 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -69,6 +69,7 @@ public: const MergeTreeIndexGranularity & index_granularity, bool need_finish_last_granule); + /// FIXME remove indices block virtual void write( const Block & block, const IColumn::Permutation * permutation, /* Blocks with already sorted index columns */ diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 9c6ac028ac1..ab95bacafe9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -23,6 +23,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_, true) + , squashing(storage.getSettings()->index_granularity, storage.getSettings()->index_granularity_bytes) { stream = std::make_unique( DATA_FILE_NAME, @@ -37,6 +38,35 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( void MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) +{ + UNUSED(primary_key_block); + UNUSED(skip_indexes_block); + + if (!header) + header = block.cloneEmpty(); + + Block result_block = block; + + if (permutation) + { + auto it = columns_list.begin(); + for (size_t i = 0; i < columns_list.size(); ++i) + { + auto & column = result_block.getByName(it->name); + column.column = column.column->permute(*permutation, 0); + } + } + + auto result = squashing.add(result_block.mutateColumns()); + if (!result.ready) + return; + + result_block = header.cloneWithColumns(std::move(result.columns)); + + writeBlock(result_block); +} + +void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) { size_t total_rows = block.rows(); size_t from_mark = current_mark; @@ -48,25 +78,10 @@ void MergeTreeDataPartWriterCompact::write( if (compute_granularity) fillIndexGranularity(block); - ColumnsWithTypeAndName columns_to_write(columns_list.size()); - auto it = columns_list.begin(); - for (size_t i = 0; i < columns_list.size(); ++i, ++it) - { - if (permutation) - { - if (primary_key_block.has(it->name)) - columns_to_write[i] = primary_key_block.getByName(it->name); - else if (skip_indexes_block.has(it->name)) - columns_to_write[i] = skip_indexes_block.getByName(it->name); - else - { - columns_to_write[i] = block.getByName(it->name); - columns_to_write[i].column = columns_to_write[i].column->permute(*permutation, 0); - } - } - else - columns_to_write[i] = block.getByName(it->name); - } + std::cerr << "(MergeTreeDataPartWriterCompact::write) marks: " << index_granularity.getMarksCount() << "\n"; + + for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) + std::cerr << "rows in mark: " << index_granularity.getMarkRows(i) << "\n"; std::cerr << "(MergeTreeDataPartWriterCompact::write) total_rows: " << total_rows << "\n"; @@ -74,50 +89,28 @@ void MergeTreeDataPartWriterCompact::write( { std::cerr << "(MergeTreeDataPartWriterCompact::write) current_row: " << current_row << "\n"; - bool write_marks = true; - // size_t rows_to_write = std::min(total_rows, index_granularity.getMarkRows(current_mark)); - size_t rows_to_write = total_rows; - // if (compute_granularity) - // index_granularity.appendMark(total_rows); + size_t rows_to_write = index_granularity.getMarkRows(from_mark); + + std::cerr << "(MergeTreeDataPartWriterCompact::write) rows_to_write: " << rows_to_write << "\n"; if (rows_to_write) data_written = true; - // if (current_row == 0 && index_offset != 0) - // { - // rows_to_write = index_offset; - // write_marks = false; - // } - // else - // { - // rows_to_write = index_granularity.getMarkRows(current_mark); - // } - - // std::cerr << "(MergeTreeDataPartWriterCompact::write) rows_to_write: " << rows_to_write << "\n"; - /// There could already be enough data to compress into the new block. if (stream->compressed.offset() >= settings.min_compress_block_size) stream->compressed.next(); size_t next_row = 0; - if (write_marks) + writeIntBinary(rows_to_write, stream->marks); + for (const auto & it : columns_list) { - writeIntBinary(rows_to_write, stream->marks); - for (size_t i = 0; i < columns_to_write.size(); ++i) - { - writeIntBinary(stream->plain_hashing.count(), stream->marks); - writeIntBinary(stream->compressed.offset(), stream->marks); - next_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); - } - ++from_mark; - } - else - { - for (size_t i = 0; i < columns_to_write.size(); ++i) - next_row = writeColumnSingleGranule(columns_to_write[i], current_row, rows_to_write); + writeIntBinary(stream->plain_hashing.count(), stream->marks); + writeIntBinary(stream->compressed.offset(), stream->marks); + next_row = writeColumnSingleGranule(block.getByName(it.name), current_row, rows_to_write); } + ++from_mark; current_row = next_row; } @@ -125,6 +118,7 @@ void MergeTreeDataPartWriterCompact::write( next_index_offset = total_rows - current_row; } + size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) { std::cerr << "(writeColumnSingleGranule) writing column: " << column.name << "\n"; @@ -146,7 +140,11 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith } void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) -{ +{ + auto result = squashing.add({}); + if (result.ready && !result.columns.empty()) + writeBlock(header.cloneWithColumns(std::move(result.columns))); + if (with_final_mark && data_written) { writeIntBinary(0ULL, stream->marks); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index efe84182640..1c56bf75ade 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -1,4 +1,5 @@ #include +#include namespace DB { @@ -28,7 +29,12 @@ private: size_t from_row, size_t number_of_rows); + void writeBlock(const Block & block); + ColumnStreamPtr stream; + + SquashingTransform squashing; + Block header; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 7b17bcb2ec8..3ce805b0e65 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -243,7 +243,7 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( while (current_row < total_rows) { size_t rows_to_write; - bool write_marks = true; /// FIXME not always true + bool write_marks = true; /// If there is `index_offset`, then the first mark goes not immediately, but after this number of rows. if (current_row == 0 && index_offset != 0) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp index b2dd5141df3..4901900b6ba 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.cpp @@ -48,6 +48,14 @@ void MergeTreeIndexGranularity::appendMark(size_t rows_count) marks_rows_partial_sums.push_back(marks_rows_partial_sums.back() + rows_count); } +void MergeTreeIndexGranularity::addRowsToLastMark(size_t rows_count) +{ + if (marks_rows_partial_sums.empty()) + marks_rows_partial_sums.push_back(rows_count); + else + marks_rows_partial_sums.back() += rows_count; +} + void MergeTreeIndexGranularity::popMark() { if (!marks_rows_partial_sums.empty()) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h index a2d78596279..053eebccb25 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -92,6 +92,8 @@ public: /// Add new mark with rows_count void appendMark(size_t rows_count); + void addRowsToLastMark(size_t rows_count); + void popMark(); /// Add `size` of marks with `fixed_granularity` rows diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 2eb53f9280d..79850ea6101 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -31,6 +31,11 @@ size_t MergeTreeRangeReader::DelayedStream::readRows(Block & block, size_t num_r { if (num_rows) { + std::cerr << "(DelayedStream::readRows) current_mark: " << current_mark << '\n'; + std::cerr << "(DelayedStream::readRows) continue_reading: " << continue_reading << '\n'; + std::cerr << "(DelayedStream::readRows) num_rows: " << num_rows << '\n'; + + size_t rows_read = merge_tree_reader->readRows(current_mark, continue_reading, num_rows, block); continue_reading = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 517c6212b1b..84d87954cdf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -54,43 +54,57 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, UNUSED(res); /// FIXME compute correct granularity - size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); + std::cerr << "(MergeTreeReaderCompact::readRows) max_rows_to_read: " << max_rows_to_read << "\n"; + size_t read_rows = 0; - for (const auto & it : columns) + while (read_rows < max_rows_to_read) { - bool append = res.has(it.name); - if (!append) - res.insert(ColumnWithTypeAndName(it.type->createColumn(), it.type, it.name)); + size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); - /// To keep offsets shared. TODO Very dangerous. Get rid of this. - MutableColumnPtr column = res.getByName(it.name).column->assumeMutable(); + std::cerr << "(MergeTreeReaderCompact::readRows) rows_to_read: " << rows_to_read << "\n"; - try + for (const auto & it : columns) { - size_t column_size_before_reading = column->size(); - size_t column_position = data_part->getColumnPosition(it.name); + bool append = res.has(it.name); + if (!append) + res.insert(ColumnWithTypeAndName(it.type->createColumn(), it.type, it.name)); - readData(it.name, *it.type, *column, from_mark, column_position, rows_to_read); + /// To keep offsets shared. TODO Very dangerous. Get rid of this. + MutableColumnPtr column = res.getByName(it.name).column->assumeMutable(); + + try + { + // size_t column_size_before_reading = column->size(); + size_t column_position = data_part->getColumnPosition(it.name); + + readData(it.name, *it.type, *column, from_mark, column_position, rows_to_read); + + /// For elements of Nested, column_size_before_reading may be greater than column size + /// if offsets are not empty and were already read, but elements are empty. + /// FIXME + // if (column->size()) + // read_rows_in_mark = std::max(read_rows, column->size() - column_size_before_reading); + } + catch (Exception & e) + { + /// Better diagnostics. + e.addMessage("(while reading column " + it.name + ")"); + throw; + } - /// For elements of Nested, column_size_before_reading may be greater than column size - /// if offsets are not empty and were already read, but elements are empty. if (column->size()) - read_rows = std::max(read_rows, column->size() - column_size_before_reading); - } - catch (Exception & e) - { - /// Better diagnostics. - e.addMessage("(while reading column " + it.name + ")"); - throw; + res.getByName(it.name).column = std::move(column); + else + res.erase(it.name); } - if (column->size()) - res.getByName(it.name).column = std::move(column); - else - res.erase(it.name); + ++from_mark; + read_rows += rows_to_read; } + std::cerr << "(MergeTreeReaderCompact::readRows) read_rows: " << read_rows << "\n"; + return read_rows; } @@ -152,7 +166,7 @@ void MergeTreeReaderCompact::initMarksLoader() auto res = std::make_shared(marks_count * columns_num); - std::cerr << "(MergeTreeReaderCompact::loadMarks) marks_count: " << marks_count << "\n"; + // std::cerr << "(MergeTreeReaderCompact::loadMarks) marks_count: " << marks_count << "\n"; ReadBufferFromFile buffer(mrk_path, file_size); size_t i = 0; @@ -161,13 +175,13 @@ void MergeTreeReaderCompact::initMarksLoader() { buffer.seek(sizeof(size_t), SEEK_CUR); buffer.readStrict(reinterpret_cast(res->data() + i * columns_num), sizeof(MarkInCompressedFile) * columns_num); - std::cerr << "(MergeTreeReaderCompact::loadMarks) i: " << i << "\n"; - std::cerr << "(MergeTreeReaderCompact::loadMarks) buffer pos in file: " << buffer.getPositionInFile() << "\n"; + // std::cerr << "(MergeTreeReaderCompact::loadMarks) i: " << i << "\n"; + // std::cerr << "(MergeTreeReaderCompact::loadMarks) buffer pos in file: " << buffer.getPositionInFile() << "\n"; ++i; } - std::cerr << "(MergeTreeReaderCompact::loadMarks) file_size: " << file_size << "\n"; - std::cerr << "(MergeTreeReaderCompact::loadMarks) correct file size: " << i * mark_size_in_bytes << "\n"; + // std::cerr << "(MergeTreeReaderCompact::loadMarks) file_size: " << file_size << "\n"; + // std::cerr << "(MergeTreeReaderCompact::loadMarks) correct file size: " << i * mark_size_in_bytes << "\n"; if (i * mark_size_in_bytes != file_size) throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); From 55deeea608814e3b240c1a96564f7192996f7f8e Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 27 Nov 2019 22:57:07 +0300 Subject: [PATCH 0066/2007] polymorphic parts (development) --- dbms/src/DataStreams/SquashingTransform.h | 2 - .../MergeTree/IMergeTreeDataPartWriter.cpp | 17 ++++----- .../MergeTree/IMergeTreeDataPartWriter.h | 1 - .../MergeTree/MergeTreeDataPartCompact.cpp | 6 ++- .../MergeTreeDataPartWriterCompact.cpp | 37 ++++++++++++------- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 2 +- 6 files changed, 37 insertions(+), 28 deletions(-) diff --git a/dbms/src/DataStreams/SquashingTransform.h b/dbms/src/DataStreams/SquashingTransform.h index de6f1cc8383..f1681c57c8c 100644 --- a/dbms/src/DataStreams/SquashingTransform.h +++ b/dbms/src/DataStreams/SquashingTransform.h @@ -40,8 +40,6 @@ public: */ Result add(MutableColumns && columns); - bool hasPendingData() { return !accumulated_columns.empty(); } - private: size_t min_block_size_rows; size_t min_block_size_bytes; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 5826fb0b1f8..c5a80c25b9d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -131,17 +131,16 @@ void fillIndexGranularityImpl( size_t current_row; for (current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) { - size_t rows_rest_in_block = rows_in_block - current_row; - std::cerr << "rows_rest_in_block: " << rows_rest_in_block << "\n"; - std::cerr << "rows_rest_in_block: " << index_granularity_for_block << "\n"; - - /// FIXME may be remove need_finish_last_granule and do it always - if (need_finish_last_granule && rows_rest_in_block < index_granularity_for_block) + size_t rows_left_in_block = rows_in_block - current_row; + + if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block) { - if (rows_rest_in_block * 2 >= index_granularity_for_block) - index_granularity.appendMark(rows_rest_in_block); + /// If enough rows are left, create a new granule. Otherwise, extend previous granule. + /// So,real size of granule differs from index_granularity_for_block not more than 50%. + if (rows_left_in_block * 2 >= index_granularity_for_block) + index_granularity.appendMark(rows_left_in_block); else - index_granularity.addRowsToLastMark(rows_rest_in_block); + index_granularity.addRowsToLastMark(rows_left_in_block); } else { diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index e3deeea859e..c182ca072fe 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -69,7 +69,6 @@ public: const MergeTreeIndexGranularity & index_granularity, bool need_finish_last_granule); - /// FIXME remove indices block virtual void write( const Block & block, const IColumn::Permutation * permutation, /* Blocks with already sorted index columns */ diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 396a3490c3d..38193bbff74 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -186,9 +186,13 @@ void MergeTreeDataPartCompact::loadIndexGranularity() readIntBinary(granularity, buffer); index_granularity.appendMark(granularity); /// Skip offsets for columns - buffer.seek(index_granularity_info.mark_size_in_bytes, SEEK_CUR); + buffer.seek(columns.size() * sizeof(MarkInCompressedFile), SEEK_CUR); } + std::cerr << "(loadIndexGranularity) marks: " << index_granularity.getMarksCount() << "\n"; + std::cerr << "(loadIndexGranularity) mark size: " << index_granularity_info.mark_size_in_bytes << "\n"; + std::cerr << "(loadIndexGranularity) marks file size: " << marks_file_size << "\n"; + if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index ab95bacafe9..f0d72e6072a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -39,23 +39,38 @@ void MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) { - UNUSED(primary_key_block); - UNUSED(skip_indexes_block); - if (!header) header = block.cloneEmpty(); - Block result_block = block; + /// Fill index granularity for this block + /// if it's unknown (in case of insert data or horizontal merge, + /// but not in case of vertical merge) + /// FIXME maybe it's wrong at this stage. + if (compute_granularity) + fillIndexGranularity(block); + + Block result_block; if (permutation) { - auto it = columns_list.begin(); - for (size_t i = 0; i < columns_list.size(); ++i) + for (const auto & it : columns_list) { - auto & column = result_block.getByName(it->name); - column.column = column.column->permute(*permutation, 0); + if (primary_key_block.has(it.name)) + result_block.insert(primary_key_block.getByName(it.name)); + else if (skip_indexes_block.has(it.name)) + result_block.insert(skip_indexes_block.getByName(it.name)); + else + { + auto column = block.getByName(it.name); + column.column = column.column->permute(*permutation, 0); + result_block.insert(column); + } } } + else + { + result_block = block; + } auto result = squashing.add(result_block.mutateColumns()); if (!result.ready) @@ -72,12 +87,6 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) size_t from_mark = current_mark; size_t current_row = 0; - /// Fill index granularity for this block - /// if it's unknown (in case of insert data or horizontal merge, - /// but not in case of vertical merge) - if (compute_granularity) - fillIndexGranularity(block); - std::cerr << "(MergeTreeDataPartWriterCompact::write) marks: " << index_granularity.getMarksCount() << "\n"; for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 3ce805b0e65..b44450ccae0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -104,7 +104,7 @@ void MergeTreeDataPartWriterWide::write(const Block & block, /// but not in case of vertical merge) if (compute_granularity) fillIndexGranularity(block); - + std::cerr << "(MergeTreeDataPartWriterWide::write) marks_count: " << index_granularity.getMarksCount() << "\n"; WrittenOffsetColumns offset_columns; From 7dbdbff74835157984f2f303c8e3cbb83d0b7d72 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 28 Nov 2019 23:14:41 +0300 Subject: [PATCH 0067/2007] polymorphic parts (development) --- dbms/src/DataTypes/DataTypeArray.cpp | 2 ++ .../MergeTree/IMergeTreeDataPartWriter.cpp | 12 ++++--- .../MergeTreeDataPartWriterCompact.cpp | 6 ++-- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 2 ++ .../MergeTree/MergeTreeDataSelectExecutor.cpp | 6 ++++ .../MergeTreeIndexGranularityInfo.cpp | 3 +- .../MergeTree/MergeTreeIndexGranularityInfo.h | 7 ++-- .../MergeTree/MergeTreeIndexReader.cpp | 1 + .../MergeTree/MergeTreeReaderCompact.cpp | 35 ++++++++++++------- .../MergeTree/MergeTreeReaderCompact.h | 2 ++ .../MergeTree/MergeTreeReaderStream.cpp | 14 ++++++-- .../MergeTree/MergeTreeReaderStream.h | 8 +++++ .../MergeTree/MergeTreeReaderWide.cpp | 6 ++-- .../MergeTreeSequentialBlockInputStream.cpp | 5 +++ 14 files changed, 80 insertions(+), 29 deletions(-) diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index dc4cddb5e62..a578fabe3df 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -265,6 +265,8 @@ void DataTypeArray::deserializeBinaryBulkWithMultipleStreams( /// Adjust value size hint. Divide it to the average array size. settings.avg_value_size_hint = nested_limit ? settings.avg_value_size_hint / nested_limit * offset_values.size() : 0; + std::cerr << "nested_limit: " << nested_limit << "\n"; + nested->deserializeBinaryBulkWithMultipleStreams(nested_column, nested_limit, settings, state); settings.path.pop_back(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index c5a80c25b9d..d81123cc32c 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -221,6 +221,8 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); /// Write index. The index contains Primary Key value for each `index_granularity` row. + + std::cerr << "writing index...\n"; for (size_t i = index_offset; i < rows;) { if (storage.hasPrimaryKey()) @@ -233,10 +235,12 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p } } - ++current_mark; - if (current_mark < index_granularity.getMarksCount()) - i += index_granularity.getMarkRows(current_mark); - else + std::cerr << "(index) i: " << i << "\n"; + std::cerr << "(index) current_mark: " << current_mark << "\n"; + std::cerr << "(index) rows in mark: " << index_granularity.getMarkRows(current_mark) << "\n"; + + i += index_granularity.getMarkRows(current_mark++); + if (current_mark >= index_granularity.getMarksCount()) break; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index f0d72e6072a..b1ef90dfd72 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -39,9 +39,6 @@ void MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) { - if (!header) - header = block.cloneEmpty(); - /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, /// but not in case of vertical merge) @@ -72,6 +69,9 @@ void MergeTreeDataPartWriterCompact::write( result_block = block; } + if (!header) + header = result_block.cloneEmpty(); + auto result = squashing.add(result_block.mutateColumns()); if (!result.ready) return; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index b44450ccae0..fd882d400b9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -106,6 +106,8 @@ void MergeTreeDataPartWriterWide::write(const Block & block, fillIndexGranularity(block); std::cerr << "(MergeTreeDataPartWriterWide::write) marks_count: " << index_granularity.getMarksCount() << "\n"; + std::cerr << "(MergeTreeDataPartWriterWide::write) current_mark: " << current_mark << "\n"; + WrittenOffsetColumns offset_columns; MarkWithOffset result; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8060159bf16..8e243e38e30 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -679,6 +679,12 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( const Settings & settings, const ReaderSettings & reader_settings) const { + std::cerr << "marks to read: "; + for (const auto & part : parts) + for (auto range : part.ranges) + std::cerr << "(" << range.begin << ", " << range.end << ") "; + + /// Count marks for each part. std::vector sum_marks_in_parts(parts.size()); size_t sum_marks = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index e8e3a12a73b..f059665d754 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -62,6 +62,7 @@ void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_, { is_adaptive = true; mark_size_in_bytes = getAdaptiveMrkSize(part_type, columns_num); + skip_index_mark_size_in_bytes = sizeof(MarkInCompressedFile) + sizeof(UInt64); marks_file_extension = getAdaptiveMrkExtension(part_type); index_granularity_bytes = index_granularity_bytes_; } @@ -69,7 +70,7 @@ void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_, void MergeTreeIndexGranularityInfo::setNonAdaptive() { is_adaptive = false; - mark_size_in_bytes = getNonAdaptiveMrkSize(); + mark_size_in_bytes = skip_index_mark_size_in_bytes = getNonAdaptiveMrkSize(); marks_file_extension = getNonAdaptiveMrkExtension(); index_granularity_bytes = 0; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index b2333813886..d974d2d88f2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -17,7 +18,9 @@ public: String marks_file_extension; /// Size of one mark in file two or three size_t numbers - UInt16 mark_size_in_bytes = 0; + UInt32 mark_size_in_bytes = 0; + + UInt8 skip_index_mark_size_in_bytes = 0; /// Is stride in rows between marks non fixed? bool is_adaptive = false; @@ -53,7 +56,7 @@ private: }; constexpr inline auto getNonAdaptiveMrkExtension() { return ".mrk"; } -constexpr inline auto getNonAdaptiveMrkSize() { return sizeof(UInt64) * 2; } +constexpr inline auto getNonAdaptiveMrkSize() { return sizeof(MarkInCompressedFile) * 2; } inline std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp index af899bba402..30e9fa050af 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -12,6 +12,7 @@ MergeTreeIndexReader::MergeTreeIndexReader( { 0, DBMS_DEFAULT_BUFFER_SIZE, false}, nullptr, nullptr, part_->getFileSizeOrZero(index->getFileName() + ".idx"), &part_->index_granularity_info, + MergeTreeReaderStream::ReadingMode::INDEX, ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE) { stream.seekToStart(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 84d87954cdf..52936397d63 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -56,14 +56,17 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, /// FIXME compute correct granularity std::cerr << "(MergeTreeReaderCompact::readRows) max_rows_to_read: " << max_rows_to_read << "\n"; - size_t read_rows = 0; + std::cerr << "(MergeTreeReaderCompact::readRows) from_mark: " << from_mark << "\n"; + std::cerr << "(MergeTreeReaderCompact::readRows) continue_reading: " << continue_reading << "\n"; + if (continue_reading) + from_mark = next_mark; + + size_t read_rows = 0; while (read_rows < max_rows_to_read) { size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); - std::cerr << "(MergeTreeReaderCompact::readRows) rows_to_read: " << rows_to_read << "\n"; - for (const auto & it : columns) { bool append = res.has(it.name); @@ -75,11 +78,16 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, try { - // size_t column_size_before_reading = column->size(); + size_t column_size_before_reading = column->size(); size_t column_position = data_part->getColumnPosition(it.name); readData(it.name, *it.type, *column, from_mark, column_position, rows_to_read); + size_t read_rows_in_column = column->size() - column_size_before_reading; + if (read_rows_in_column < rows_to_read) + throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) + + ". Rows expected: "+ toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA); + /// For elements of Nested, column_size_before_reading may be greater than column size /// if offsets are not empty and were already read, but elements are empty. /// FIXME @@ -101,9 +109,13 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, ++from_mark; read_rows += rows_to_read; + + std::cerr << "(MergeTreeReaderCompact::readRows) cur mark: " << from_mark << "\n"; + std::cerr << "(MergeTreeReaderCompact::readRows) read_rows: " << read_rows << "\n"; + std::cerr << "(MergeTreeReaderCompact::readRows) rows_to_read: " << rows_to_read << "\n"; } - std::cerr << "(MergeTreeReaderCompact::readRows) read_rows: " << read_rows << "\n"; + next_mark = from_mark; return read_rows; } @@ -118,23 +130,20 @@ void MergeTreeReaderCompact::readData( std::cerr << "(MergeTreeReaderCompact::readData) rows_to_read: " << rows_to_read << "\n"; std::cerr << "(MergeTreeReaderCompact::readData) start reading column: " << name << "\n"; + /// FIXME seek only if needed seekToMark(from_mark, column_position); IDataType::DeserializeBinaryBulkSettings deserialize_settings; deserialize_settings.getter = [&](IDataType::SubstreamPath) -> ReadBuffer * { return data_buffer; }; - deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; + // deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; deserialize_settings.position_independent_encoding = false; IDataType::DeserializeBinaryBulkStatePtr state; type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); type.deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state); - std::cerr << "(MergeTreeReaderCompact::readData) end reading column rows: " << column.size() << "\n"; - std::cerr << "(MergeTreeReaderCompact::readData) end reading column: " << name << "\n"; - - // if (column.size() != rows_to_read) - // throw Exception("Cannot read all data in NativeBlockInputStream. Rows read: " + toString(column.size()) + ". Rows expected: "+ toString(rows_to_read) + ".", - // ErrorCodes::CANNOT_READ_ALL_DATA); + // std::cerr << "(MergeTreeReaderCompact::readData) end reading column rows: " << column.size() << "\n"; + // std::cerr << "(MergeTreeReaderCompact::readData) end reading column: " << name << "\n"; } @@ -198,7 +207,7 @@ void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) { MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); - std::cerr << "(MergeTreeReaderCompact::seekToMark) mark: (" << mark.offset_in_compressed_file << ", " << mark.offset_in_decompressed_block << "\n"; + // std::cerr << "(MergeTreeReaderCompact::seekToMark) mark: (" << mark.offset_in_compressed_file << ", " << mark.offset_in_decompressed_block << "\n"; try { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 49bc9e25c38..d6e148be1dc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -33,6 +33,8 @@ private: MergeTreeMarksLoader marks_loader; + size_t next_mark = 0; + void initMarksLoader(); void seekToStart(); void seekToMark(size_t row, size_t col); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 8fde48c6791..793d3b25dcb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -15,16 +15,18 @@ namespace ErrorCodes MergeTreeReaderStream::MergeTreeReaderStream( - const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, + const String & path_prefix_,const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const ReaderSettings & settings, MarkCache * mark_cache_, UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, + ReadingMode mode_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type) : path_prefix(path_prefix_), data_file_extension(data_file_extension_), marks_count(marks_count_) , mark_cache(mark_cache_), save_marks_in_cache(settings.save_marks_in_cache) , index_granularity_info(index_granularity_info_) + , mode(mode_) { /// Compute the size of the buffer. size_t max_mark_range_bytes = 0; @@ -115,8 +117,14 @@ void MergeTreeReaderStream::initMarksLoader() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); + std::cerr << "data_file_extension: " << data_file_extension << '\n'; + size_t file_size = Poco::File(mrk_path).getSize(); - size_t expected_file_size = index_granularity_info->mark_size_in_bytes * marks_count; + size_t mark_size = mode == ReadingMode::INDEX + ? index_granularity_info->skip_index_mark_size_in_bytes + : index_granularity_info->mark_size_in_bytes; + + size_t expected_file_size = mark_size * marks_count; if (expected_file_size != file_size) throw Exception( "Bad size of marks file '" + mrk_path + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), @@ -143,7 +151,7 @@ void MergeTreeReaderStream::initMarksLoader() buffer.seek(sizeof(size_t), SEEK_CUR); ++i; } - if (i * index_granularity_info->mark_size_in_bytes != file_size) + if (i * mark_size != file_size) throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); } res->protect(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index d7961e38d3d..7c6d8f956af 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -16,12 +16,19 @@ namespace DB class MergeTreeReaderStream { public: + enum class ReadingMode + { + COLUMN, + INDEX, + }; + MergeTreeReaderStream( const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const ReaderSettings & settings_, MarkCache * mark_cache, UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, + ReadingMode mode_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); void seekToMark(size_t index); @@ -42,6 +49,7 @@ private: bool save_marks_in_cache; const MergeTreeIndexGranularityInfo * index_granularity_info; + ReadingMode mode; std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 348d6908847..4f7cc48026a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -50,9 +50,8 @@ MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) { - std::cerr << "(MergeTreeReaderWide::readRows) columns: " << columns.toString() << "\n"; - std::cerr << "(MergeTreeReaderWide::readRows) from_rows: " << from_mark << "\n"; - std::cerr << "(MergeTreeReaderWide::readRows) block: " << res.dumpStructure() << "\n"; + std::cerr << "(MergeTreeReaderWide::readRows) from_mark: " << from_mark << "\n"; + std::cerr << "(MergeTreeReaderWide::readRows) continue_reading: " << continue_reading << "\n"; size_t read_rows = 0; try @@ -169,6 +168,7 @@ void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), &data_part->index_granularity_info, + MergeTreeReaderStream::ReadingMode::COLUMN, profile_callback, clock_type)); }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 1db67ee2e47..545f0891846 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -101,6 +101,11 @@ try bool continue_reading = (current_mark != 0); size_t rows_readed = reader->readRows(current_mark, continue_reading, rows_to_read, res); + std::cerr << "(MergeTreeSequentialBlockInputStream) rows_to_read: " << rows_to_read << '\n'; + std::cerr << "(MergeTreeSequentialBlockInputStream) current_mark: " << current_mark << '\n'; + std::cerr << "(MergeTreeSequentialBlockInputStream) rows_readed: " << rows_readed << '\n'; + + if (res) { res.checkNumberOfRows(); From a3875a6ca2b2a294a9cffd2bdb5c7af640bfa092 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 2 Dec 2019 18:21:07 +0300 Subject: [PATCH 0068/2007] polymorphic parts (development) --- .../AggregateFunctionFactory.cpp | 2 +- dbms/src/DataStreams/SplittingTransform.h | 89 +++++++++++++++++++ dbms/src/DataTypes/DataTypeArray.cpp | 2 - .../MergeTree/IMergeTreeDataPartWriter.cpp | 25 +++--- .../MergeTreeDataPartWriterCompact.cpp | 17 +--- .../MergeTree/MergeTreeReaderCompact.cpp | 2 +- 6 files changed, 110 insertions(+), 27 deletions(-) create mode 100644 dbms/src/DataStreams/SplittingTransform.h diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index 0c9a3b1ad63..e939934b7d1 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -24,8 +24,8 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_AGGREGATE_FUNCTION; - extern const int LOGICAL_ERROR; } + extern const int LOGICAL_ERROR; void AggregateFunctionFactory::registerFunction(const String & name, Creator creator, CaseSensitiveness case_sensitiveness) diff --git a/dbms/src/DataStreams/SplittingTransform.h b/dbms/src/DataStreams/SplittingTransform.h new file mode 100644 index 00000000000..8cd563b89a7 --- /dev/null +++ b/dbms/src/DataStreams/SplittingTransform.h @@ -0,0 +1,89 @@ +#pragma once + +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int EMPTY_LIST_OF_COLUMNS_PASSED; + extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; +} + +class SplittingTransform +{ +public: + SplittingTransform() {} + + void add(MutableColumns && columns) + { + assertColumnsHasEqualRows(columns); + blocks_queue.push(std::move(columns)); + } + + MutableColumns get(size_t preferable_size) + { + if (blocks_queue.empty()) + return {}; + + auto & columns = blocks_queue.front(); + size_t rows = columns[0]->size(); + + MutableColumns result; + if (rows <= preferable_size && current_offset == 0) + { + result = std::move(columns); + blocks_queue.pop(); + current_offset = 0; + } + else + { + result.reserve(columns.size()); + size_t result_size = std::min(rows - current_offset, preferable_size); + for (const auto & column : columns) + result.push_back((*column->cut(current_offset, result_size)).mutate()); + + current_offset += result_size; + } + + if (current_offset == rows) + { + blocks_queue.pop(); + current_offset = 0; + } + + return result; + } + + MutableColumns addAndGet(MutableColumns && columns, size_t preferable_size) + { + add(std::move(columns)); + return get(preferable_size); + } + +private: + + void assertColumnsHasEqualRows(const MutableColumns & columns) + { + if (columns.empty()) + throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); + + size_t rows = 0; + for (const auto & column : columns) + { + if (!rows) + rows = column->size(); + else if (column->size() != rows) + throw Exception("Sizes of columns doesn't match", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); + } + } + + // size_t max_block_size_rows = 0; + // size_t max_block_size_bytes = 0; + std::queue blocks_queue; + size_t current_offset = 0; +}; + +} diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index a578fabe3df..dc4cddb5e62 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -265,8 +265,6 @@ void DataTypeArray::deserializeBinaryBulkWithMultipleStreams( /// Adjust value size hint. Divide it to the average array size. settings.avg_value_size_hint = nested_limit ? settings.avg_value_size_hint / nested_limit * offset_values.size() : 0; - std::cerr << "nested_limit: " << nested_limit << "\n"; - nested->deserializeBinaryBulkWithMultipleStreams(nested_column, nested_limit, settings, state); settings.path.pop_back(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index d81123cc32c..dfbf09cee0c 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -133,19 +133,24 @@ void fillIndexGranularityImpl( { size_t rows_left_in_block = rows_in_block - current_row; + // if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block) + // { + // /// If enough rows are left, create a new granule. Otherwise, extend previous granule. + // /// So,real size of granule differs from index_granularity_for_block not more than 50%. + // if (rows_left_in_block * 2 >= index_granularity_for_block) + // index_granularity.appendMark(rows_left_in_block); + // else + // index_granularity.addRowsToLastMark(rows_left_in_block); + // } + // else + // { + // index_granularity.appendMark(index_granularity_for_block); + // } + if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block) - { - /// If enough rows are left, create a new granule. Otherwise, extend previous granule. - /// So,real size of granule differs from index_granularity_for_block not more than 50%. - if (rows_left_in_block * 2 >= index_granularity_for_block) - index_granularity.appendMark(rows_left_in_block); - else - index_granularity.addRowsToLastMark(rows_left_in_block); - } + index_granularity.appendMark(rows_left_in_block); else - { index_granularity.appendMark(index_granularity_for_block); - } } for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index b1ef90dfd72..35d12863677 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -23,7 +23,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_, true) - , squashing(storage.getSettings()->index_granularity, storage.getSettings()->index_granularity_bytes) + , squashing(storage.getSettings()->index_granularity, storage.getSettings()->index_granularity_bytes) /// FIXME { stream = std::make_unique( DATA_FILE_NAME, @@ -39,6 +39,8 @@ void MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) { + std::cerr << "(MergeTreeDataPartWriterCompact::write) block111: " << block.dumpStructure() << "\n"; + /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, /// but not in case of vertical merge) @@ -87,21 +89,10 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) size_t from_mark = current_mark; size_t current_row = 0; - std::cerr << "(MergeTreeDataPartWriterCompact::write) marks: " << index_granularity.getMarksCount() << "\n"; - - for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) - std::cerr << "rows in mark: " << index_granularity.getMarkRows(i) << "\n"; - - std::cerr << "(MergeTreeDataPartWriterCompact::write) total_rows: " << total_rows << "\n"; - while (current_row < total_rows) { - std::cerr << "(MergeTreeDataPartWriterCompact::write) current_row: " << current_row << "\n"; - size_t rows_to_write = index_granularity.getMarkRows(from_mark); - std::cerr << "(MergeTreeDataPartWriterCompact::write) rows_to_write: " << rows_to_write << "\n"; - if (rows_to_write) data_written = true; @@ -138,7 +129,7 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.getter = [this](IDataType::SubstreamPath) -> WriteBuffer * { return &stream->compressed; }; - serialize_settings.position_independent_encoding = false; + serialize_settings.position_independent_encoding = true; serialize_settings.low_cardinality_max_dictionary_size = 0; column.type->serializeBinaryBulkStatePrefix(serialize_settings, state); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 52936397d63..c0d2094865e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -136,7 +136,7 @@ void MergeTreeReaderCompact::readData( IDataType::DeserializeBinaryBulkSettings deserialize_settings; deserialize_settings.getter = [&](IDataType::SubstreamPath) -> ReadBuffer * { return data_buffer; }; // deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; - deserialize_settings.position_independent_encoding = false; + deserialize_settings.position_independent_encoding = true; IDataType::DeserializeBinaryBulkStatePtr state; type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); From 511ae82e275a73cf4b35985d6e689328fb720c6d Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 2 Dec 2019 20:10:22 +0300 Subject: [PATCH 0069/2007] polymorphic parts (development) fix adjust last granule --- .../AggregateFunctionFactory.cpp | 2 +- .../src/Storages/MergeTree/IMergeTreeReader.h | 2 ++ .../MergeTree/MergeTreeRangeReader.cpp | 24 ++++++++++++++++--- .../Storages/MergeTree/MergeTreeRangeReader.h | 1 + .../MergeTree/MergeTreeReaderCompact.h | 2 ++ .../Storages/MergeTree/MergeTreeReaderWide.h | 2 ++ 6 files changed, 29 insertions(+), 4 deletions(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index e939934b7d1..0c9a3b1ad63 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -24,8 +24,8 @@ namespace DB namespace ErrorCodes { extern const int UNKNOWN_AGGREGATE_FUNCTION; -} extern const int LOGICAL_ERROR; +} void AggregateFunctionFactory::registerFunction(const String & name, Creator creator, CaseSensitiveness case_sensitiveness) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index df92f89b4e6..109146b6cb2 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -31,6 +31,8 @@ public: /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark virtual size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) = 0; + virtual bool canReadIncompleteGranules() const = 0; + virtual ~IMergeTreeReader(); const ValueSizeMap & getAvgValueSizeHints() const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 79850ea6101..75d88635ede 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -472,6 +472,19 @@ size_t MergeTreeRangeReader::Stream::numPendingRows() const return rows_between_marks - offset_after_current_mark; } + +size_t MergeTreeRangeReader::Stream::ceilRowsToCompleteGranules(size_t rows_num) const +{ + /// FIXME suboptimal + size_t result = 0; + size_t from_mark = current_mark; + while (result < rows_num && from_mark < last_mark) + result += index_granularity->getMarkRows(from_mark++); + + return result; +} + + bool MergeTreeRangeReader::isCurrentRangeFinished() const { return prev_reader ? prev_reader->isCurrentRangeFinished() : stream.isFinished(); @@ -595,14 +608,19 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t ranges.pop_back(); } - auto rows_to_read = std::min(space_left, stream.numPendingRowsInCurrentGranule()); + size_t current_space = space_left; - std::cerr << "(startReadingChain) rows_to_read: " << rows_to_read << "\n"; + /// If reader can't read part of granule, we have to increase number of reading rows + /// to read complete granules and exceed max_rows a bit. + if (!merge_tree_reader->canReadIncompleteGranules()) + current_space = stream.ceilRowsToCompleteGranules(space_left); + + auto rows_to_read = std::min(current_space, stream.numPendingRowsInCurrentGranule()); bool last = rows_to_read == space_left; result.addRows(stream.read(result.block, rows_to_read, !last)); result.addGranule(rows_to_read); - space_left -= rows_to_read; + space_left = (rows_to_read > space_left ? 0 : space_left - rows_to_read); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h index e0161cc2421..df41fe30c17 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -114,6 +114,7 @@ public: void checkEnoughSpaceInCurrentGranule(size_t num_rows) const; size_t readRows(Block & block, size_t num_rows); void toNextMark(); + size_t ceilRowsToCompleteGranules(size_t rows_num) const; }; /// Statistics after next reading step. diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index d6e148be1dc..909f0b3c6c8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -26,6 +26,8 @@ public: /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) override; + bool canReadIncompleteGranules() const override { return false; } + private: ReadBuffer * data_buffer; std::unique_ptr cached_buffer; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h index 782f79ead56..37266e83164 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -28,6 +28,8 @@ public: /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) override; + bool canReadIncompleteGranules() const override { return true; } + private: using FileStreams = std::map>; From be0e13d28f42d2565b3e585843a107ee484ed40b Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 3 Dec 2019 03:23:11 +0300 Subject: [PATCH 0070/2007] polymorphic parts (development) columns sizes --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 26 +--- .../Storages/MergeTree/IMergeTreeDataPart.h | 27 ++--- .../MergeTree/MergeTreeDataPartCompact.cpp | 111 ++++++++++-------- .../MergeTree/MergeTreeDataPartCompact.h | 11 +- .../MergeTree/MergeTreeDataPartWide.cpp | 17 +++ .../MergeTree/MergeTreeDataPartWide.h | 7 +- .../MergeTree/MergeTreeReaderCompact.cpp | 2 +- 7 files changed, 108 insertions(+), 93 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 61500997ddd..577d13df4b3 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -163,24 +163,6 @@ IMergeTreeDataPart::IMergeTreeDataPart( } -ColumnSize IMergeTreeDataPart::getColumnSize(const String & column_name, const IDataType & type) const -{ - return getColumnSizeImpl(column_name, type, nullptr); -} - -ColumnSize IMergeTreeDataPart::getTotalColumnsSize() const -{ - ColumnSize totals; - std::unordered_set processed_substreams; - for (const NameAndTypePair & column : columns) - { - ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams); - totals.add(size); - } - return totals; -} - - String IMergeTreeDataPart::getNewName(const MergeTreePartInfo & new_part_info) const { if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) @@ -198,9 +180,11 @@ String IMergeTreeDataPart::getNewName(const MergeTreePartInfo & new_part_info) c return new_part_info.getPartName(); } -size_t IMergeTreeDataPart::getColumnPosition(const String & column_name) const +std::optional IMergeTreeDataPart::getColumnPosition(const String & column_name) const { - return sample_block.getPositionByName(column_name); + if (!sample_block.has(column_name)) + return {}; + return sample_block.getPositionByName(column_name); } DayNum IMergeTreeDataPart::getMinDate() const @@ -513,7 +497,7 @@ void IMergeTreeDataPart::loadRowsCount() if (!column_col->isFixedAndContiguous() || column_col->lowCardinality()) continue; - size_t column_size = getColumnSizeImpl(column.name, *column.type, nullptr).data_uncompressed; + size_t column_size = getColumnSize(column.name, *column.type).data_uncompressed; if (!column_size) continue; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index a6af875a334..0f355318f2c 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -52,6 +52,8 @@ public: using MergeTreeReaderPtr = std::unique_ptr; using MergeTreeWriterPtr = std::unique_ptr; + using ColumnSizeByName = std::unordered_map; + virtual MergeTreeReaderPtr getReader( const NamesAndTypesList & columns_, const MarkRanges & mark_ranges, @@ -70,23 +72,22 @@ public: virtual bool isStoredOnDisk() const = 0; - void remove() const; virtual bool supportsVerticalMerge() const { return false; } /// NOTE: Returns zeros if column files are not found in checksums. /// NOTE: You must ensure that no ALTERs are in progress when calculating ColumnSizes. /// (either by locking columns_lock, or by locking table structure). - ColumnSize getColumnSize(const String & name, const IDataType & type); + virtual ColumnSize getColumnSize(const String & /* name */, const IDataType & /* type */) const { return {}; } - /// Initialize columns (from columns.txt if exists, or create from column files if not). - /// Load checksums from checksums.txt if exists. Load index if required. - void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency); + virtual ColumnSize getTotalColumnsSize() const { return {}; } /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. virtual String getColumnNameWithMinumumCompressedSize() const { return columns.front().name; } + virtual ~IMergeTreeDataPart(); + // virtual Checksums check( // bool require_checksums, // const DataTypes & primary_key_data_types, /// Check the primary key. If it is not necessary, pass an empty array. @@ -97,20 +98,14 @@ public: // } using ColumnToSize = std::map; - virtual void accumulateColumnSizes(ColumnToSize & column_to_size) const; using Type = MergeTreeDataPartType; - virtual Type getType() const = 0; - // virtual void renameTo() = 0; - static String typeToString(Type type); - String getTypeName() { return typeToString(getType()); } - virtual ~IMergeTreeDataPart(); IMergeTreeDataPart( const MergeTreeData & storage_, @@ -127,9 +122,11 @@ public: void assertOnDisk() const; - ColumnSize getColumnSize(const String & column_name, const IDataType & type) const; + void remove() const; - ColumnSize getTotalColumnsSize() const; + /// Initialize columns (from columns.txt if exists, or create from column files if not). + /// Load checksums from checksums.txt if exists. Load index if required. + void loadColumnsChecksumsIndexes(bool require_columns_checksums, bool check_consistency); String getMarksFileExtension() const { return index_granularity_info.marks_file_extension; } @@ -138,7 +135,7 @@ public: String getNewName(const MergeTreePartInfo & new_part_info) const; // Block sample_block; - size_t getColumnPosition(const String & column_name) const; + std::optional getColumnPosition(const String & column_name) const; bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } @@ -348,8 +345,6 @@ private: void loadPartitionAndMinMaxIndex(); String getRelativePathForDetachedPart(const String & prefix) const; - - virtual ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const = 0; }; using MergeTreeDataPartState = IMergeTreeDataPart::State; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 38193bbff74..ef70450a607 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -98,38 +98,28 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( default_codec, writer_settings, computed_index_granularity); } -/// Takes into account the fact that several columns can e.g. share their .size substreams. -/// When calculating totals these should be counted only once. -ColumnSize MergeTreeDataPartCompact::getColumnSizeImpl( - const String & column_name, const IDataType & type, std::unordered_set * processed_substreams) const +ColumnSize MergeTreeDataPartCompact::getColumnSize(const String & column_name, const IDataType & /* type */) const { - UNUSED(column_name); - UNUSED(type); - UNUSED(processed_substreams); - // ColumnSize size; - // if (checksums.empty()) - // return size; + auto column_size = columns_sizes.find(column_name); + if (column_size == columns_sizes.end()) + return {}; + return column_size->second; +} - // type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - // { - // String file_name = IDataType::getFileNameForStream(column_name, substream_path); - - // if (processed_substreams && !processed_substreams->insert(file_name).second) - // return; - - // auto bin_checksum = checksums.files.find(file_name + ".bin"); - // if (bin_checksum != checksums.files.end()) - // { - // size.data_compressed += bin_checksum->second.file_size; - // size.data_uncompressed += bin_checksum->second.uncompressed_size; - // } - - // auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); - // if (mrk_checksum != checksums.files.end()) - // size.marks += mrk_checksum->second.file_size; - // }, {}); - - return ColumnSize{}; +ColumnSize MergeTreeDataPartCompact::getTotalColumnsSize() const +{ + ColumnSize totals; + size_t marks_size = 0; + for (const auto & column : columns) + { + auto column_size = getColumnSize(column.name, *column.type); + totals.add(column_size); + if (!marks_size && column_size.marks) + marks_size = column_size.marks; + } + /// Marks are shared between all columns + totals.marks = marks_size; + return totals; } /** Returns the name of a column with minimum compressed size (as returned by getColumnSize()). @@ -137,29 +127,26 @@ ColumnSize MergeTreeDataPartCompact::getColumnSizeImpl( */ String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const { - /// FIXME: save column sizes + const auto & storage_columns = storage.getColumns().getAllPhysical(); + const std::string * minimum_size_column = nullptr; + UInt64 minimum_size = std::numeric_limits::max(); + for (const auto & column : storage_columns) + { + if (!getColumnPosition(column.name)) + continue; - // const auto & storage_columns = storage.getColumns().getAllPhysical(); - // const std::string * minimum_size_column = nullptr; - // UInt64 minimum_size = std::numeric_limits::max(); + auto size = getColumnSize(column.name, *column.type).data_compressed; + if (size < minimum_size) + { + minimum_size = size; + minimum_size_column = &column.name; + } + } - // for (const auto & column : storage_columns) - // { - // if (!hasColumnFiles(column.name, *column.type)) - // continue; - - // const auto size = getColumnSizeImpl(column.name, *column.type, nullptr).data_compressed; - // if (size < minimum_size) - // { - // minimum_size = size; - // minimum_size_column = &column.name; - // } - // } - - // if (!minimum_size_column) - // throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); - - return columns.front().name; + if (!minimum_size_column) + throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); + + return *minimum_size_column; } void MergeTreeDataPartCompact::loadIndexGranularity() @@ -199,6 +186,28 @@ void MergeTreeDataPartCompact::loadIndexGranularity() index_granularity.setInitialized(); } +void MergeTreeDataPartCompact::loadColumnSizes() +{ + size_t columns_num = columns.size(); + + if (columns_num == 0) + throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + auto column_sizes_path = getFullPath() + "columns_sizes.txt"; + auto columns_sizes_file = Poco::File(column_sizes_path); + if (!columns_sizes_file.exists()) + { + LOG_WARNING(storage.log, "No file column_sizes.txt in part " + name); + return; + } + + ReadBufferFromFile buffer(column_sizes_path, columns_sizes_file.getSize()); + auto it = columns.begin(); + for (size_t i = 0; i < columns_num; ++i, ++it) + readPODBinary(columns_sizes[it->name], buffer); + assertEOF(buffer); +} + void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { UNUSED(require_part_metadata); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 1aa030a194a..dcf43305300 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -69,7 +69,11 @@ public: /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; - virtual Type getType() const override { return Type::COMPACT; } + Type getType() const override { return Type::COMPACT; } + + ColumnSize getColumnSize(const String & name, const IDataType & type0) const override; + + ColumnSize getTotalColumnsSize() const override; void checkConsistency(bool /* require_part_metadata */) const override {} @@ -79,9 +83,12 @@ private: /// Loads marks index granularity into memory void loadIndexGranularity() override; - ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const override; + void loadColumnSizes(); + void checkConsistency(bool require_part_metadata); + + ColumnSizeByName columns_sizes; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index d214939967c..1c66ecad923 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -158,6 +158,23 @@ String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const return *minimum_size_column; } +ColumnSize MergeTreeDataPartWide::getTotalColumnsSize() const +{ + ColumnSize totals; + std::unordered_set processed_substreams; + for (const NameAndTypePair & column : columns) + { + ColumnSize size = getColumnSizeImpl(column.name, *column.type, &processed_substreams); + totals.add(size); + } + return totals; +} + +ColumnSize MergeTreeDataPartWide::getColumnSize(const String & column_name, const IDataType & type) const +{ + return getColumnSizeImpl(column_name, type, nullptr); +} + void MergeTreeDataPartWide::loadIndexGranularity() { String full_path = getFullPath(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index b1a31e96b98..da839c6b31e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -75,6 +75,10 @@ public: Type getType() const override { return Type::WIDE; } + ColumnSize getTotalColumnsSize() const override; + + ColumnSize getColumnSize(const String & column_name, const IDataType & type) const override; + ~MergeTreeDataPartWide() override; protected: @@ -84,8 +88,7 @@ private: /// Loads marks index granularity into memory void loadIndexGranularity() override; - ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const override; - + ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const; }; // using MergeTreeDataPartState =IMergeTreeDataPart::State; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index c0d2094865e..0cb328ce550 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -79,7 +79,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, try { size_t column_size_before_reading = column->size(); - size_t column_position = data_part->getColumnPosition(it.name); + auto column_position = *data_part->getColumnPosition(it.name); readData(it.name, *it.type, *column, from_mark, column_position, rows_to_read); From 31ffad0fb0f582942b97517deaa2ace6957e28d1 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 3 Dec 2019 17:33:56 +0300 Subject: [PATCH 0071/2007] polymorphic parts (development) columns sizes --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 24 +++++++++++++++++++ .../Storages/MergeTree/IMergeTreeDataPart.h | 5 ++++ .../MergeTree/IMergeTreeDataPartWriter.h | 7 ++++++ .../MergeTree/MergeTreeDataPartCompact.cpp | 22 ----------------- .../MergeTree/MergeTreeDataPartCompact.h | 5 ---- .../MergeTreeDataPartWriterCompact.cpp | 21 ++++++++++++---- .../MergeTree/MergedBlockOutputStream.cpp | 17 +++++++++++++ 7 files changed, 70 insertions(+), 31 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 577d13df4b3..662b4312a22 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -366,6 +366,7 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadChecksums(require_columns_checksums); loadIndexGranularity(); loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` + loadColumnSizes(); loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. loadPartitionAndMinMaxIndex(); loadTTLInfos(); @@ -588,6 +589,28 @@ void IMergeTreeDataPart::loadColumns(bool require) sample_block.insert({it.type, it.name}); } +void IMergeTreeDataPart::loadColumnSizes() +{ + size_t columns_num = columns.size(); + + if (columns_num == 0) + throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); + + auto column_sizes_path = getFullPath() + "columns_sizes.txt"; + auto columns_sizes_file = Poco::File(column_sizes_path); + if (!columns_sizes_file.exists()) + { + LOG_WARNING(storage.log, "No file column_sizes.txt in part " + name); + return; + } + + ReadBufferFromFile buffer(column_sizes_path, columns_sizes_file.getSize()); + auto it = columns.begin(); + for (size_t i = 0; i < columns_num; ++i, ++it) + readPODBinary(columns_sizes[it->name], buffer); + assertEOF(buffer); +} + UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) { @@ -638,6 +661,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ relative_path = new_relative_path; } + void IMergeTreeDataPart::remove() const { if (!isStoredOnDisk()) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 0f355318f2c..82e5e4b40a5 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -299,6 +299,8 @@ public: */ mutable std::mutex alter_mutex; + ColumnSizeByName columns_sizes; + /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; @@ -317,6 +319,7 @@ public: static UInt64 calculateTotalSizeOnDisk(const String & from); protected: + void removeIfNeeded(); virtual void checkConsistency(bool require_part_metadata) const; @@ -344,6 +347,8 @@ private: void loadPartitionAndMinMaxIndex(); + void loadColumnSizes(); + String getRelativePathForDetachedPart(const String & prefix) const; }; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index c182ca072fe..5b4c431565d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -88,6 +88,11 @@ public: return Columns(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); } + const MergeTreeData::ColumnSizeByName & getColumnsSizes() const + { + return columns_sizes; + } + void initSkipIndices(); void initPrimaryIndex(); void calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows); @@ -143,6 +148,8 @@ protected: bool data_written = false; bool primary_index_initialized = false; bool skip_indices_initialized = false; + + MergeTreeData::ColumnSizeByName columns_sizes; }; using MergeTreeWriterPtr = std::unique_ptr; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index ef70450a607..232571eb9ac 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -186,28 +186,6 @@ void MergeTreeDataPartCompact::loadIndexGranularity() index_granularity.setInitialized(); } -void MergeTreeDataPartCompact::loadColumnSizes() -{ - size_t columns_num = columns.size(); - - if (columns_num == 0) - throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - auto column_sizes_path = getFullPath() + "columns_sizes.txt"; - auto columns_sizes_file = Poco::File(column_sizes_path); - if (!columns_sizes_file.exists()) - { - LOG_WARNING(storage.log, "No file column_sizes.txt in part " + name); - return; - } - - ReadBufferFromFile buffer(column_sizes_path, columns_sizes_file.getSize()); - auto it = columns.begin(); - for (size_t i = 0; i < columns_num; ++i, ++it) - readPODBinary(columns_sizes[it->name], buffer); - assertEOF(buffer); -} - void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { UNUSED(require_part_metadata); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index dcf43305300..23ed067a449 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -83,12 +83,7 @@ private: /// Loads marks index granularity into memory void loadIndexGranularity() override; - void loadColumnSizes(); - - void checkConsistency(bool require_part_metadata); - - ColumnSizeByName columns_sizes; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 35d12863677..e5ad131694b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -104,11 +104,7 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) writeIntBinary(rows_to_write, stream->marks); for (const auto & it : columns_list) - { - writeIntBinary(stream->plain_hashing.count(), stream->marks); - writeIntBinary(stream->compressed.offset(), stream->marks); next_row = writeColumnSingleGranule(block.getByName(it.name), current_row, rows_to_write); - } ++from_mark; current_row = next_row; @@ -125,6 +121,13 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith std::cerr << "(writeColumnSingleGranule) from_row: " << from_row << "\n"; std::cerr << "(writeColumnSingleGranule) number_of_rows: " << number_of_rows << "\n"; + /// FIXME compressed size does not work + size_t old_compressed_size = stream->compressed_buf.getCompressedBytes() + stream->plain_hashing.count(); + size_t old_uncompressed_size = stream->compressed.count(); + + writeIntBinary(stream->plain_hashing.count(), stream->marks); + writeIntBinary(stream->compressed.offset(), stream->marks); + IDataType::SerializeBinaryBulkStatePtr state; IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -136,6 +139,12 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith column.type->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, state); column.type->serializeBinaryBulkStateSuffix(serialize_settings, state); + /// FIXME compressed size does not work + size_t compressed_size = stream->compressed_buf.getCompressedBytes() + stream->plain_hashing.count(); + size_t uncompressed_size = stream->compressed.count(); + + columns_sizes[column.name].add(ColumnSize{0, compressed_size - old_compressed_size, uncompressed_size - old_uncompressed_size}); + return from_row + number_of_rows; } @@ -155,6 +164,10 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: } } + size_t marks_size = stream->marks.count(); + for (auto it = columns_sizes.begin(); it != columns_sizes.end(); ++it) + it->second.marks = marks_size; + stream->finalize(); if (sync) stream->sync(); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 9cfc920c8e7..49ebfc979a8 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -132,6 +132,22 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( checksums.files["ttl.txt"].file_hash = out_hashing.getHash(); } + const auto & columns_sizes = writer->getColumnsSizes(); + if (!columns_sizes.empty()) + { + WriteBufferFromFile out(part_path + "columns_sizes.txt", 4096); + HashingWriteBuffer out_hashing(out); + for (const auto & column : columns_list) + { + auto it = columns_sizes.find(column.name); + if (it == columns_sizes.end()) + throw Exception("Not found size for column " + column.name, ErrorCodes::LOGICAL_ERROR); + writePODBinary(it->second, out_hashing); + checksums.files["columns_sizes.txt"].file_size = out_hashing.count(); + checksums.files["columns_sizes.txt"].file_hash = out_hashing.getHash(); + } + } + { /// Write a file with a description of columns. WriteBufferFromFile out(part_path + "columns.txt", 4096); @@ -151,6 +167,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->checksums = checksums; new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); new_part->index_granularity = writer->getIndexGranularity(); + new_part->columns_sizes = columns_sizes; std::cerr << "(writeSuffixAndFinalizePart) part: " << new_part->getFullPath() << "\n"; std::cerr << "(writeSuffixAndFinalizePart) marks_count: " << new_part->index_granularity.getMarksCount() << "\n"; } From 9df0d45d0b3de905b97f09907dc9688101c9f71b Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 5 Dec 2019 14:42:54 +0300 Subject: [PATCH 0072/2007] polymorphic parts (development) fix prewhere --- .../Storages/MergeTree/MergeTreeRangeReader.cpp | 16 +++++++++++----- .../Storages/MergeTree/MergeTreeRangeReader.h | 4 ++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 75d88635ede..0aa034579d6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -263,7 +263,7 @@ void MergeTreeRangeReader::ReadResult::clear() filter = nullptr; } -void MergeTreeRangeReader::ReadResult::optimize() +void MergeTreeRangeReader::ReadResult::optimize(bool can_read_incomplete_granules) { if (total_rows_per_granule == 0 || filter == nullptr) return; @@ -292,7 +292,7 @@ void MergeTreeRangeReader::ReadResult::optimize() size_t rows_in_last_granule = rows_per_granule.back(); - collapseZeroTails(filter->getData(), new_data, zero_tails); + collapseZeroTails(filter->getData(), new_data, zero_tails, can_read_incomplete_granules); total_rows_per_granule = new_filter->size(); num_rows_to_skip_in_last_granule += rows_in_last_granule - rows_per_granule.back(); @@ -323,7 +323,7 @@ size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & } void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & filter_vec, IColumn::Filter & new_filter_vec, - const NumRows & zero_tails) + const NumRows & zero_tails, bool can_read_incomplete_granules) { auto filter_data = filter_vec.data(); auto new_filter_data = new_filter_vec.data(); @@ -333,7 +333,8 @@ void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & auto & rows_to_read = rows_per_granule[i]; auto filtered_rows_num_at_granule_end = zero_tails[i]; - rows_to_read -= filtered_rows_num_at_granule_end; + if (can_read_incomplete_granules || filtered_rows_num_at_granule_end == rows_to_read) + rows_to_read -= filtered_rows_num_at_granule_end; memcpySmallAllowReadWriteOverflow15(new_filter_data, filter_data, rows_to_read); filter_data += rows_to_read; @@ -598,6 +599,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t /// result.num_rows_read if the last granule in range also the last in part (so we have to adjust last granule). { size_t space_left = max_rows; + size_t started_mark = 0; while (space_left && (!stream.isFinished() || !ranges.empty())) { if (stream.isFinished()) @@ -606,6 +608,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t stream = Stream(ranges.back().begin, ranges.back().end, merge_tree_reader); result.addRange(ranges.back()); ranges.pop_back(); + started_mark = stream.currentMark(); } size_t current_space = space_left; @@ -617,6 +620,9 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t auto rows_to_read = std::min(current_space, stream.numPendingRowsInCurrentGranule()); + std::cerr << "(startReadingChain) rows_to_read: " << rows_to_read << "\n"; + std::cerr << "(startReadingChain) current_space: " << current_space << "\n"; + bool last = rows_to_read == space_left; result.addRows(stream.read(result.block, rows_to_read, !last)); result.addGranule(rows_to_read); @@ -701,7 +707,7 @@ void MergeTreeRangeReader::executePrewhereActionsAndFilterColumns(ReadResult & r result.setFilter(filter); if (!last_reader_in_chain) - result.optimize(); + result.optimize(merge_tree_reader->canReadIncompleteGranules()); bool filter_always_true = !result.getFilter() && result.totalRowsPerGranule() == filter->size(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h index df41fe30c17..8877a0471c8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -152,7 +152,7 @@ public: /// Set filter or replace old one. Filter must have more zeroes than previous. void setFilter(const ColumnPtr & new_filter); /// For each granule calculate the number of filtered rows at the end. Remove them and update filter. - void optimize(); + void optimize(bool can_read_incomplete_granules); /// Remove all rows from granules. void clear(); @@ -178,7 +178,7 @@ public: ColumnPtr filter_holder; const ColumnUInt8 * filter = nullptr; - void collapseZeroTails(const IColumn::Filter & filter, IColumn::Filter & new_filter, const NumRows & zero_tails); + void collapseZeroTails(const IColumn::Filter & filter, IColumn::Filter & new_filter, const NumRows & zero_tails, bool can_read_incomplete_granules); size_t countZeroTails(const IColumn::Filter & filter, NumRows & zero_tails) const; static size_t numZerosInTail(const UInt8 * begin, const UInt8 * end); }; From 7803aee5046840980ef8208cc170c1390b5f5403 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 5 Dec 2019 15:27:31 +0300 Subject: [PATCH 0073/2007] polymorphic parts (development) fix prewhere --- .../Storages/MergeTree/MergeTreeRangeReader.cpp | 16 +++++++++------- .../Storages/MergeTree/MergeTreeRangeReader.h | 4 ++-- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 0aa034579d6..f8ace348bd1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -269,7 +269,7 @@ void MergeTreeRangeReader::ReadResult::optimize(bool can_read_incomplete_granule return; NumRows zero_tails; - auto total_zero_rows_in_tails = countZeroTails(filter->getData(), zero_tails); + auto total_zero_rows_in_tails = countZeroTails(filter->getData(), zero_tails, can_read_incomplete_granules); if (total_zero_rows_in_tails == filter->size()) { @@ -292,7 +292,7 @@ void MergeTreeRangeReader::ReadResult::optimize(bool can_read_incomplete_granule size_t rows_in_last_granule = rows_per_granule.back(); - collapseZeroTails(filter->getData(), new_data, zero_tails, can_read_incomplete_granules); + collapseZeroTails(filter->getData(), new_data, zero_tails); total_rows_per_granule = new_filter->size(); num_rows_to_skip_in_last_granule += rows_in_last_granule - rows_per_granule.back(); @@ -302,7 +302,7 @@ void MergeTreeRangeReader::ReadResult::optimize(bool can_read_incomplete_granule } } -size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & filter_vec, NumRows & zero_tails) const +size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & filter_vec, NumRows & zero_tails, bool can_read_incomplete_granules) const { zero_tails.resize(0); zero_tails.reserve(rows_per_granule.size()); @@ -314,7 +314,10 @@ size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & for (auto rows_to_read : rows_per_granule) { /// Count the number of zeros at the end of filter for rows were read from current granule. - zero_tails.push_back(numZerosInTail(filter_data, filter_data + rows_to_read)); + size_t zero_tail = numZerosInTail(filter_data, filter_data + rows_to_read); + if (!can_read_incomplete_granules && zero_tail != rows_to_read) + zero_tail = 0; + zero_tails.push_back(zero_tail); total_zero_rows_in_tails += zero_tails.back(); filter_data += rows_to_read; } @@ -323,7 +326,7 @@ size_t MergeTreeRangeReader::ReadResult::countZeroTails(const IColumn::Filter & } void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & filter_vec, IColumn::Filter & new_filter_vec, - const NumRows & zero_tails, bool can_read_incomplete_granules) + const NumRows & zero_tails) { auto filter_data = filter_vec.data(); auto new_filter_data = new_filter_vec.data(); @@ -333,8 +336,7 @@ void MergeTreeRangeReader::ReadResult::collapseZeroTails(const IColumn::Filter & auto & rows_to_read = rows_per_granule[i]; auto filtered_rows_num_at_granule_end = zero_tails[i]; - if (can_read_incomplete_granules || filtered_rows_num_at_granule_end == rows_to_read) - rows_to_read -= filtered_rows_num_at_granule_end; + rows_to_read -= filtered_rows_num_at_granule_end; memcpySmallAllowReadWriteOverflow15(new_filter_data, filter_data, rows_to_read); filter_data += rows_to_read; diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h index 8877a0471c8..a9551b87f1c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.h @@ -178,8 +178,8 @@ public: ColumnPtr filter_holder; const ColumnUInt8 * filter = nullptr; - void collapseZeroTails(const IColumn::Filter & filter, IColumn::Filter & new_filter, const NumRows & zero_tails, bool can_read_incomplete_granules); - size_t countZeroTails(const IColumn::Filter & filter, NumRows & zero_tails) const; + void collapseZeroTails(const IColumn::Filter & filter, IColumn::Filter & new_filter, const NumRows & zero_tails); + size_t countZeroTails(const IColumn::Filter & filter, NumRows & zero_tails, bool can_read_incomplete_granules) const; static size_t numZerosInTail(const UInt8 * begin, const UInt8 * end); }; From bd08520436b1ff5ed70911aaaf011461dbb5752a Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 5 Dec 2019 16:23:36 +0300 Subject: [PATCH 0074/2007] polymorphic parts (development) --- dbms/src/Common/ErrorCodes.cpp | 1 + .../Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp | 8 +++++++- .../Storages/MergeTree/MergeTreeIndexGranularityInfo.h | 9 +++++++-- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/dbms/src/Common/ErrorCodes.cpp b/dbms/src/Common/ErrorCodes.cpp index cfa89af96d4..e789a5b323b 100644 --- a/dbms/src/Common/ErrorCodes.cpp +++ b/dbms/src/Common/ErrorCodes.cpp @@ -459,6 +459,7 @@ namespace ErrorCodes extern const int DICTIONARY_ACCESS_DENIED = 482; extern const int TOO_MANY_REDIRECTS = 483; extern const int INTERNAL_REDIS_ERROR = 484; + extern const int UNKNOWN_PART_TYPE = 485; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index f059665d754..f29b276c2bd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -7,6 +7,12 @@ namespace DB { + +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; +} + std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(const std::string & path_to_part) { if (Poco::File(path_to_part).exists()) @@ -42,7 +48,7 @@ void MergeTreeIndexGranularityInfo::initialize(const MergeTreeData & storage, Me if (!storage.canUseAdaptiveGranularity()) { if (part_type != MergeTreeDataPartType::WIDE) - throw ""; /// FIXME normal exception + throw Exception("Only Wide parts can be used with non-adaptive granularity.", ErrorCodes::NOT_IMPLEMENTED); setNonAdaptive(); } else diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index d974d2d88f2..ae91d604b34 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -8,6 +8,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int UNKNOWN_PART_TYPE; +} + class MergeTreeData; /// Meta information about index granularity @@ -67,7 +72,7 @@ inline std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) case MergeTreeDataPartType::COMPACT: return ".mrk3"; default: - throw "unknown part type"; /// FIXME normal exception + throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); } } @@ -80,7 +85,7 @@ inline size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns case MergeTreeDataPartType::COMPACT: return sizeof(UInt64) * (columns_num * 2 + 1); default: - throw "unknown part type"; /// FIXME normal exception + throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); } } From d3b0800a6361f86842fcbbff86231ea3fbe8f030 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 10 Dec 2019 00:21:17 +0300 Subject: [PATCH 0075/2007] polymorphic parts (development) alter --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 14 +++--- .../Storages/MergeTree/IMergeTreeDataPart.h | 2 +- .../MergeTree/IMergeTreeDataPartWriter.h | 13 ++++- .../MergeTree/IMergedBlockOutputStream.cpp | 19 +++++++ .../MergeTree/IMergedBlockOutputStream.h | 2 + dbms/src/Storages/MergeTree/MergeTreeData.cpp | 49 ++++++++++++------- .../MergeTree/MergeTreeDataMergerMutator.cpp | 6 +-- .../MergeTree/MergeTreeDataPartWide.cpp | 27 +++++----- .../MergeTree/MergeTreeDataPartWide.h | 2 + .../MergeTreeDataPartWriterCompact.cpp | 7 +-- .../MergeTreeDataPartWriterCompact.h | 4 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 41 +++++++--------- .../MergeTree/MergeTreeDataPartWriterWide.h | 13 ++--- .../MergeTree/MergeTreeReaderSettings.h | 1 + .../MergeTree/MergedBlockOutputStream.cpp | 32 +----------- .../MergedColumnOnlyOutputStream.cpp | 37 +++++--------- .../MergeTree/MergedColumnOnlyOutputStream.h | 12 ++--- dbms/src/Storages/StorageMergeTree.cpp | 1 - 18 files changed, 133 insertions(+), 149 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 662b4312a22..3964b335d37 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -577,13 +577,14 @@ void IMergeTreeDataPart::loadColumns(bool require) columns.writeText(out); } Poco::File(path + ".tmp").renameTo(path); - - return; + } + else + { + is_frozen = !poco_file_path.canWrite(); + ReadBufferFromFile file = openForReading(path); + columns.readText(file); } - is_frozen = !poco_file_path.canWrite(); - ReadBufferFromFile file = openForReading(path); - columns.readText(file); index_granularity_info.initialize(storage, getType(), columns.size()); for (const auto & it : columns) sample_block.insert({it.type, it.name}); @@ -599,10 +600,7 @@ void IMergeTreeDataPart::loadColumnSizes() auto column_sizes_path = getFullPath() + "columns_sizes.txt"; auto columns_sizes_file = Poco::File(column_sizes_path); if (!columns_sizes_file.exists()) - { - LOG_WARNING(storage.log, "No file column_sizes.txt in part " + name); return; - } ReadBufferFromFile buffer(column_sizes_path, columns_sizes_file.getSize()); auto it = columns.begin(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 82e5e4b40a5..1d4a59634da 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -314,7 +314,7 @@ public: void makeCloneOnDiskDetached(const DiskSpace::ReservationPtr & reservation) const; /// Checks that .bin and .mrk files exist - bool hasColumnFiles(const String & /* column */, const IDataType & /* type */ ) const { return true; } + virtual bool hasColumnFiles(const String & /* column */, const IDataType & /* type */ ) const { return true; } static UInt64 calculateTotalSizeOnDisk(const String & from); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 5b4c431565d..f147cafb9e7 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -70,7 +70,7 @@ public: bool need_finish_last_granule); virtual void write( - const Block & block, const IColumn::Permutation * permutation, + const Block & block, const IColumn::Permutation * permutation = nullptr, /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; @@ -93,6 +93,12 @@ public: return columns_sizes; } + void setOffsetColumns(WrittenOffsetColumns * written_offset_columns_, bool skip_offsets_) + { + written_offset_columns = written_offset_columns_; + skip_offsets = skip_offsets_; + } + void initSkipIndices(); void initPrimaryIndex(); void calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows); @@ -150,6 +156,11 @@ protected: bool skip_indices_initialized = false; MergeTreeData::ColumnSizeByName columns_sizes; + + /// To correctly write Nested elements column-by-column. + WrittenOffsetColumns * written_offset_columns = nullptr; + bool skip_offsets = false; + }; using MergeTreeWriterPtr = std::unique_ptr; diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 245c016cb0f..7ca9d77b96c 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -25,6 +25,25 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( { } +Block IMergedBlockOutputStream::getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation) +{ + Block result; + for (size_t i = 0, size = names.size(); i < size; ++i) + { + const auto & name = names[i]; + result.insert(i, block.getByName(name)); + + /// Reorder primary key columns in advance and add them to `primary_key_columns`. + if (permutation) + { + auto & column = result.getByPosition(i); + column.column = column.column->permute(*permutation, 0); + } + } + + return result; +} + /// Implementation of IMergedBlockOutputStream::ColumnStream. } diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index 8cb7bea9764..defb77db2d4 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -59,6 +59,8 @@ protected: // MergeTreeIndexAggregators skip_indices_aggregators; // std::vector skip_index_filling; + static Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation); + MergeTreeWriterPtr writer; // const bool with_final_mark; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index dba837a80df..3b18305cd0e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1426,7 +1426,7 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name if (part) part_mrk_file_extension = part->index_granularity_info.marks_file_extension; else - part_mrk_file_extension = settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE); /// FIXME support compact parts + part_mrk_file_extension = settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE); /// FIXME which extension should we use?? using NameToType = std::map; NameToType new_types; @@ -1436,7 +1436,6 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name /// For every column that need to be converted: source column name, column name of calculated expression for conversion. std::vector> conversions; - /// Remove old indices std::set new_indices_set; for (const auto & index_decl : new_indices) @@ -1522,6 +1521,8 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name } } + std::cerr << "(alterDataPart) conversions.size(): " << conversions.size() << "\n"; + if (!conversions.empty()) { /// Give proper names for temporary columns with conversion results. @@ -1529,30 +1530,36 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name NamesWithAliases projection; projection.reserve(conversions.size()); - for (const auto & source_and_expression : conversions) + if (part && part->getType() == MergeTreeDataPartType::COMPACT) + { + out_rename_map["data_converting.bin"] = "data.bin"; + out_rename_map["data_converting.mrk3"] = "data.mrk3"; + } + + for (const auto & [source_name, expression_name] : conversions) { /// Column name for temporary filenames before renaming. NOTE The is unnecessarily tricky. + String temporary_column_name = source_name + " converting"; - String original_column_name = source_and_expression.first; - String temporary_column_name = original_column_name + " converting"; - - projection.emplace_back(source_and_expression.second, temporary_column_name); + projection.emplace_back(expression_name, temporary_column_name); /// After conversion, we need to rename temporary files into original. - - new_types[source_and_expression.first]->enumerateStreams( - [&](const IDataType::SubstreamPath & substream_path) + if (!part || part->getType() == MergeTreeDataPartType::WIDE) + { + new_types[source_name]->enumerateStreams( + [&, source_name=source_name](const IDataType::SubstreamPath & substream_path) { /// Skip array sizes, because they cannot be modified in ALTER. if (!substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) return; - String original_file_name = IDataType::getFileNameForStream(original_column_name, substream_path); + String original_file_name = IDataType::getFileNameForStream(source_name, substream_path); String temporary_file_name = IDataType::getFileNameForStream(temporary_column_name, substream_path); out_rename_map[temporary_file_name + ".bin"] = original_file_name + ".bin"; out_rename_map[temporary_file_name + part_mrk_file_extension] = original_file_name + part_mrk_file_extension; }, {}); + } } out_expression->add(ExpressionAction::project(projection)); @@ -1563,15 +1570,15 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name WriteBufferFromOwnString out; out << "Will "; bool first = true; - for (const auto & from_to : out_rename_map) + for (const auto & [from, to] : out_rename_map) { if (!first) out << ", "; first = false; - if (from_to.second.empty()) - out << "remove " << from_to.first; + if (to.empty()) + out << "remove " << from; else - out << "rename " << from_to.first << " to " << from_to.second; + out << "rename " << from << " to " << to; } out << " in part " << part->name; LOG_DEBUG(log, out.str()); @@ -1720,6 +1727,11 @@ void MergeTreeData::alterDataPart( DataPart::Checksums add_checksums; + std::cerr << "(alterDataPart) map size: " << transaction->rename_map.size() << "\n"; + for (const auto & elem : transaction->rename_map) + std::cerr << "(" << elem.first << ", " << elem.second << ") "; + std::cerr << "\n"; + if (transaction->rename_map.empty() && !force_update_metadata) { transaction->clear(); @@ -1729,10 +1741,10 @@ void MergeTreeData::alterDataPart( /// Apply the expression and write the result to temporary files. if (expression) { + std::cerr << "(alterDataPart) expression: " << expression->dumpActions() << "\n"; BlockInputStreamPtr part_in = std::make_shared( *this, part, expression->getRequiredColumns(), false, /* take_column_types_from_storage = */ false); - auto compression_codec = global_context.chooseCompressionCodec( part->bytes_on_disk, static_cast(part->bytes_on_disk) / this->getTotalActiveSizeInBytes()); @@ -1755,9 +1767,10 @@ void MergeTreeData::alterDataPart( true /* skip_offsets */, /// Don't recalc indices because indices alter is restricted std::vector{}, - unused_written_offsets, + nullptr /* offset_columns */, part->index_granularity, - &part->index_granularity_info); + &part->index_granularity_info, + "_converting"); in.readPrefix(); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 9f65a2fe964..e4cbd2d47bc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -849,7 +849,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor /// because all of them were already recalculated and written /// as key part of vertical merge std::vector{}, - written_offset_columns, + &written_offset_columns, to.getIndexGranularity()); size_t column_elems_written = 0; @@ -1118,8 +1118,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor merge_entry->columns_written = all_columns.size() - updated_header.columns(); - IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; - MergedColumnOnlyOutputStream out( new_data_part, updated_header, @@ -1127,7 +1125,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor compression_codec, /* skip_offsets = */ false, std::vector(indices_to_recalc.begin(), indices_to_recalc.end()), - unused_written_offsets, + nullptr, source_part->index_granularity, &source_part->index_granularity_info ); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 1c66ecad923..c080ba10c3c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -180,6 +180,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() String full_path = getFullPath(); index_granularity_info.changeGranularityIfRequired(full_path); + if (columns.empty()) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); @@ -339,22 +340,22 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const -// bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDataType & type) const -// { -// bool res = true; +bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDataType & type) const +{ + bool res = true; -// type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) -// { -// String file_name = IDataType::getFileNameForStream(column_name, substream_path); + type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + String file_name = IDataType::getFileNameForStream(column_name, substream_path); -// auto bin_checksum = checksums.files.find(file_name + ".bin"); -// auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); + auto bin_checksum = checksums.files.find(file_name + ".bin"); + auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); -// if (bin_checksum == checksums.files.end() || mrk_checksum == checksums.files.end()) -// res = false; -// }, {}); + if (bin_checksum == checksums.files.end() || mrk_checksum == checksums.files.end()) + res = false; + }, {}); -// return res; -// } + return res; +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index da839c6b31e..8984fef5dac 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -81,6 +81,8 @@ public: ~MergeTreeDataPartWide() override; + bool hasColumnFiles(const String & column, const IDataType & type) const override; + protected: void checkConsistency(bool require_part_metadata) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index e5ad131694b..59521ae45c3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -25,10 +25,11 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( default_codec_, settings_, index_granularity_, true) , squashing(storage.getSettings()->index_granularity, storage.getSettings()->index_granularity_bytes) /// FIXME { + String data_file_name = DATA_FILE_NAME + settings.filename_suffix; stream = std::make_unique( - DATA_FILE_NAME, - part_path + DATA_FILE_NAME, DATA_FILE_EXTENSION, - part_path + DATA_FILE_NAME, marks_file_extension, + data_file_name, + part_path + data_file_name, DATA_FILE_EXTENSION, + part_path + data_file_name, marks_file_extension, default_codec, settings.max_compress_block_size, settings.estimated_size, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 1c56bf75ade..2f6fc7b99c4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -17,8 +17,8 @@ public: const WriterSettings & settings, const MergeTreeIndexGranularity & index_granularity); - void write(const Block & block, const IColumn::Permutation * permutation, - const Block & primary_key_block, const Block & skip_indexes_block) override; + void write(const Block & block, const IColumn::Permutation * permutation = nullptr, + const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) override; void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index fd882d400b9..f6f165e1372 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -24,15 +24,14 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( { const auto & columns = storage.getColumns(); for (const auto & it : columns_list) - addStreams(it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec), settings.estimated_size, false); + addStreams(it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec), settings.estimated_size); } void MergeTreeDataPartWriterWide::addStreams( const String & name, const IDataType & type, const CompressionCodecPtr & effective_codec, - size_t estimated_size, - bool skip_offsets) + size_t estimated_size) { IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path) { @@ -63,9 +62,9 @@ void MergeTreeDataPartWriterWide::addStreams( IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( - const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets) + const String & name, WrittenOffsetColumns & offset_columns) { - return [&, skip_offsets] (const IDataType::SubstreamPath & substream_path) -> WriteBuffer * + return [&, this] (const IDataType::SubstreamPath & substream_path) -> WriteBuffer * { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; if (is_offsets && skip_offsets) @@ -108,9 +107,7 @@ void MergeTreeDataPartWriterWide::write(const Block & block, std::cerr << "(MergeTreeDataPartWriterWide::write) marks_count: " << index_granularity.getMarksCount() << "\n"; std::cerr << "(MergeTreeDataPartWriterWide::write) current_mark: " << current_mark << "\n"; - - WrittenOffsetColumns offset_columns; - MarkWithOffset result; + auto offset_columns = written_offset_columns ? *written_offset_columns : WrittenOffsetColumns{}; auto it = columns_list.begin(); for (size_t i = 0; i < columns_list.size(); ++i, ++it) @@ -122,23 +119,23 @@ void MergeTreeDataPartWriterWide::write(const Block & block, if (primary_key_block.has(it->name)) { const auto & primary_column = *primary_key_block.getByName(it->name).column; - result = writeColumn(column.name, *column.type, primary_column, offset_columns); + writeColumn(column.name, *column.type, primary_column, offset_columns); } else if (skip_indexes_block.has(it->name)) { const auto & index_column = *skip_indexes_block.getByName(it->name).column; - result = writeColumn(column.name, *column.type, index_column, offset_columns); + writeColumn(column.name, *column.type, index_column, offset_columns); } else { /// We rearrange the columns that are not included in the primary key here; Then the result is released - to save RAM. ColumnPtr permuted_column = column.column->permute(*permutation, 0); - result = writeColumn(column.name, *column.type, *permuted_column, offset_columns); + writeColumn(column.name, *column.type, *permuted_column, offset_columns); } } else { - result = writeColumn(column.name, *column.type, *column.column, offset_columns); + writeColumn(column.name, *column.type, *column.column, offset_columns); } } } @@ -147,7 +144,6 @@ void MergeTreeDataPartWriterWide::writeSingleMark( const String & name, const IDataType & type, WrittenOffsetColumns & offset_columns, - bool skip_offsets, size_t number_of_rows, DB::IDataType::SubstreamPath & path) { @@ -181,7 +177,6 @@ size_t MergeTreeDataPartWriterWide::writeSingleGranule( const IDataType & type, const IColumn & column, WrittenOffsetColumns & offset_columns, - bool skip_offsets, IDataType::SerializeBinaryBulkStatePtr & serialization_state, IDataType::SerializeBinaryBulkSettings & serialize_settings, size_t from_row, @@ -189,7 +184,7 @@ size_t MergeTreeDataPartWriterWide::writeSingleGranule( bool write_marks) { if (write_marks) - writeSingleMark(name, type, offset_columns, skip_offsets, number_of_rows, serialize_settings.path); + writeSingleMark(name, type, offset_columns, number_of_rows, serialize_settings.path); type.serializeBinaryBulkWithMultipleStreams(column, from_row, number_of_rows, serialize_settings, serialization_state); @@ -218,14 +213,13 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( const String & name, const IDataType & type, const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets) + WrittenOffsetColumns & offset_columns) { auto [it, inserted] = serialization_states.emplace(name, nullptr); if (inserted) { IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.getter = createStreamGetter(name, offset_columns, false); + serialize_settings.getter = createStreamGetter(name, offset_columns); type.serializeBinaryBulkStatePrefix(serialize_settings, it->second); } @@ -235,7 +229,7 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( auto & settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.getter = createStreamGetter(name, offset_columns, skip_offsets); + serialize_settings.getter = createStreamGetter(name, offset_columns); serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; @@ -271,7 +265,6 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( type, column, offset_columns, - skip_offsets, it->second, serialize_settings, current_row, @@ -315,13 +308,14 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch { if (!serialization_states.empty()) { - serialize_settings.getter = createStreamGetter(it->name, offset_columns, false); + /// FIXME maybe we need skip_offsets=false in some cases + serialize_settings.getter = createStreamGetter(it->name, written_offset_columns ? *written_offset_columns : offset_columns); it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[it->name]); } if (write_final_mark) { - writeFinalMark(it->name, it->type, offset_columns, false, serialize_settings.path); + writeFinalMark(it->name, it->type, offset_columns, serialize_settings.path); } } } @@ -342,10 +336,9 @@ void MergeTreeDataPartWriterWide::writeFinalMark( const std::string & column_name, const DataTypePtr column_type, WrittenOffsetColumns & offset_columns, - bool skip_offsets, DB::IDataType::SubstreamPath & path) { - writeSingleMark(column_name, *column_type, offset_columns, skip_offsets, 0, path); + writeSingleMark(column_name, *column_type, offset_columns, 0, path); /// Memoize information about offsets column_type->enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 21749d97d88..fbbe2e71580 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -19,12 +19,12 @@ public: const WriterSettings & settings, const MergeTreeIndexGranularity & index_granularity); - void write(const Block & block, const IColumn::Permutation * permutation, + void write(const Block & block, const IColumn::Permutation * permutation = nullptr, const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) override; void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) override; - IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); + IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns); /// Write data of one column. /// Return how many marks were written and @@ -33,8 +33,7 @@ public: const String & name, const IDataType & type, const IColumn & column, - WrittenOffsetColumns & offset_columns, - bool skip_offsets = false); + WrittenOffsetColumns & offset_columns); private: /// Write single granule of one column (rows between 2 marks) @@ -43,7 +42,6 @@ private: const IDataType & type, const IColumn & column, WrittenOffsetColumns & offset_columns, - bool skip_offsets, IDataType::SerializeBinaryBulkStatePtr & serialization_state, IDataType::SerializeBinaryBulkSettings & serialize_settings, size_t from_row, @@ -55,7 +53,6 @@ private: const String & name, const IDataType & type, WrittenOffsetColumns & offset_columns, - bool skip_offsets, size_t number_of_rows, DB::IDataType::SubstreamPath & path); @@ -63,15 +60,13 @@ private: const std::string & column_name, const DataTypePtr column_type, WrittenOffsetColumns & offset_columns, - bool skip_offsets, DB::IDataType::SubstreamPath & path); void addStreams( const String & name, const IDataType & type, const CompressionCodecPtr & effective_codec, - size_t estimated_size, - bool skip_offsets); + size_t estimated_size); SerializationStates serialization_states; bool can_use_adaptive_granularity; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h index 877cfc6b7d9..3fc23f90e38 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h @@ -26,6 +26,7 @@ struct WriterSettings size_t aio_threshold; bool can_use_adaptive_granularity; bool blocks_are_granules_size; + String filename_suffix = ""; size_t estimated_size = 0; }; } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 49ebfc979a8..d1a181ab0be 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -187,42 +187,14 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm if (!rows) return; - Block primary_key_block; - Block skip_indexes_block; - - auto primary_key_column_names = storage.primary_key_columns; - std::set skip_indexes_column_names_set; for (const auto & index : storage.skip_indices) std::copy(index->columns.cbegin(), index->columns.cend(), std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); - for (size_t i = 0, size = primary_key_column_names.size(); i < size; ++i) - { - const auto & name = primary_key_column_names[i]; - primary_key_block.insert(i, block.getByName(name)); - - /// Reorder primary key columns in advance and add them to `primary_key_columns`. - if (permutation) - { - auto & column = primary_key_block.getByPosition(i); - column.column = column.column->permute(*permutation, 0); - } - } - - for (size_t i = 0, size = skip_indexes_column_names.size(); i < size; ++i) - { - const auto & name = skip_indexes_column_names[i]; - skip_indexes_block.insert(i, block.getByName(name)); - - /// Reorder index columns in advance. - if (permutation) - { - auto & column = skip_indexes_block.getByPosition(i); - column.column = column.column->permute(*permutation, 0); - } - } + Block primary_key_block = getBlockAndPermute(block, storage.primary_key_columns, permutation); + Block skip_indexes_block = getBlockAndPermute(block, skip_indexes_column_names, permutation); writer->write(block, permutation, primary_key_block, skip_indexes_block); writer->calculateAndSerializeSkipIndices(skip_indexes_block, rows); diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 29a659193ca..28e838f8f02 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -7,12 +7,12 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( const MergeTreeDataPartPtr & data_part, const Block & header_, bool sync_, CompressionCodecPtr default_codec, bool skip_offsets_, const std::vector & indices_to_recalc, - WrittenOffsetColumns & already_written_offset_columns_, + WrittenOffsetColumns * offset_columns_, const MergeTreeIndexGranularity & index_granularity, - const MergeTreeIndexGranularityInfo * index_granularity_info) + const MergeTreeIndexGranularityInfo * index_granularity_info, + const String & filename_suffix) : IMergedBlockOutputStream(data_part), - header(header_), sync(sync_), skip_offsets(skip_offsets_), - already_written_offset_columns(already_written_offset_columns_) + header(header_), sync(sync_) { // std::cerr << "(MergedColumnOnlyOutputStream) storage: " << storage.getTableName() << "\n"; // std::cerr << "(MergedColumnOnlyOutputStream) can_use_adaptive_granularity: " << can_use_adaptive_granularity << "\n"; @@ -20,16 +20,13 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( // if (index_granularity_info_) // std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; - WriterSettings writer_settings( data_part->storage.global_context.getSettings(), index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity()); + writer_settings.filename_suffix = filename_suffix; writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, default_codec, writer_settings, index_granularity); - writer_wide = typeid_cast(writer.get()); - if (!writer_wide) - throw Exception("MergedColumnOnlyOutputStream can be used only for writing Wide parts", ErrorCodes::LOGICAL_ERROR); - + writer->setOffsetColumns(offset_columns_, skip_offsets_); writer->initSkipIndices(); } @@ -41,14 +38,7 @@ void MergedColumnOnlyOutputStream::write(const Block & block) std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); - std::vector skip_indexes_columns(skip_indexes_column_names.size()); - std::map skip_indexes_column_name_to_position; - for (size_t i = 0, size = skip_indexes_column_names.size(); i < size; ++i) - { - const auto & name = skip_indexes_column_names[i]; - skip_indexes_column_name_to_position.emplace(name, i); - skip_indexes_columns[i] = block.getByName(name); - } + Block skip_indexes_block = getBlockAndPermute(block, skip_indexes_column_names, nullptr); size_t rows = block.rows(); if (!rows) @@ -56,15 +46,9 @@ void MergedColumnOnlyOutputStream::write(const Block & block) std::cerr << "(MergedColumnOnlyOutputStream::write) writing rows: " << rows << "\n"; - WrittenOffsetColumns offset_columns = already_written_offset_columns; - for (size_t i = 0; i < header.columns(); ++i) - { - const auto & column = block.getByName(header.getByPosition(i).name); - writer_wide->writeColumn(column.name, *column.type, *column.column, offset_columns, skip_offsets); - } - - writer_wide->calculateAndSerializeSkipIndices(skip_indexes_columns, rows); - writer_wide->next(); + writer->write(block); + writer->calculateAndSerializeSkipIndices(skip_indexes_block, rows); + writer->next(); } void MergedColumnOnlyOutputStream::writeSuffix() @@ -77,6 +61,7 @@ MergeTreeData::DataPart::Checksums MergedColumnOnlyOutputStream::writeSuffixAndG /// Finish columns serialization. MergeTreeData::DataPart::Checksums checksums; writer->finishDataSerialization(checksums, sync); + writer->finishSkipIndicesSerialization(checksums); return checksums; } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index 351969f5477..0833c89fde9 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -20,9 +20,10 @@ public: const MergeTreeDataPartPtr & data_part, const Block & header_, bool sync_, CompressionCodecPtr default_codec_, bool skip_offsets_, const std::vector & indices_to_recalc_, - WrittenOffsetColumns & already_written_offset_columns_, + WrittenOffsetColumns * offset_columns_ = nullptr, const MergeTreeIndexGranularity & index_granularity = {}, - const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr); + const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr, + const String & filename_suffix = ""); Block getHeader() const override { return header; } void write(const Block & block) override; @@ -31,14 +32,7 @@ public: private: Block header; - bool sync; - bool skip_offsets; - - /// To correctly write Nested elements column-by-column. - WrittenOffsetColumns & already_written_offset_columns; - - MergeTreeDataPartWriterWide * writer_wide; }; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index daffbbd149f..a3f621f621e 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -307,7 +307,6 @@ void StorageMergeTree::alter( context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier); - /// Reinitialize primary key because primary key column types might have changed. setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); From 26d159e88964b3c3b1b8c50da7353f3ac4cfaa4d Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 12 Dec 2019 21:55:19 +0300 Subject: [PATCH 0076/2007] polymorphic parts (development) alter --- .../MergeTree/IMergeTreeDataPartWriter.h | 1 - dbms/src/Storages/MergeTree/MergeTreeData.cpp | 282 +++++++++--------- dbms/src/Storages/MergeTree/MergeTreeData.h | 31 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 29 +- .../MergeTree/MergeTreeDataPartCompact.h | 3 + .../MergeTreeDataPartWriterCompact.cpp | 9 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 12 +- .../MergeTree/MergeTreeRangeReader.cpp | 5 - .../MergeTree/MergeTreeReaderCompact.cpp | 34 ++- .../MergeTree/MergeTreeReaderCompact.h | 2 + .../MergeTree/MergeTreeReaderWide.cpp | 14 +- .../MergeTreeThreadSelectBlockInputStream.cpp | 4 + 12 files changed, 221 insertions(+), 205 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index f147cafb9e7..b4884d18d8e 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -124,7 +124,6 @@ protected: WriterSettings settings; - bool compute_granularity; bool with_final_mark; bool need_finish_last_granule; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 3b18305cd0e..5c26b5c2492 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1405,49 +1405,25 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c checkSettingCanBeChanged(setting.name); /// Check that type conversions are possible. - ExpressionActionsPtr unused_expression; - NameToNameMap unused_map; - bool unused_bool; - createConvertExpression(nullptr, getColumns().getAllPhysical(), new_columns.getAllPhysical(), - getIndices().indices, new_indices.indices, unused_expression, unused_map, unused_bool); + analyzeAlterConversions(getColumns().getAllPhysical(), new_columns.getAllPhysical(), getIndices().indices, new_indices.indices); } - /// FIXME implement alter for compact parts -void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, - const IndicesASTs & old_indices, const IndicesASTs & new_indices, ExpressionActionsPtr & out_expression, - NameToNameMap & out_rename_map, bool & out_force_update_metadata) const +NameToNameMap MergeTreeData::createRenameMap( + const DataPartPtr & part, + const NamesAndTypesList & old_columns, + const AlterAnalysisResult & analysis_result) const { - const auto settings = getSettings(); - out_expression = nullptr; - out_rename_map = {}; - out_force_update_metadata = false; - String part_mrk_file_extension; - if (part) - part_mrk_file_extension = part->index_granularity_info.marks_file_extension; - else - part_mrk_file_extension = settings->index_granularity_bytes == 0 ? getNonAdaptiveMrkExtension() : getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE); /// FIXME which extension should we use?? + if (!part) + return {}; - using NameToType = std::map; - NameToType new_types; - for (const NameAndTypePair & column : new_columns) - new_types.emplace(column.name, column.type.get()); + const auto & part_mrk_file_extension = part->index_granularity_info.marks_file_extension; + NameToNameMap rename_map; - /// For every column that need to be converted: source column name, column name of calculated expression for conversion. - std::vector> conversions; - - /// Remove old indices - std::set new_indices_set; - for (const auto & index_decl : new_indices) - new_indices_set.emplace(index_decl->as().name); - for (const auto & index_decl : old_indices) + for (const auto & index_name : analysis_result.removed_indices) { - const auto & index = index_decl->as(); - if (!new_indices_set.count(index.name)) - { - out_rename_map["skp_idx_" + index.name + ".idx"] = ""; - out_rename_map["skp_idx_" + index.name + part_mrk_file_extension] = ""; - } + rename_map["skp_idx_" + index_name + ".idx"] = ""; + rename_map["skp_idx_" + index_name + part_mrk_file_extension] = ""; } /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. @@ -1460,117 +1436,53 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name }, {}); } - for (const NameAndTypePair & column : old_columns) + for (const auto & column : analysis_result.removed_columns) { - if (!new_types.count(column.name)) + if (part->hasColumnFiles(column.name, *column.type)) { - /// The column was deleted. - if (!part || part->hasColumnFiles(column.name, *column.type)) + column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { - column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(column.name, substream_path); + String file_name = IDataType::getFileNameForStream(column.name, substream_path); - /// Delete files if they are no longer shared with another column. - if (--stream_counts[file_name] == 0) - { - out_rename_map[file_name + ".bin"] = ""; - out_rename_map[file_name + part_mrk_file_extension] = ""; - } - }, {}); - } - } - else - { - /// The column was converted. Collect conversions. - const auto * new_type = new_types[column.name]; - const String new_type_name = new_type->getName(); - const auto * old_type = column.type.get(); - - if (!new_type->equals(*old_type) && (!part || part->hasColumnFiles(column.name, *column.type))) - { - if (isMetadataOnlyConversion(old_type, new_type)) + /// Delete files if they are no longer shared with another column. + if (--stream_counts[file_name] == 0) { - out_force_update_metadata = true; - continue; + rename_map[file_name + ".bin"] = ""; + rename_map[file_name + part_mrk_file_extension] = ""; } - - /// Need to modify column type. - if (!out_expression) - out_expression = std::make_shared(NamesAndTypesList(), global_context); - - out_expression->addInput(ColumnWithTypeAndName(nullptr, column.type, column.name)); - - Names out_names; - - /// This is temporary name for expression. TODO Invent the name more safely. - const String new_type_name_column = '#' + new_type_name + "_column"; - out_expression->add(ExpressionAction::addColumn( - { DataTypeString().createColumnConst(1, new_type_name), std::make_shared(), new_type_name_column })); - - const auto & function = FunctionFactory::instance().get("CAST", global_context); - out_expression->add(ExpressionAction::applyFunction( - function, Names{column.name, new_type_name_column}), out_names); - - out_expression->add(ExpressionAction::removeColumn(new_type_name_column)); - out_expression->add(ExpressionAction::removeColumn(column.name)); - - conversions.emplace_back(column.name, out_names.at(0)); - - } + }, {}); } } - std::cerr << "(alterDataPart) conversions.size(): " << conversions.size() << "\n"; - - if (!conversions.empty()) + for (const auto & elem : analysis_result.conversions) { - /// Give proper names for temporary columns with conversion results. + /// Column name for temporary filenames before renaming. NOTE The is unnecessarily tricky. + const auto & source_name = elem.first; + String temporary_column_name = source_name + " converting"; - NamesWithAliases projection; - projection.reserve(conversions.size()); - - if (part && part->getType() == MergeTreeDataPartType::COMPACT) - { - out_rename_map["data_converting.bin"] = "data.bin"; - out_rename_map["data_converting.mrk3"] = "data.mrk3"; - } - - for (const auto & [source_name, expression_name] : conversions) - { - /// Column name for temporary filenames before renaming. NOTE The is unnecessarily tricky. - String temporary_column_name = source_name + " converting"; - - projection.emplace_back(expression_name, temporary_column_name); - - /// After conversion, we need to rename temporary files into original. - if (!part || part->getType() == MergeTreeDataPartType::WIDE) + /// After conversion, we need to rename temporary files into original. + analysis_result.new_types.at(source_name)->enumerateStreams( + [&](const IDataType::SubstreamPath & substream_path) { - new_types[source_name]->enumerateStreams( - [&, source_name=source_name](const IDataType::SubstreamPath & substream_path) - { - /// Skip array sizes, because they cannot be modified in ALTER. - if (!substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) - return; + /// Skip array sizes, because they cannot be modified in ALTER. + if (!substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) + return; - String original_file_name = IDataType::getFileNameForStream(source_name, substream_path); - String temporary_file_name = IDataType::getFileNameForStream(temporary_column_name, substream_path); + String original_file_name = IDataType::getFileNameForStream(source_name, substream_path); + String temporary_file_name = IDataType::getFileNameForStream(temporary_column_name, substream_path); - out_rename_map[temporary_file_name + ".bin"] = original_file_name + ".bin"; - out_rename_map[temporary_file_name + part_mrk_file_extension] = original_file_name + part_mrk_file_extension; - }, {}); - } - } - - out_expression->add(ExpressionAction::project(projection)); + rename_map[temporary_file_name + ".bin"] = original_file_name + ".bin"; + rename_map[temporary_file_name + part_mrk_file_extension] = original_file_name + part_mrk_file_extension; + }, {}); } - if (part && !out_rename_map.empty()) + + if (part && !rename_map.empty()) { WriteBufferFromOwnString out; out << "Will "; bool first = true; - for (const auto & [from, to] : out_rename_map) + for (const auto & [from, to] : rename_map) { if (!first) out << ", "; @@ -1583,6 +1495,95 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name out << " in part " << part->name; LOG_DEBUG(log, out.str()); } + + return rename_map; +} + + +MergeTreeData::AlterAnalysisResult MergeTreeData::analyzeAlterConversions( + const NamesAndTypesList & old_columns, + const NamesAndTypesList & new_columns, + const IndicesASTs & old_indices, + const IndicesASTs & new_indices) const +{ + AlterAnalysisResult res; + const auto settings = getSettings(); + + std::set new_indices_set; + for (const auto & index_decl : new_indices) + new_indices_set.emplace(index_decl->as().name); + for (const auto & index_decl : old_indices) + { + const auto & index = index_decl->as(); + if (!new_indices_set.count(index.name)) + res.removed_indices.push_back(index.name); + } + + for (const NameAndTypePair & column : new_columns) + res.new_types.emplace(column.name, column.type.get()); + + for (const NameAndTypePair & column : old_columns) + { + if (!res.new_types.count(column.name)) + { + res.removed_columns.push_back(column); + } + else + { + /// The column was converted. Collect conversions. + const auto * new_type = res.new_types[column.name]; + const String new_type_name = new_type->getName(); + const auto * old_type = column.type.get(); + + if (!new_type->equals(*old_type)) + { + if (isMetadataOnlyConversion(old_type, new_type)) + { + res.force_update_metadata = true; + continue; + } + + /// Need to modify column type. + if (!res.expression) + res.expression = std::make_shared(NamesAndTypesList(), global_context); + + res.expression->addInput(ColumnWithTypeAndName(nullptr, column.type, column.name)); + + Names out_names; + + /// This is temporary name for expression. TODO Invent the name more safely. + const String new_type_name_column = '#' + new_type_name + "_column"; + res.expression->add(ExpressionAction::addColumn( + { DataTypeString().createColumnConst(1, new_type_name), std::make_shared(), new_type_name_column })); + + const auto & function = FunctionFactory::instance().get("CAST", global_context); + res.expression->add(ExpressionAction::applyFunction( + function, Names{column.name, new_type_name_column}), out_names); + + res.expression->add(ExpressionAction::removeColumn(new_type_name_column)); + res.expression->add(ExpressionAction::removeColumn(column.name)); + + res.conversions.emplace_back(column.name, out_names.at(0)); + } + } + } + + if (!res.conversions.empty()) + { + /// Give proper names for temporary columns with conversion results. + NamesWithAliases projection; + projection.reserve(res.conversions.size()); + + for (const auto & source_and_expression : res.conversions) + { + String temporary_column_name = source_and_expression.first + " converting"; + projection.emplace_back(source_and_expression.second, temporary_column_name); + } + + res.expression->add(ExpressionAction::project(projection)); + } + + return res; } @@ -1612,13 +1613,13 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const String & name, const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, - const NamesAndTypesList & columns, + const NamesAndTypesList & columns_list, size_t bytes_uncompressed, size_t rows_count, const String & relative_path) const { auto part = createPart(name, choosePartType(bytes_uncompressed, rows_count), part_info, disk, relative_path); - part->setColumns(columns); + part->setColumns(columns_list); /// Don't save rows_count count here as it can change later return part; } @@ -1667,12 +1668,10 @@ void MergeTreeData::alterDataPart( AlterDataPartTransactionPtr & transaction) { const auto settings = getSettings(); - ExpressionActionsPtr expression; const auto & part = transaction->getDataPart(); - bool force_update_metadata; - createConvertExpression(part, part->columns, new_columns, - getIndices().indices, new_indices, - expression, transaction->rename_map, force_update_metadata); + + auto res = analyzeAlterConversions(part->columns, new_columns, getIndices().indices, new_indices); + transaction->rename_map = createRenameMap(part, part->columns, res); size_t num_files_to_modify = transaction->rename_map.size(); size_t num_files_to_remove = 0; @@ -1726,29 +1725,24 @@ void MergeTreeData::alterDataPart( } DataPart::Checksums add_checksums; - - std::cerr << "(alterDataPart) map size: " << transaction->rename_map.size() << "\n"; - for (const auto & elem : transaction->rename_map) - std::cerr << "(" << elem.first << ", " << elem.second << ") "; - std::cerr << "\n"; - if (transaction->rename_map.empty() && !force_update_metadata) + if (transaction->rename_map.empty() && !res.force_update_metadata) { transaction->clear(); return; } /// Apply the expression and write the result to temporary files. - if (expression) + if (res.expression) { - std::cerr << "(alterDataPart) expression: " << expression->dumpActions() << "\n"; + std::cerr << "(alterDataPart) expression: " << res.expression->dumpActions() << "\n"; BlockInputStreamPtr part_in = std::make_shared( - *this, part, expression->getRequiredColumns(), false, /* take_column_types_from_storage = */ false); + *this, part, res.expression->getRequiredColumns(), false, /* take_column_types_from_storage = */ false); auto compression_codec = global_context.chooseCompressionCodec( part->bytes_on_disk, static_cast(part->bytes_on_disk) / this->getTotalActiveSizeInBytes()); - ExpressionBlockInputStream in(part_in, expression); + ExpressionBlockInputStream in(part_in, res.expression); /** Don't write offsets for arrays, because ALTER never change them * (MODIFY COLUMN could only change types of elements but never modify array sizes). diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index be1fa069f1b..f5be20a63f2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -904,6 +904,31 @@ protected: void setTTLExpressions(const ColumnsDescription::ColumnTTLs & new_column_ttls, const ASTPtr & new_ttl_table_ast, bool only_check = false); + using NameToType = std::map; + + struct AlterAnalysisResult + { + ExpressionActionsPtr expression = nullptr; + bool force_update_metadata = false; + NameToType new_types; + /// For every column that need to be converted: source column name, + /// column name of calculated expression for conversion. + std::vector> conversions; + NamesAndTypesList removed_columns; + Names removed_indices; + }; + + AlterAnalysisResult analyzeAlterConversions( + const NamesAndTypesList & old_columns, + const NamesAndTypesList & new_columns, + const IndicesASTs & old_indices, + const IndicesASTs & new_indices) const; + + NameToNameMap createRenameMap( + const DataPartPtr & part, + const NamesAndTypesList & old_columns, + const AlterAnalysisResult & analysis_result) const; + /// Expression for column type conversion. /// If no conversions are needed, out_expression=nullptr. /// out_rename_map maps column files for the out_expression onto new table files. @@ -911,9 +936,9 @@ protected: /// for transformation-free changing of Enum values list). /// Files to be deleted are mapped to an empty string in out_rename_map. /// If part == nullptr, just checks that all type conversions are possible. - void createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, - const IndicesASTs & old_indices, const IndicesASTs & new_indices, - ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const; + // void createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, + // const IndicesASTs & old_indices, const IndicesASTs & new_indices, + // ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const; /// Calculates column sizes in compressed form for the current state of data_parts. Call with data_parts mutex locked. void calculateColumnSizesImpl(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 232571eb9ac..66b1a8b1511 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -186,6 +186,17 @@ void MergeTreeDataPartCompact::loadIndexGranularity() index_granularity.setInitialized(); } +bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const IDataType &) const +{ + if (!getColumnPosition(column_name)) + return false; + /// FIXME replace everywhere hardcoded "data" + auto bin_checksum = checksums.files.find("data.bin"); + auto mrk_checksum = checksums.files.find("data" + index_granularity_info.marks_file_extension); + + return (bin_checksum != checksums.files.end() && mrk_checksum != checksums.files.end()); +} + void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { UNUSED(require_part_metadata); @@ -298,22 +309,4 @@ MergeTreeDataPartCompact::~MergeTreeDataPartCompact() removeIfNeeded(); } -// bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const IDataType & type) const -// { -// bool res = true; - -// type.enumerateStreams([&](const IDataType::SubstreamPath & substream_path) -// { -// String file_name = IDataType::getFileNameForStream(column_name, substream_path); - -// auto bin_checksum = checksums.files.find(file_name + ".bin"); -// auto mrk_checksum = checksums.files.find(file_name + index_granularity_info.marks_file_extension); - -// if (bin_checksum == checksums.files.end() || mrk_checksum == checksums.files.end()) -// res = false; -// }, {}); - -// return res; -// } - } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 23ed067a449..43e588ccab2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -77,6 +77,9 @@ public: void checkConsistency(bool /* require_part_metadata */) const override {} + bool hasColumnFiles(const String & column_name, const IDataType & type) const override; + + ~MergeTreeDataPartCompact() override; private: diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 59521ae45c3..4d1009c8092 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -102,7 +102,6 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) stream->compressed.next(); size_t next_row = 0; - writeIntBinary(rows_to_write, stream->marks); for (const auto & it : columns_list) next_row = writeColumnSingleGranule(block.getByName(it.name), current_row, rows_to_write); @@ -122,8 +121,6 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith std::cerr << "(writeColumnSingleGranule) from_row: " << from_row << "\n"; std::cerr << "(writeColumnSingleGranule) number_of_rows: " << number_of_rows << "\n"; - /// FIXME compressed size does not work - size_t old_compressed_size = stream->compressed_buf.getCompressedBytes() + stream->plain_hashing.count(); size_t old_uncompressed_size = stream->compressed.count(); writeIntBinary(stream->plain_hashing.count(), stream->marks); @@ -140,11 +137,9 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith column.type->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, state); column.type->serializeBinaryBulkStateSuffix(serialize_settings, state); - /// FIXME compressed size does not work - size_t compressed_size = stream->compressed_buf.getCompressedBytes() + stream->plain_hashing.count(); + /// We can't calculate compressed size by single column in compact format. size_t uncompressed_size = stream->compressed.count(); - - columns_sizes[column.name].add(ColumnSize{0, compressed_size - old_compressed_size, uncompressed_size - old_uncompressed_size}); + columns_sizes[column.name].add(ColumnSize{0, 0, uncompressed_size - old_uncompressed_size}); return from_row + number_of_rows; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index f6f165e1372..6f65a47fceb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -227,11 +227,11 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( std::cerr << "(writeColumn) column: " << name << "\n"; std::cerr << "(writeColumn) index_offset: " << index_offset << "\n"; - auto & settings = storage.global_context.getSettingsRef(); + const auto & global_settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.getter = createStreamGetter(name, offset_columns); - serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; - serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; + serialize_settings.low_cardinality_max_dictionary_size = global_settings.low_cardinality_max_dictionary_size; + serialize_settings.low_cardinality_use_single_dictionary_for_part = global_settings.low_cardinality_use_single_dictionary_for_part != 0; size_t total_rows = column.size(); size_t current_row = 0; @@ -294,10 +294,10 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) { - const auto & settings = storage.global_context.getSettingsRef(); + const auto & global_settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; - serialize_settings.low_cardinality_max_dictionary_size = settings.low_cardinality_max_dictionary_size; - serialize_settings.low_cardinality_use_single_dictionary_for_part = settings.low_cardinality_use_single_dictionary_for_part != 0; + serialize_settings.low_cardinality_max_dictionary_size = global_settings.low_cardinality_max_dictionary_size; + serialize_settings.low_cardinality_use_single_dictionary_for_part = global_settings.low_cardinality_use_single_dictionary_for_part != 0; WrittenOffsetColumns offset_columns; bool write_final_mark = (with_final_mark && data_written); diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index f8ace348bd1..46f9188beb8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -601,7 +601,6 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t /// result.num_rows_read if the last granule in range also the last in part (so we have to adjust last granule). { size_t space_left = max_rows; - size_t started_mark = 0; while (space_left && (!stream.isFinished() || !ranges.empty())) { if (stream.isFinished()) @@ -610,7 +609,6 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t stream = Stream(ranges.back().begin, ranges.back().end, merge_tree_reader); result.addRange(ranges.back()); ranges.pop_back(); - started_mark = stream.currentMark(); } size_t current_space = space_left; @@ -622,9 +620,6 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::startReadingChain(size_t auto rows_to_read = std::min(current_space, stream.numPendingRowsInCurrentGranule()); - std::cerr << "(startReadingChain) rows_to_read: " << rows_to_read << "\n"; - std::cerr << "(startReadingChain) current_space: " << current_space << "\n"; - bool last = rows_to_read == space_left; result.addRows(stream.read(result.block, rows_to_read, !last)); result.addGranule(rows_to_read); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 0cb328ce550..8d57fa27868 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -44,6 +44,10 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr non_cached_buffer = std::move(buffer); data_buffer = non_cached_buffer.get(); } + + column_positions.reserve(columns.size()); + for (const auto & column : columns) + column_positions.push_back(data_part->getColumnPosition(column.name)); } size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) @@ -63,30 +67,34 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, from_mark = next_mark; size_t read_rows = 0; + size_t num_columns = columns.size(); + while (read_rows < max_rows_to_read) { size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); - for (const auto & it : columns) + auto it = columns.begin(); + for (size_t i = 0; i < num_columns; ++i, ++it) { - bool append = res.has(it.name); + if (!column_positions[i]) + continue; + + bool append = res.has(it->name); if (!append) - res.insert(ColumnWithTypeAndName(it.type->createColumn(), it.type, it.name)); + res.insert(ColumnWithTypeAndName(it->type->createColumn(), it->type, it->name)); /// To keep offsets shared. TODO Very dangerous. Get rid of this. - MutableColumnPtr column = res.getByName(it.name).column->assumeMutable(); + MutableColumnPtr column = res.getByName(it->name).column->assumeMutable(); try { size_t column_size_before_reading = column->size(); - auto column_position = *data_part->getColumnPosition(it.name); - - readData(it.name, *it.type, *column, from_mark, column_position, rows_to_read); - + readData(it->name, *it->type, *column, from_mark, *column_positions[i], rows_to_read); size_t read_rows_in_column = column->size() - column_size_before_reading; + if (read_rows_in_column < rows_to_read) throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) + - ". Rows expected: "+ toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA); + ". Rows expected: " + toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA); /// For elements of Nested, column_size_before_reading may be greater than column size /// if offsets are not empty and were already read, but elements are empty. @@ -97,14 +105,14 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading column " + it.name + ")"); + e.addMessage("(while reading column " + it->name + ")"); throw; } if (column->size()) - res.getByName(it.name).column = std::move(column); + res.getByName(it->name).column = std::move(column); else - res.erase(it.name); + res.erase(it->name); } ++from_mark; @@ -172,7 +180,6 @@ void MergeTreeReaderCompact::initMarksLoader() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - auto res = std::make_shared(marks_count * columns_num); // std::cerr << "(MergeTreeReaderCompact::loadMarks) marks_count: " << marks_count << "\n"; @@ -236,4 +243,3 @@ void MergeTreeReaderCompact::seekToStart() } } - diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 909f0b3c6c8..f62ea6381e1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -35,6 +35,8 @@ private: MergeTreeMarksLoader marks_loader; + std::vector> column_positions; + size_t next_mark = 0; void initMarksLoader(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 4f7cc48026a..923e16221c1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -213,19 +213,19 @@ void MergeTreeReaderWide::readData( }; double & avg_value_size_hint = avg_value_size_hints[name]; - IDataType::DeserializeBinaryBulkSettings settings; - settings.avg_value_size_hint = avg_value_size_hint; + IDataType::DeserializeBinaryBulkSettings deserialize_settings; + deserialize_settings.avg_value_size_hint = avg_value_size_hint; if (deserialize_binary_bulk_state_map.count(name) == 0) { - settings.getter = get_stream_getter(true); - type.deserializeBinaryBulkStatePrefix(settings, deserialize_binary_bulk_state_map[name]); + deserialize_settings.getter = get_stream_getter(true); + type.deserializeBinaryBulkStatePrefix(deserialize_settings, deserialize_binary_bulk_state_map[name]); } - settings.getter = get_stream_getter(false); - settings.continuous_reading = continue_reading; + deserialize_settings.getter = get_stream_getter(false); + deserialize_settings.continuous_reading = continue_reading; auto & deserialize_state = deserialize_binary_bulk_state_map[name]; - type.deserializeBinaryBulkWithMultipleStreams(column, max_rows_to_read, settings, deserialize_state); + type.deserializeBinaryBulkWithMultipleStreams(column, max_rows_to_read, deserialize_settings, deserialize_state); IDataType::updateAvgValueSizeHint(column, avg_value_size_hint); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp index 36a0b194a58..93da54913c0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp @@ -80,6 +80,10 @@ bool MergeTreeThreadSelectBlockInputStream::getNewTask() owned_uncompressed_cache = storage.global_context.getUncompressedCache(); owned_mark_cache = storage.global_context.getMarkCache(); + std::cerr << "In Part: " << task->data_part->getFullPath() << "\n"; + std::cerr << "task->columns: " << task->columns.toString() << "\n"; + std::cerr << "part->columns: " << task->data_part->columns.toString() << "\n"; + reader = task->data_part->getReader(task->columns, rest_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, IMergeTreeReader::ValueSizeMap{}, profile_callback); From 831f39a3df3841b2614e87b2cdd30b0e35e1501b Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 16 Dec 2019 17:51:19 +0300 Subject: [PATCH 0077/2007] polymorphic parts (development) alter --- dbms/src/Parsers/ParserCreateQuery.h | 1 - .../Storages/MergeTree/AlterAnalysisResult.h | 17 +++ .../Storages/MergeTree/IMergeTreeDataPart.h | 8 ++ dbms/src/Storages/MergeTree/MergeTreeData.cpp | 134 ++++-------------- dbms/src/Storages/MergeTree/MergeTreeData.h | 20 +-- .../MergeTree/MergeTreeDataPartCompact.cpp | 67 ++++++++- .../MergeTree/MergeTreeDataPartCompact.h | 6 + .../MergeTree/MergeTreeDataPartWide.cpp | 82 ++++++++++- .../MergeTree/MergeTreeDataPartWide.h | 5 +- .../MergeTree/MergeTreeReaderCompact.cpp | 8 +- .../MergeTree/MergeTreeReaderCompact.h | 2 - .../MergeTree/MergeTreeReaderWide.cpp | 1 - 12 files changed, 205 insertions(+), 146 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/AlterAnalysisResult.h diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index a4f4da8907e..d9c88353f70 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -266,7 +266,6 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; - class ParserIndexDeclarationList : public IParserBase { protected: diff --git a/dbms/src/Storages/MergeTree/AlterAnalysisResult.h b/dbms/src/Storages/MergeTree/AlterAnalysisResult.h new file mode 100644 index 00000000000..40fd3dfe353 --- /dev/null +++ b/dbms/src/Storages/MergeTree/AlterAnalysisResult.h @@ -0,0 +1,17 @@ +#pragma once +#include + +namespace DB +{ + struct AlterAnalysisResult + { + ExpressionActionsPtr expression = nullptr; + bool force_update_metadata = false; + std::map new_types; + /// For every column that need to be converted: source column name, + /// column name of calculated expression for conversion. + std::vector> conversions; + NamesAndTypesList removed_columns; + Names removed_indices; + }; +} diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 1d4a59634da..7701aff5b73 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -14,6 +14,7 @@ #include #include #include +#include #include // #include // #include @@ -86,6 +87,13 @@ public: /// If no checksums are present returns the name of the first physically existing column. virtual String getColumnNameWithMinumumCompressedSize() const { return columns.front().name; } + virtual NameToNameMap createRenameMapForAlter( + AlterAnalysisResult & /* analysis_result */, + const NamesAndTypesList & /* old_columns */) const + { + return {}; + } + virtual ~IMergeTreeDataPart(); // virtual Checksums check( diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 5c26b5c2492..6121c2d8a50 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1408,106 +1408,14 @@ void MergeTreeData::checkAlter(const AlterCommands & commands, const Context & c analyzeAlterConversions(getColumns().getAllPhysical(), new_columns.getAllPhysical(), getIndices().indices, new_indices.indices); } -/// FIXME implement alter for compact parts -NameToNameMap MergeTreeData::createRenameMap( - const DataPartPtr & part, - const NamesAndTypesList & old_columns, - const AlterAnalysisResult & analysis_result) const -{ - if (!part) - return {}; - const auto & part_mrk_file_extension = part->index_granularity_info.marks_file_extension; - NameToNameMap rename_map; - - for (const auto & index_name : analysis_result.removed_indices) - { - rename_map["skp_idx_" + index_name + ".idx"] = ""; - rename_map["skp_idx_" + index_name + part_mrk_file_extension] = ""; - } - - /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. - std::map stream_counts; - for (const NameAndTypePair & column : old_columns) - { - column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - ++stream_counts[IDataType::getFileNameForStream(column.name, substream_path)]; - }, {}); - } - - for (const auto & column : analysis_result.removed_columns) - { - if (part->hasColumnFiles(column.name, *column.type)) - { - column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(column.name, substream_path); - - /// Delete files if they are no longer shared with another column. - if (--stream_counts[file_name] == 0) - { - rename_map[file_name + ".bin"] = ""; - rename_map[file_name + part_mrk_file_extension] = ""; - } - }, {}); - } - } - - for (const auto & elem : analysis_result.conversions) - { - /// Column name for temporary filenames before renaming. NOTE The is unnecessarily tricky. - const auto & source_name = elem.first; - String temporary_column_name = source_name + " converting"; - - /// After conversion, we need to rename temporary files into original. - analysis_result.new_types.at(source_name)->enumerateStreams( - [&](const IDataType::SubstreamPath & substream_path) - { - /// Skip array sizes, because they cannot be modified in ALTER. - if (!substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) - return; - - String original_file_name = IDataType::getFileNameForStream(source_name, substream_path); - String temporary_file_name = IDataType::getFileNameForStream(temporary_column_name, substream_path); - - rename_map[temporary_file_name + ".bin"] = original_file_name + ".bin"; - rename_map[temporary_file_name + part_mrk_file_extension] = original_file_name + part_mrk_file_extension; - }, {}); - } - - - if (part && !rename_map.empty()) - { - WriteBufferFromOwnString out; - out << "Will "; - bool first = true; - for (const auto & [from, to] : rename_map) - { - if (!first) - out << ", "; - first = false; - if (to.empty()) - out << "remove " << from; - else - out << "rename " << from << " to " << to; - } - out << " in part " << part->name; - LOG_DEBUG(log, out.str()); - } - - return rename_map; -} - - -MergeTreeData::AlterAnalysisResult MergeTreeData::analyzeAlterConversions( +AlterAnalysisResult MergeTreeData::analyzeAlterConversions( const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, const IndicesASTs & old_indices, const IndicesASTs & new_indices) const { AlterAnalysisResult res; - const auto settings = getSettings(); std::set new_indices_set; for (const auto & index_decl : new_indices) @@ -1568,21 +1476,6 @@ MergeTreeData::AlterAnalysisResult MergeTreeData::analyzeAlterConversions( } } - if (!res.conversions.empty()) - { - /// Give proper names for temporary columns with conversion results. - NamesWithAliases projection; - projection.reserve(res.conversions.size()); - - for (const auto & source_and_expression : res.conversions) - { - String temporary_column_name = source_and_expression.first + " converting"; - projection.emplace_back(source_and_expression.second, temporary_column_name); - } - - res.expression->add(ExpressionAction::project(projection)); - } - return res; } @@ -1671,7 +1564,28 @@ void MergeTreeData::alterDataPart( const auto & part = transaction->getDataPart(); auto res = analyzeAlterConversions(part->columns, new_columns, getIndices().indices, new_indices); - transaction->rename_map = createRenameMap(part, part->columns, res); + + NamesAndTypesList additional_columns; + transaction->rename_map = part->createRenameMapForAlter(res, part->columns); + + if (!transaction->rename_map.empty()) + { + WriteBufferFromOwnString out; + out << "Will "; + bool first = true; + for (const auto & [from, to] : transaction->rename_map) + { + if (!first) + out << ", "; + first = false; + if (to.empty()) + out << "remove " << from; + else + out << "rename " << from << " to " << to; + } + out << " in part " << part->name; + LOG_DEBUG(log, out.str()); + } size_t num_files_to_modify = transaction->rename_map.size(); size_t num_files_to_remove = 0; @@ -1744,6 +1658,8 @@ void MergeTreeData::alterDataPart( static_cast(part->bytes_on_disk) / this->getTotalActiveSizeInBytes()); ExpressionBlockInputStream in(part_in, res.expression); + std::cerr << "im.header: " << in.getHeader().dumpStructure() << "\n"; + /** Don't write offsets for arrays, because ALTER never change them * (MODIFY COLUMN could only change types of elements but never modify array sizes). * Also note that they does not participate in 'rename_map'. diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index f5be20a63f2..942b44444cf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -904,31 +905,12 @@ protected: void setTTLExpressions(const ColumnsDescription::ColumnTTLs & new_column_ttls, const ASTPtr & new_ttl_table_ast, bool only_check = false); - using NameToType = std::map; - - struct AlterAnalysisResult - { - ExpressionActionsPtr expression = nullptr; - bool force_update_metadata = false; - NameToType new_types; - /// For every column that need to be converted: source column name, - /// column name of calculated expression for conversion. - std::vector> conversions; - NamesAndTypesList removed_columns; - Names removed_indices; - }; - AlterAnalysisResult analyzeAlterConversions( const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, const IndicesASTs & old_indices, const IndicesASTs & new_indices) const; - NameToNameMap createRenameMap( - const DataPartPtr & part, - const NamesAndTypesList & old_columns, - const AlterAnalysisResult & analysis_result) const; - /// Expression for column type conversion. /// If no conversions are needed, out_expression=nullptr. /// out_rename_map maps column files for the out_expression onto new table files. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 66b1a8b1511..2413fa3be72 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -92,8 +92,16 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( const WriterSettings & writer_settings, const MergeTreeIndexGranularity & computed_index_granularity) const { - return std::make_unique( - getFullPath(), storage, columns_list, indices_to_recalc, + NamesAndTypesList ordered_columns_list; + std::copy_if(columns_list.begin(), columns_list.end(),std::back_inserter(ordered_columns_list), + [this](const auto & column) { return getColumnPosition(column.name) != std::nullopt; }); + + /// Order of writing is important in compact format + ordered_columns_list.sort([this](const auto & lhs, const auto & rhs) + { return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); }); + + return std::make_unique( + getFullPath(), storage, ordered_columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); } @@ -191,12 +199,63 @@ bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const if (!getColumnPosition(column_name)) return false; /// FIXME replace everywhere hardcoded "data" - auto bin_checksum = checksums.files.find("data.bin"); - auto mrk_checksum = checksums.files.find("data" + index_granularity_info.marks_file_extension); + auto bin_checksum = checksums.files.find(String(DATA_FILE_NAME) + DATA_FILE_EXTENSION); + auto mrk_checksum = checksums.files.find(DATA_FILE_NAME + index_granularity_info.marks_file_extension); return (bin_checksum != checksums.files.end() && mrk_checksum != checksums.files.end()); } +NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( + AlterAnalysisResult & analysis_result, + const NamesAndTypesList & /* old_columns */) const +{ + const auto & part_mrk_file_extension = index_granularity_info.marks_file_extension; + NameToNameMap rename_map; + + for (const auto & index_name : analysis_result.removed_indices) + { + rename_map["skp_idx_" + index_name + ".idx"] = ""; + rename_map["skp_idx_" + index_name + part_mrk_file_extension] = ""; + } + + /// We have to rewrite all data if any column has been changed. + if (!analysis_result.removed_columns.empty() || !analysis_result.conversions.empty()) + { + if (!analysis_result.expression) + analysis_result.expression = std::make_shared(NamesAndTypesList(), storage.global_context); + + NameSet altered_columns; + NamesWithAliases projection; + + for (const auto & column : analysis_result.removed_columns) + altered_columns.insert(column.name); + + for (const auto & [source_name, result_name] : analysis_result.conversions) + { + altered_columns.insert(source_name); + projection.emplace_back(result_name, source_name); + } + + /// Add other part columns to read + for (const auto & column : columns) + { + if (!altered_columns.count(column.name)) + { + analysis_result.expression->addInput(column); + projection.emplace_back(column.name, ""); + } + } + + analysis_result.expression->add(ExpressionAction::project(projection)); + + String data_temp_name = String(DATA_FILE_NAME) + "_converting"; + rename_map[data_temp_name + DATA_FILE_EXTENSION] = String(DATA_FILE_NAME) + DATA_FILE_EXTENSION; + rename_map[data_temp_name + part_mrk_file_extension] = DATA_FILE_NAME + part_mrk_file_extension; + } + + return rename_map; +} + void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { UNUSED(require_part_metadata); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 43e588ccab2..ad898efe890 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -34,6 +34,9 @@ public: using Checksums = MergeTreeDataPartChecksums; using Checksum = MergeTreeDataPartChecksums::Checksum; + static constexpr auto DATA_FILE_NAME = "data"; + static constexpr auto DATA_FILE_EXTENSION = ".bin"; + MergeTreeDataPartCompact( const MergeTreeData & storage_, const String & name_, @@ -79,6 +82,9 @@ public: bool hasColumnFiles(const String & column_name, const IDataType & type) const override; + NameToNameMap createRenameMapForAlter( + AlterAnalysisResult & analysis_result, + const NamesAndTypesList & old_columns) const override; ~MergeTreeDataPartCompact() override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index c080ba10c3c..8bf9ce034b0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -31,10 +31,6 @@ namespace DB { -// namespace -// { -// } - namespace ErrorCodes { extern const int FILE_DOESNT_EXIST; @@ -338,8 +334,6 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const } } - - bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDataType & type) const { bool res = true; @@ -358,4 +352,80 @@ bool MergeTreeDataPartWide::hasColumnFiles(const String & column_name, const IDa return res; } +NameToNameMap MergeTreeDataPartWide::createRenameMapForAlter( + AlterAnalysisResult & analysis_result, + const NamesAndTypesList & old_columns) const +{ + const auto & part_mrk_file_extension = index_granularity_info.marks_file_extension; + NameToNameMap rename_map; + + for (const auto & index_name : analysis_result.removed_indices) + { + rename_map["skp_idx_" + index_name + ".idx"] = ""; + rename_map["skp_idx_" + index_name + part_mrk_file_extension] = ""; + } + + /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. + std::map stream_counts; + for (const NameAndTypePair & column : old_columns) + { + column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + ++stream_counts[IDataType::getFileNameForStream(column.name, substream_path)]; + }, {}); + } + + for (const auto & column : analysis_result.removed_columns) + { + if (hasColumnFiles(column.name, *column.type)) + { + column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + String file_name = IDataType::getFileNameForStream(column.name, substream_path); + + /// Delete files if they are no longer shared with another column. + if (--stream_counts[file_name] == 0) + { + rename_map[file_name + ".bin"] = ""; + rename_map[file_name + part_mrk_file_extension] = ""; + } + }, {}); + } + } + + if (!analysis_result.conversions.empty()) + { + /// Give proper names for temporary columns with conversion results. + NamesWithAliases projection; + projection.reserve(analysis_result.conversions.size()); + for (const auto & source_and_expression : analysis_result.conversions) + { + /// Column name for temporary filenames before renaming. NOTE The is unnecessarily tricky. + const auto & source_name = source_and_expression.first; + String temporary_column_name = source_name + " converting"; + + projection.emplace_back(source_and_expression.second, temporary_column_name); + + /// After conversion, we need to rename temporary files into original. + analysis_result.new_types.at(source_name)->enumerateStreams( + [&](const IDataType::SubstreamPath & substream_path) + { + /// Skip array sizes, because they cannot be modified in ALTER. + if (!substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) + return; + + String original_file_name = IDataType::getFileNameForStream(source_name, substream_path); + String temporary_file_name = IDataType::getFileNameForStream(temporary_column_name, substream_path); + + rename_map[temporary_file_name + ".bin"] = original_file_name + ".bin"; + rename_map[temporary_file_name + part_mrk_file_extension] = original_file_name + part_mrk_file_extension; + }, {}); + } + + analysis_result.expression->add(ExpressionAction::project(projection)); + } + + return rename_map; +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 8984fef5dac..02c86403644 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -26,7 +26,6 @@ namespace DB struct ColumnSize; class MergeTreeData; - /// Description of the data part. class MergeTreeDataPartWide : public IMergeTreeDataPart { @@ -79,6 +78,10 @@ public: ColumnSize getColumnSize(const String & column_name, const IDataType & type) const override; + NameToNameMap createRenameMapForAlter( + AlterAnalysisResult & analysis_result, + const NamesAndTypesList & old_columns) const override; + ~MergeTreeDataPartWide() override; bool hasColumnFiles(const String & column, const IDataType & type) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 8d57fa27868..3030db4aa9c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -1,4 +1,5 @@ #include +#include #include namespace DB @@ -21,11 +22,12 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr { initMarksLoader(); size_t buffer_size = settings.max_read_buffer_size; + const String full_data_path = path + MergeTreeDataPartCompact::DATA_FILE_NAME + MergeTreeDataPartCompact::DATA_FILE_EXTENSION; if (uncompressed_cache) { auto buffer = std::make_unique( - path + "data.bin", uncompressed_cache, 0, settings.min_bytes_to_use_direct_io, buffer_size); + full_data_path, uncompressed_cache, 0, settings.min_bytes_to_use_direct_io, buffer_size); // if (profile_callback) // buffer->setProfileCallback(profile_callback, clock_type); @@ -36,7 +38,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr else { auto buffer = std::make_unique( - path + "data.bin", 0, settings.min_bytes_to_use_direct_io, buffer_size); + full_data_path, 0, settings.min_bytes_to_use_direct_io, buffer_size); // if (profile_callback) // buffer->setProfileCallback(profile_callback, clock_type); @@ -206,7 +208,7 @@ void MergeTreeReaderCompact::initMarksLoader() return res; }; - auto mrk_path = data_part->index_granularity_info.getMarksFilePath(path + NAME_OF_FILE_WITH_DATA); + auto mrk_path = data_part->index_granularity_info.getMarksFilePath(path + MergeTreeDataPartCompact::DATA_FILE_NAME); marks_loader = MergeTreeMarksLoader{mark_cache, std::move(mrk_path), load, settings.save_marks_in_cache, columns_num}; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index f62ea6381e1..b09b68b38e3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -47,8 +47,6 @@ private: void readData(const String & name, const IDataType & type, IColumn & column, size_t from_mark, size_t column_position, size_t rows_to_read); - static auto constexpr NAME_OF_FILE_WITH_DATA = "data"; - /// Columns that are read. friend class MergeTreeRangeReader::DelayedStream; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 923e16221c1..5bc41422405 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -15,7 +15,6 @@ namespace DB namespace { using OffsetColumns = std::map; - constexpr auto DATA_FILE_EXTENSION = ".bin"; } From 59faa4927b7d30308b34237279ef1f7134876ac5 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 18 Dec 2019 16:09:58 +0300 Subject: [PATCH 0078/2007] polymorphic parts (development) alter update --- dbms/src/DataStreams/ExpressionBlockInputStream.cpp | 1 + dbms/src/Interpreters/MutationsInterpreter.cpp | 2 ++ dbms/src/Interpreters/MutationsInterpreter.h | 4 ++-- dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp | 10 ++++++++++ dbms/src/Storages/MergeTree/IMergeTreeDataPart.h | 3 +++ dbms/src/Storages/MergeTree/MergeTreeData.cpp | 8 +++++--- .../Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 10 ++++++---- .../Storages/MergeTree/MergeTreeDataMergerMutator.h | 2 +- .../Storages/MergeTree/MergeTreeDataPartCompact.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp | 5 +++-- dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../src/Storages/MergeTree/MergedBlockOutputStream.cpp | 8 ++++---- .../Storages/MergeTree/StorageFromMergeTreeDataPart.h | 8 ++++---- dbms/src/Storages/MutationCommands.h | 3 +++ 14 files changed, 47 insertions(+), 23 deletions(-) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index 51adc462ef6..4e709c503be 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -8,6 +8,7 @@ namespace DB ExpressionBlockInputStream::ExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_) : expression(expression_) { + std::cerr << "expression: " << expression->dumpActions(); children.push_back(input); cached_header = children.back()->getHeader(); expression->execute(cached_header, true); diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index a025ba87c1a..f2ff79def75 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -314,6 +314,8 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector &p if (i > 0) prepared_stages[i].output_columns = prepared_stages[i - 1].output_columns; + else if (!commands.additional_columns.empty()) + prepared_stages[i].output_columns.insert(commands.additional_columns.begin(), commands.additional_columns.end()); if (prepared_stages[i].output_columns.size() < all_columns.size()) { diff --git a/dbms/src/Interpreters/MutationsInterpreter.h b/dbms/src/Interpreters/MutationsInterpreter.h index 5df5b1ca2f7..7e18c8dc5a7 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.h +++ b/dbms/src/Interpreters/MutationsInterpreter.h @@ -18,7 +18,7 @@ class Context; class MutationsInterpreter { public: - MutationsInterpreter(StoragePtr storage_, std::vector commands_, const Context & context_) + MutationsInterpreter(StoragePtr storage_, MutationCommands commands_, const Context & context_) : storage(std::move(storage_)) , commands(std::move(commands_)) , context(context_) @@ -48,7 +48,7 @@ private: BlockInputStreamPtr addStreamsForLaterStages(const std::vector & prepared_stages, BlockInputStreamPtr in) const; StoragePtr storage; - std::vector commands; + MutationCommands commands; const Context & context; /// A sequence of mutation commands is executed as a sequence of stages. Each stage consists of several diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 3964b335d37..b555a75c140 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -847,4 +847,14 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const DiskSpace::ReservationPtr cloning_directory.copyTo(path_to_clone); } +bool isCompactPart(const MergeTreeDataPartPtr & data_part) +{ + return (data_part && data_part->getType() == MergeTreeDataPartType::COMPACT); +} + +bool isWidePart(const MergeTreeDataPartPtr & data_part) +{ + return (data_part && data_part->getType() == MergeTreeDataPartType::WIDE); +} + } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 7701aff5b73..abc237dcc85 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -363,4 +363,7 @@ private: using MergeTreeDataPartState = IMergeTreeDataPart::State; using MergeTreeDataPartPtr = std::shared_ptr; +bool isCompactPart(const MergeTreeDataPartPtr & data_part); +bool isWidePart(const MergeTreeDataPartPtr & data_part); + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 6121c2d8a50..65c055b8b14 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2599,9 +2599,11 @@ void MergeTreeData::loadPartAndFixMetadata(MutableDataPartPtr part) { String full_part_path = part->getFullPath(); - /// Earlier the list of columns was written incorrectly. Delete it and re-create. - if (Poco::File(full_part_path + "columns.txt").exists()) - Poco::File(full_part_path + "columns.txt").remove(); + /// Earlier the list of columns was written incorrectly. Delete it and re-create. + /// FIXME looks not right + if (isWidePart(part)) + if (Poco::File(full_part_path + "columns.txt").exists()) + Poco::File(full_part_path + "columns.txt").remove(); part->loadColumnsChecksumsIndexes(false, true); part->modification_time = Poco::File(full_part_path).getLastModified().epochTime(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index e4cbd2d47bc..29e5606fbe3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -909,7 +909,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTemporaryPart( const FutureMergedMutatedPart & future_part, - const std::vector & commands, + const MutationCommands & commands, MergeListEntry & merge_entry, const Context & context, DiskSpace::Reservation * space_reservation, @@ -931,13 +931,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor CurrentMetrics::Increment num_mutations{CurrentMetrics::PartMutation}; const auto & source_part = future_part.parts[0]; - auto storage_from_source_part = StorageFroIMergeTreeDataPart::create(source_part); + auto storage_from_source_part = StorageFromMergeTreeDataPart::create(source_part); auto context_for_reading = context; - context_for_reading.getSettingsRef().merge_tree_uniform_read_distribution = 0; context_for_reading.getSettingsRef().max_threads = 1; - std::vector commands_for_part; + MutationCommands commands_for_part; std::copy_if( std::cbegin(commands), std::cend(commands), std::back_inserter(commands_for_part), @@ -947,6 +946,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor future_part.parts[0]->info.partition_id == data.getPartitionIDFromQuery( command.partition, context_for_reading); }); + + if (isCompactPart(source_part)) + commands_for_part.additional_columns = source_part->columns.getNames(); MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 4d87cfe35ff..d43f8201251 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -102,7 +102,7 @@ public: /// Mutate a single data part with the specified commands. Will create and return a temporary part. MergeTreeData::MutableDataPartPtr mutatePartToTemporaryPart( const FutureMergedMutatedPart & future_part, - const std::vector & commands, + const MutationCommands & commands, MergeListEntry & merge_entry, const Context & context, DiskSpace::Reservation * disk_reservation, TableStructureReadLockHolder & table_lock_holder); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 2413fa3be72..affce9a2969 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -93,7 +93,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( const MergeTreeIndexGranularity & computed_index_granularity) const { NamesAndTypesList ordered_columns_list; - std::copy_if(columns_list.begin(), columns_list.end(),std::back_inserter(ordered_columns_list), + std::copy_if(columns_list.begin(), columns_list.end(), std::back_inserter(ordered_columns_list), [this](const auto & column) { return getColumnPosition(column.name) != std::nullopt; }); /// Order of writing is important in compact format @@ -198,7 +198,7 @@ bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const { if (!getColumnPosition(column_name)) return false; - /// FIXME replace everywhere hardcoded "data" + auto bin_checksum = checksums.files.find(String(DATA_FILE_NAME) + DATA_FILE_EXTENSION); auto mrk_checksum = checksums.files.find(DATA_FILE_NAME + index_granularity_info.marks_file_extension); diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 2dbdc6d7ad6..a444f94ba6b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -20,8 +20,9 @@ const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, siz if (!marks) loadMarks(); if (column_index >= columns_num) - throw Exception("", ErrorCodes::LOGICAL_ERROR); /// FIXME better exception - + throw Exception("Column index: " + toString(column_index) + + " is out of range (" + toString(columns_num) + ")", ErrorCodes::LOGICAL_ERROR); + return (*marks)[row_index * columns_num + column_index]; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 3030db4aa9c..bcfb774b921 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -216,7 +216,7 @@ void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) { MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); - // std::cerr << "(MergeTreeReaderCompact::seekToMark) mark: (" << mark.offset_in_compressed_file << ", " << mark.offset_in_decompressed_block << "\n"; + std::cerr << "(MergeTreeReaderCompact::seekToMark) mark: (" << mark.offset_in_compressed_file << ", " << mark.offset_in_decompressed_block << "\n"; try { diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index d1a181ab0be..038d18af95a 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -49,11 +49,11 @@ MergedBlockOutputStream::MergedBlockOutputStream( if (aio_threshold > 0 && !merged_column_to_size.empty()) { - for (const auto & it : columns_list) + for (const auto & column : columns_list) { - auto it2 = merged_column_to_size.find(it.name); - if (it2 != merged_column_to_size.end()) - writer_settings.estimated_size += it2->second; + auto size_it = merged_column_to_size.find(column.name); + if (size_it != merged_column_to_size.end()) + writer_settings.estimated_size += size_it->second; } } diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 2c2a0748cfc..8a1de4ab1f8 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -12,11 +12,11 @@ namespace DB { /// A Storage that allows reading from a single MergeTree data part. -class StorageFroIMergeTreeDataPart : public ext::shared_ptr_helper, public IStorage +class StorageFromMergeTreeDataPart : public ext::shared_ptr_helper, public IStorage { - friend struct ext::shared_ptr_helper; + friend struct ext::shared_ptr_helper; public: - String getName() const override { return "FroIMergeTreeDataPart"; } + String getName() const override { return "FromMergeTreeDataPart"; } String getTableName() const override { return part->storage.getTableName() + " (part " + part->name + ")"; } String getDatabaseName() const override { return part->storage.getDatabaseName(); } @@ -40,7 +40,7 @@ public: } protected: - StorageFroIMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) + StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(part_->storage.getVirtuals()), part(part_) { setColumns(part_->storage.getColumns()); diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index c03bed3b7b5..5b2a7d906fb 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -46,6 +47,8 @@ public: void writeText(WriteBuffer & out) const; void readText(ReadBuffer & in); + + Names additional_columns; }; } From ae74d28f87b821e1680b647126a7dc863c6765ae Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 18 Dec 2019 17:23:27 +0300 Subject: [PATCH 0079/2007] polymorphic parts (development) fix alter --- dbms/src/DataStreams/ExpressionBlockInputStream.cpp | 1 - dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp | 1 + dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp | 2 +- dbms/src/Storages/MutationCommands.h | 1 + 4 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp index 4e709c503be..51adc462ef6 100644 --- a/dbms/src/DataStreams/ExpressionBlockInputStream.cpp +++ b/dbms/src/DataStreams/ExpressionBlockInputStream.cpp @@ -8,7 +8,6 @@ namespace DB ExpressionBlockInputStream::ExpressionBlockInputStream(const BlockInputStreamPtr & input, const ExpressionActionsPtr & expression_) : expression(expression_) { - std::cerr << "expression: " << expression->dumpActions(); children.push_back(input); cached_header = children.back()->getHeader(); expression->execute(cached_header, true); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index b555a75c140..79c8584f5fd 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -224,6 +224,7 @@ time_t IMergeTreeDataPart::getMaxTime() const void IMergeTreeDataPart::setColumns(const NamesAndTypesList & columns_) { columns = columns_; + sample_block.clear(); for (const auto & column : columns) sample_block.insert({column.type, column.name}); index_granularity_info.initialize(storage, getType(), columns.size()); diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index a444f94ba6b..087cada6762 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -21,7 +21,7 @@ const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, siz loadMarks(); if (column_index >= columns_num) throw Exception("Column index: " + toString(column_index) - + " is out of range (" + toString(columns_num) + ")", ErrorCodes::LOGICAL_ERROR); + + " is out of range [0, " + toString(columns_num) + ")", ErrorCodes::LOGICAL_ERROR); return (*marks)[row_index * columns_num + column_index]; } diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index 5b2a7d906fb..b465740bfde 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -48,6 +48,7 @@ public: void writeText(WriteBuffer & out) const; void readText(ReadBuffer & in); + /// Columns that we need to read except ones needed for expressions. Names additional_columns; }; From 55b7db716ace1eb4913b578a095a33a18f634cc4 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 18 Dec 2019 18:54:45 +0300 Subject: [PATCH 0080/2007] polymorphic parts (development) cleanup --- .../AsynchronousBlockInputStream.h | 1 + dbms/src/DataStreams/SplittingTransform.h | 89 ------------------- dbms/src/DataStreams/TTLBlockInputStream.cpp | 2 +- dbms/src/DataStreams/TTLBlockInputStream.h | 4 +- dbms/src/Interpreters/SystemLog.h | 1 - dbms/src/Parsers/ParserCreateQuery.h | 1 + .../Storages/MergeTree/IMergeTreeDataPart.h | 12 +-- .../MergeTree/IMergeTreeDataPartWriter.cpp | 2 +- .../MergeTree/IMergeTreeDataPartWriter.h | 29 +++--- .../Storages/MergeTree/IMergeTreeReader.cpp | 2 +- .../src/Storages/MergeTree/IMergeTreeReader.h | 6 +- .../MergeTree/IMergedBlockOutputStream.cpp | 2 +- .../MergeTree/IMergedBlockOutputStream.h | 26 +----- .../MergeTreeBaseSelectBlockInputStream.cpp | 2 +- .../MergeTreeBaseSelectBlockInputStream.h | 4 +- .../MergeTree/MergeTreeBlockReadUtils.h | 2 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 5 +- .../MergeTree/MergeTreeDataPartCompact.h | 4 +- .../MergeTree/MergeTreeDataPartWide.cpp | 4 +- .../MergeTree/MergeTreeDataPartWide.h | 4 +- .../MergeTreeDataPartWriterCompact.cpp | 2 +- .../MergeTreeDataPartWriterCompact.h | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 9 +- .../MergeTree/MergeTreeDataPartWriterWide.h | 4 +- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 8 +- .../MergeTree/MergeTreeDataSelectExecutor.h | 6 +- ...ReaderSettings.h => MergeTreeIOSettings.h} | 6 +- .../MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../MergeTree/MergeTreeReaderCompact.h | 4 +- .../MergeTree/MergeTreeReaderStream.cpp | 2 +- .../MergeTree/MergeTreeReaderStream.h | 4 +- .../MergeTree/MergeTreeReaderWide.cpp | 2 +- .../Storages/MergeTree/MergeTreeReaderWide.h | 2 +- ...MergeTreeReverseSelectBlockInputStream.cpp | 2 +- .../MergeTreeReverseSelectBlockInputStream.h | 2 +- .../MergeTreeSelectBlockInputStream.cpp | 2 +- .../MergeTreeSelectBlockInputStream.h | 2 +- .../MergeTreeSequentialBlockInputStream.cpp | 2 +- .../MergeTreeThreadSelectBlockInputStream.cpp | 2 +- .../MergeTreeThreadSelectBlockInputStream.h | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 4 +- .../MergedColumnOnlyOutputStream.cpp | 2 +- dbms/src/Storages/MergeTree/checkDataPart.cpp | 2 +- 43 files changed, 78 insertions(+), 199 deletions(-) delete mode 100644 dbms/src/DataStreams/SplittingTransform.h rename dbms/src/Storages/MergeTree/{MergeTreeReaderSettings.h => MergeTreeIOSettings.h} (80%) diff --git a/dbms/src/DataStreams/AsynchronousBlockInputStream.h b/dbms/src/DataStreams/AsynchronousBlockInputStream.h index a32d3049995..93c695f20c9 100644 --- a/dbms/src/DataStreams/AsynchronousBlockInputStream.h +++ b/dbms/src/DataStreams/AsynchronousBlockInputStream.h @@ -70,6 +70,7 @@ public: return ready.tryWait(milliseconds); } + Block getHeader() const override { return children.at(0)->getHeader(); } void cancel(bool kill) override diff --git a/dbms/src/DataStreams/SplittingTransform.h b/dbms/src/DataStreams/SplittingTransform.h deleted file mode 100644 index 8cd563b89a7..00000000000 --- a/dbms/src/DataStreams/SplittingTransform.h +++ /dev/null @@ -1,89 +0,0 @@ -#pragma once - -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int EMPTY_LIST_OF_COLUMNS_PASSED; - extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; -} - -class SplittingTransform -{ -public: - SplittingTransform() {} - - void add(MutableColumns && columns) - { - assertColumnsHasEqualRows(columns); - blocks_queue.push(std::move(columns)); - } - - MutableColumns get(size_t preferable_size) - { - if (blocks_queue.empty()) - return {}; - - auto & columns = blocks_queue.front(); - size_t rows = columns[0]->size(); - - MutableColumns result; - if (rows <= preferable_size && current_offset == 0) - { - result = std::move(columns); - blocks_queue.pop(); - current_offset = 0; - } - else - { - result.reserve(columns.size()); - size_t result_size = std::min(rows - current_offset, preferable_size); - for (const auto & column : columns) - result.push_back((*column->cut(current_offset, result_size)).mutate()); - - current_offset += result_size; - } - - if (current_offset == rows) - { - blocks_queue.pop(); - current_offset = 0; - } - - return result; - } - - MutableColumns addAndGet(MutableColumns && columns, size_t preferable_size) - { - add(std::move(columns)); - return get(preferable_size); - } - -private: - - void assertColumnsHasEqualRows(const MutableColumns & columns) - { - if (columns.empty()) - throw Exception("Empty list of columns passed", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); - - size_t rows = 0; - for (const auto & column : columns) - { - if (!rows) - rows = column->size(); - else if (column->size() != rows) - throw Exception("Sizes of columns doesn't match", ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH); - } - } - - // size_t max_block_size_rows = 0; - // size_t max_block_size_bytes = 0; - std::queue blocks_queue; - size_t current_offset = 0; -}; - -} diff --git a/dbms/src/DataStreams/TTLBlockInputStream.cpp b/dbms/src/DataStreams/TTLBlockInputStream.cpp index 3e92555f890..234343a5016 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.cpp +++ b/dbms/src/DataStreams/TTLBlockInputStream.cpp @@ -36,7 +36,7 @@ TTLBlockInputStream::TTLBlockInputStream( { if (force || isTTLExpired(ttl_info.min)) { - new_ttl_infos.columns_ttl.emplace(name,IMergeTreeDataPart::TTLInfo{}); + new_ttl_infos.columns_ttl.emplace(name, IMergeTreeDataPart::TTLInfo{}); empty_columns.emplace(name); auto it = column_defaults.find(name); diff --git a/dbms/src/DataStreams/TTLBlockInputStream.h b/dbms/src/DataStreams/TTLBlockInputStream.h index 067118279cb..29e1b73902e 100644 --- a/dbms/src/DataStreams/TTLBlockInputStream.h +++ b/dbms/src/DataStreams/TTLBlockInputStream.h @@ -39,8 +39,8 @@ private: time_t current_time; bool force; - IMergeTreeDataPart::TTLInfos old_ttl_infos; - IMergeTreeDataPart::TTLInfos new_ttl_infos; + IMergeTreeDataPart::TTLInfos old_ttl_infos; + IMergeTreeDataPart::TTLInfos new_ttl_infos; NameSet empty_columns; size_t rows_removed = 0; diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 01c963a53ef..b00f77b7622 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -354,7 +354,6 @@ void SystemLog::flushImpl(EntryType reason) /// In case of exception, also clean accumulated data - to avoid locking. data.clear(); } - if (reason == EntryType::FORCE_FLUSH) { std::lock_guard lock(condvar_mutex); diff --git a/dbms/src/Parsers/ParserCreateQuery.h b/dbms/src/Parsers/ParserCreateQuery.h index d9c88353f70..a4f4da8907e 100644 --- a/dbms/src/Parsers/ParserCreateQuery.h +++ b/dbms/src/Parsers/ParserCreateQuery.h @@ -266,6 +266,7 @@ protected: bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override; }; + class ParserIndexDeclarationList : public IParserBase { protected: diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index abc237dcc85..dc267569607 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -14,17 +14,11 @@ #include #include #include +#include #include #include -// #include -// #include -// #include -// #include -// #include #include -#include - #include #include @@ -60,7 +54,7 @@ public: const MarkRanges & mark_ranges, UncompressedCache * uncompressed_cache, MarkCache * mark_cache, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}) const = 0; @@ -68,7 +62,7 @@ public: const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec_, - const WriterSettings & writer_settings, + const MergeTreeWriterSettings & writer_settings, const MergeTreeIndexGranularity & computed_index_granularity = {}) const = 0; virtual bool isStoredOnDisk() const = 0; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index dfbf09cee0c..e96ba1b299f 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -65,7 +65,7 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, - const WriterSettings & settings_, + const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_, bool need_finish_last_granule_) : part_path(part_path_) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index b4884d18d8e..75219f080b5 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -19,7 +19,6 @@ class IMergeTreeDataPartWriter { public: using WrittenOffsetColumns = std::set; - using MarkWithOffset = std::pair; struct ColumnStream { @@ -65,18 +64,24 @@ public: const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, - const WriterSettings & settings, + const MergeTreeWriterSettings & settings, const MergeTreeIndexGranularity & index_granularity, bool need_finish_last_granule); + virtual ~IMergeTreeDataPartWriter(); + virtual void write( const Block & block, const IColumn::Permutation * permutation = nullptr, /* Blocks with already sorted index columns */ const Block & primary_key_block = {}, const Block & skip_indexes_block = {}) = 0; - virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) = 0; + void calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows); + void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, size_t rows); - virtual ~IMergeTreeDataPartWriter(); + /// Shift mark and offset to prepare read next mark. + /// You must call it after calling write method and optionally + /// calling calculations of primary and skip indices. + void next(); /// Count index_granularity for block and store in `index_granularity` void fillIndexGranularity(const Block & block); @@ -88,10 +93,7 @@ public: return Columns(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); } - const MergeTreeData::ColumnSizeByName & getColumnsSizes() const - { - return columns_sizes; - } + const MergeTreeData::ColumnSizeByName & getColumnsSizes() const { return columns_sizes; } void setOffsetColumns(WrittenOffsetColumns * written_offset_columns_, bool skip_offsets_) { @@ -101,11 +103,10 @@ public: void initSkipIndices(); void initPrimaryIndex(); - void calculateAndSerializePrimaryIndex(const Block & primary_index_block, size_t rows); - void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, size_t rows); + + virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) = 0; void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums); void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); - void next(); protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; @@ -122,13 +123,15 @@ protected: std::vector skip_indices; - WriterSettings settings; + MergeTreeWriterSettings settings; bool compute_granularity; bool with_final_mark; bool need_finish_last_granule; size_t current_mark = 0; + + /// The offset to the first row of the block for which you want to write the index. size_t index_offset = 0; size_t next_mark = 0; @@ -162,6 +165,4 @@ protected: }; -using MergeTreeWriterPtr = std::unique_ptr; - } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp index 75b01a22081..4135c8eda36 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -28,7 +28,7 @@ namespace ErrorCodes IMergeTreeReader::IMergeTreeReader(const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, - const MarkRanges & all_mark_ranges_, const ReaderSettings & settings_, + const MarkRanges & all_mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_) : data_part(data_part_), avg_value_size_hints(avg_value_size_hints_), path(data_part_->getFullPath()) , columns(columns_), uncompressed_cache(uncompressed_cache_), mark_cache(mark_cache_) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 109146b6cb2..f08589f034b 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -24,7 +24,7 @@ public: UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & all_mark_ranges_, - const ReaderSettings & settings_, + const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}); /// Return the number of rows has been read or zero if there is no columns to read. @@ -73,7 +73,7 @@ protected: MarkCache * mark_cache; /// If save_marks_in_cache is false, then, if marks are not in cache, we will load them but won't save in the cache, to avoid evicting other data. - ReaderSettings settings; + MergeTreeReaderSettings settings; const MergeTreeData & storage; MarkRanges all_mark_ranges; @@ -81,6 +81,4 @@ protected: friend class MergeTreeRangeReader::DelayedStream; }; -using MergeTreeReaderPtr = std::unique_ptr; - } diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 7ca9d77b96c..e54c190c03c 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include namespace DB diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index defb77db2d4..6f312c5071e 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -37,33 +37,9 @@ protected: String part_path; - /// The offset to the first row of the block for which you want to write the index. - // size_t index_offset = 0; - - // size_t current_mark = 0; - - /// Number of mark in data from which skip indices have to start - /// aggregation. I.e. it's data mark number, not skip indices mark. - // size_t skip_index_data_mark = 0; - - // const bool can_use_adaptive_granularity; - // const std::string marks_file_extension; - // const bool blocks_are_granules_size; - - // MergeTreeIndexGranularity index_granularity; - - // const bool compute_granularity; - - // std::vector skip_indices; - // std::vector> skip_indices_streams; - // MergeTreeIndexAggregators skip_indices_aggregators; - // std::vector skip_index_filling; - static Block getBlockAndPermute(const Block & block, const Names & names, const IColumn::Permutation * permutation); - MergeTreeWriterPtr writer; - - // const bool with_final_mark; + IMergeTreeDataPart::MergeTreeWriterPtr writer; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp index c89f87da881..043bb6c1f42 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.cpp @@ -26,7 +26,7 @@ MergeTreeBaseSelectBlockInputStream::MergeTreeBaseSelectBlockInputStream( UInt64 max_block_size_rows_, UInt64 preferred_block_size_bytes_, UInt64 preferred_max_column_in_block_size_bytes_, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, bool use_uncompressed_cache_, const Names & virt_column_names_) : diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h index 842f663c2f9..e360a4ffced 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectBlockInputStream.h @@ -23,7 +23,7 @@ public: UInt64 max_block_size_rows_, UInt64 preferred_block_size_bytes_, UInt64 preferred_max_column_in_block_size_bytes_, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, bool use_uncompressed_cache_, const Names & virt_column_names_ = {}); @@ -59,7 +59,7 @@ protected: UInt64 preferred_block_size_bytes; UInt64 preferred_max_column_in_block_size_bytes; - ReaderSettings reader_settings; + MergeTreeReaderSettings reader_settings; bool use_uncompressed_cache; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index 077ae84824a..b5fe7151872 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -4,7 +4,7 @@ #include #include #include -// #include +// #include namespace DB diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index affce9a2969..8ec6997523d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -76,10 +76,11 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( const MarkRanges & mark_ranges, UncompressedCache * uncompressed_cache, MarkCache * mark_cache, - const ReaderSettings & reader_settings, + const MergeTreeReaderSettings & reader_settings, const ValueSizeMap & avg_value_size_hints, const ReadBufferFromFileBase::ProfileCallback & /* profile_callback */) const { + /// FIXME maybe avoid shared_from_this return std::make_unique( shared_from_this(), columns_to_read, uncompressed_cache, mark_cache, mark_ranges, reader_settings, avg_value_size_hints); @@ -89,7 +90,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec, - const WriterSettings & writer_settings, + const MergeTreeWriterSettings & writer_settings, const MergeTreeIndexGranularity & computed_index_granularity) const { NamesAndTypesList ordered_columns_list; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index ad898efe890..797f5e1d1e0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -55,7 +55,7 @@ public: const MarkRanges & mark_ranges, UncompressedCache * uncompressed_cache, MarkCache * mark_cache, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, const ValueSizeMap & avg_value_size_hints = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{}) const override; @@ -63,7 +63,7 @@ public: const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec_, - const WriterSettings & writer_settings, + const MergeTreeWriterSettings & writer_settings, const MergeTreeIndexGranularity & computed_index_granularity = {}) const override; bool isStoredOnDisk() const override { return true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 8bf9ce034b0..e6ac23f386f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -73,7 +73,7 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader( const MarkRanges & mark_ranges, UncompressedCache * uncompressed_cache, MarkCache * mark_cache, - const ReaderSettings & reader_settings, + const MergeTreeReaderSettings & reader_settings, const ValueSizeMap & avg_value_size_hints, const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { @@ -85,7 +85,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec, - const WriterSettings & writer_settings, + const MergeTreeWriterSettings & writer_settings, const MergeTreeIndexGranularity & computed_index_granularity) const { return std::make_unique( diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 02c86403644..25a42fbfad1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -51,7 +51,7 @@ public: const MarkRanges & mark_ranges, UncompressedCache * uncompressed_cache, MarkCache * mark_cache, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, const ValueSizeMap & avg_value_size_hints = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{}) const override; @@ -59,7 +59,7 @@ public: const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, const CompressionCodecPtr & default_codec_, - const WriterSettings & writer_settings, + const MergeTreeWriterSettings & writer_settings, const MergeTreeIndexGranularity & computed_index_granularity = {}) const override; bool isStoredOnDisk() const override { return true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 4d1009c8092..70bf634a0bd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -17,7 +17,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, - const WriterSettings & settings_, + const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 2f6fc7b99c4..f7b6ba394e5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -14,7 +14,7 @@ public: const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, - const WriterSettings & settings, + const MergeTreeWriterSettings & settings, const MergeTreeIndexGranularity & index_granularity); void write(const Block & block, const IColumn::Permutation * permutation = nullptr, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 6f65a47fceb..23ebdd4eea4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -15,7 +15,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, - const WriterSettings & settings_, + const MergeTreeWriterSettings & settings_, const MergeTreeIndexGranularity & index_granularity_) : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, indices_to_recalc_, @@ -207,9 +207,8 @@ size_t MergeTreeDataPartWriterWide::writeSingleGranule( return from_row + number_of_rows; } -/// column must not be empty. (column.size() !== 0) - -std::pair MergeTreeDataPartWriterWide::writeColumn( +/// Column must not be empty. (column.size() !== 0) +void MergeTreeDataPartWriterWide::writeColumn( const String & name, const IDataType & type, const IColumn & column, @@ -288,8 +287,6 @@ std::pair MergeTreeDataPartWriterWide::writeColumn( next_mark = current_column_mark; next_index_offset = current_row - total_rows; - - return std::make_pair(current_column_mark, current_row - total_rows); } void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index fbbe2e71580..6a8715f9ea8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -16,7 +16,7 @@ public: const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, - const WriterSettings & settings, + const MergeTreeWriterSettings & settings, const MergeTreeIndexGranularity & index_granularity); void write(const Block & block, const IColumn::Permutation * permutation = nullptr, @@ -29,7 +29,7 @@ public: /// Write data of one column. /// Return how many marks were written and /// how many rows were written for last mark - MarkWithOffset writeColumn( + void writeColumn( const String & name, const IDataType & type, const IColumn & column, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 8e243e38e30..abb15415004 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -567,7 +567,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( BlockInputStreams res; - ReaderSettings reader_settings = + MergeTreeReaderSettings reader_settings = { .min_bytes_to_use_direct_io = settings.min_bytes_to_use_direct_io, .max_read_buffer_size = settings.max_read_buffer_size, @@ -677,7 +677,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const ReaderSettings & reader_settings) const + const MergeTreeReaderSettings & reader_settings) const { std::cerr << "marks to read: "; for (const auto & part : parts) @@ -844,7 +844,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithO const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, const Settings & settings, - const ReaderSettings & reader_settings) const + const MergeTreeReaderSettings & reader_settings) const { size_t sum_marks = 0; SortingInfoPtr sorting_info = query_info.sorting_info; @@ -1052,7 +1052,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const ReaderSettings & reader_settings) const + const MergeTreeReaderSettings & reader_settings) const { const auto data_settings = data.getSettings(); size_t sum_marks = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 61b4a59b363..b34d50537f3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -55,7 +55,7 @@ private: const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const ReaderSettings & reader_settings) const; + const MergeTreeReaderSettings & reader_settings) const; BlockInputStreams spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts, @@ -67,7 +67,7 @@ private: const ExpressionActionsPtr & sorting_key_prefix_expr, const Names & virt_columns, const Settings & settings, - const ReaderSettings & reader_settings) const; + const MergeTreeReaderSettings & reader_settings) const; BlockInputStreams spreadMarkRangesAmongStreamsFinal( RangesInDataParts && parts, @@ -77,7 +77,7 @@ private: const SelectQueryInfo & query_info, const Names & virt_columns, const Settings & settings, - const ReaderSettings & reader_settings) const; + const MergeTreeReaderSettings & reader_settings) const; /// Get the approximate value (bottom estimate - only by full marks) of the number of rows falling under the index. size_t getApproximateTotalRowsToRead( diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h similarity index 80% rename from dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h rename to dbms/src/Storages/MergeTree/MergeTreeIOSettings.h index 3fc23f90e38..a6f6203e3ff 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -5,16 +5,16 @@ namespace DB { -struct ReaderSettings +struct MergeTreeReaderSettings { size_t min_bytes_to_use_direct_io = 0; size_t max_read_buffer_size = 0; bool save_marks_in_cache = false; }; -struct WriterSettings +struct MergeTreeWriterSettings { - WriterSettings(const Settings & global_settings, bool can_use_adaptive_granularity_, bool blocks_are_granules_size_ = false) + MergeTreeWriterSettings(const Settings & global_settings, bool can_use_adaptive_granularity_, bool blocks_are_granules_size_ = false) : min_compress_block_size(global_settings.min_compress_block_size) , max_compress_block_size(global_settings.min_compress_block_size) , aio_threshold(global_settings.min_bytes_to_use_direct_io) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index bcfb774b921..4708c7eb4d8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -15,7 +15,7 @@ namespace ErrorCodes MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, - const MarkRanges & mark_ranges_, const ReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_) + const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_) : IMergeTreeReader(data_part_, columns_ , uncompressed_cache_, mark_cache_, mark_ranges_ , settings_, avg_value_size_hints_) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index b09b68b38e3..ef51555bee4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -10,7 +10,7 @@ namespace DB /// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. /// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. -/// Avoids loading the marks file if it is not needed (e.g. when reading the whole part). +/// Avoids loading the marks file if it is not needed (e.g. when reading the whole part).ca class MergeTreeReaderCompact : public IMergeTreeReader { public: @@ -19,7 +19,7 @@ public: UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & mark_ranges_, - const ReaderSettings & settings_, + const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}); /// Return the number of rows has been read or zero if there is no columns to read. diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 793d3b25dcb..58830493bb7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -17,7 +17,7 @@ namespace ErrorCodes MergeTreeReaderStream::MergeTreeReaderStream( const String & path_prefix_,const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, - const ReaderSettings & settings, + const MergeTreeReaderSettings & settings, MarkCache * mark_cache_, UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index 7c6d8f956af..a369b13ec64 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include @@ -25,7 +25,7 @@ public: MergeTreeReaderStream( const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, - const ReaderSettings & settings_, + const MergeTreeReaderSettings & settings_, MarkCache * mark_cache, UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, ReadingMode mode_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 5bc41422405..ff12cd1f277 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -28,7 +28,7 @@ namespace ErrorCodes MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, - const MarkRanges & mark_ranges_, const ReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, + const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) : IMergeTreeReader(data_part_, columns_ diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h index 37266e83164..83ef594fce9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -19,7 +19,7 @@ public: UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & mark_ranges_, - const ReaderSettings & settings_, + const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}, clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp index 2f991a53b01..c2d16278d8b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp @@ -24,7 +24,7 @@ MergeTreeReverseSelectBlockInputStream::MergeTreeReverseSelectBlockInputStream( bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, bool check_columns, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, size_t part_index_in_query_, bool quiet) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h index e7abeb7a933..d27a2dfb7fa 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.h @@ -27,7 +27,7 @@ public: bool use_uncompressed_cache, const PrewhereInfoPtr & prewhere_info, bool check_columns, - const ReaderSettings & reader_settings, + const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, size_t part_index_in_query = 0, bool quiet = false); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp index d3763e30dd3..909076c751f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp @@ -24,7 +24,7 @@ MergeTreeSelectBlockInputStream::MergeTreeSelectBlockInputStream( bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, bool check_columns_, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_, size_t part_index_in_query_, bool quiet) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h index b3ebf8b2d7e..0b1aa1d9c13 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.h @@ -27,7 +27,7 @@ public: bool use_uncompressed_cache, const PrewhereInfoPtr & prewhere_info, bool check_columns, - const ReaderSettings & reader_settings, + const MergeTreeReaderSettings & reader_settings, const Names & virt_column_names = {}, size_t part_index_in_query = 0, bool quiet = false); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 545f0891846..b4e4bea8128 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -55,7 +55,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( std::cerr << "(MergeTreeSequentialBlockInputStream) part: " << data_part_->getFullPath() << "\n"; std::cerr << "(MergeTreeSequentialBlockInputStream) columns_for_reader: " << columns_for_reader.toString() << "\n"; - ReaderSettings reader_settings = + MergeTreeReaderSettings reader_settings = { /// This is hack .min_bytes_to_use_direct_io = read_with_direct_io ? 1UL : std::numeric_limits::max(), diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp index 93da54913c0..2dc1f4a2419 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.cpp @@ -17,7 +17,7 @@ MergeTreeThreadSelectBlockInputStream::MergeTreeThreadSelectBlockInputStream( const MergeTreeData & storage_, const bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_) : MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h index e47513909e7..3875e6260d8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputStream.h @@ -24,7 +24,7 @@ public: const MergeTreeData & storage_, const bool use_uncompressed_cache_, const PrewhereInfoPtr & prewhere_info_, - const ReaderSettings & reader_settings_, + const MergeTreeReaderSettings & reader_settings_, const Names & virt_column_names_); String getName() const override { return "MergeTreeThread"; } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 038d18af95a..110b6134c14 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -27,7 +27,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( : IMergedBlockOutputStream(data_part) , columns_list(columns_list_) { - WriterSettings writer_settings(data_part->storage.global_context.getSettings(), + MergeTreeWriterSettings writer_settings(data_part->storage.global_context.getSettings(), data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); init(); @@ -43,7 +43,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( : IMergedBlockOutputStream(data_part) , columns_list(columns_list_) { - WriterSettings writer_settings(data_part->storage.global_context.getSettings(), + MergeTreeWriterSettings writer_settings(data_part->storage.global_context.getSettings(), data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); writer_settings.aio_threshold = aio_threshold; diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 28e838f8f02..6f834e4ac83 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -20,7 +20,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( // if (index_granularity_info_) // std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; - WriterSettings writer_settings( + MergeTreeWriterSettings writer_settings( data_part->storage.global_context.getSettings(), index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity()); writer_settings.filename_suffix = filename_suffix; diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index 2303ec38efa..0474dcb9b03 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -201,7 +201,7 @@ MergeTreeData::DataPart::Checksums checkDataPart( ReadBufferFromFile buf(path + "columns.txt"); columns.readText(buf); assertEOF(buf); - } + } /// Checksums from file checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums. MergeTreeData::DataPart::Checksums checksums_txt; From 6f673407ff48fe4e19851ff80af7c8825adc7299 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 18 Dec 2019 19:27:49 +0300 Subject: [PATCH 0081/2007] polymorphic parts (development) cleanup --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 1 - .../MergeTree/IMergeTreeDataPartWriter.h | 1 - .../MergeTree/MergeTreeIndexGranularity.h | 2 +- .../MergeTreeIndexGranularityInfo.cpp | 26 ++++++++++++++++++ .../MergeTree/MergeTreeIndexGranularityInfo.h | 27 ++----------------- .../MergeTree/MergeTreePartsMover.cpp | 2 +- .../Storages/MergeTree/MergeTreeReadPool.cpp | 2 +- .../MergeTree/MergeTreeReaderCompact.h | 3 +-- dbms/src/Storages/MergeTree/MergeTreeWriter.h | 13 --------- dbms/src/Storages/StorageMergeTree.cpp | 1 + 10 files changed, 33 insertions(+), 45 deletions(-) delete mode 100644 dbms/src/Storages/MergeTree/MergeTreeWriter.h diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index e96ba1b299f..0a0ec3d6422 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -7,7 +7,6 @@ namespace DB namespace { - // constexpr auto DATA_FILE_EXTENSION = ".bin"; constexpr auto INDEX_FILE_EXTENSION = ".idx"; } diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 75219f080b5..b958376ba43 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -8,7 +8,6 @@ #include #include #include -// #include #include diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h index 053eebccb25..7a17ddfa132 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -5,7 +5,7 @@ namespace DB { -/// Class contains information about index granularity in rows ofIMergeTreeDataPart +/// Class contains information about index granularity in rows of MergeTreeDataPart /// Inside it contains vector of partial sums of rows after mark: /// |-----|---|----|----| /// | 5 | 8 | 12 | 16 | diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index f29b276c2bd..460ed4df3ed 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -81,4 +81,30 @@ void MergeTreeIndexGranularityInfo::setNonAdaptive() index_granularity_bytes = 0; } +std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) +{ + switch(part_type) + { + case MergeTreeDataPartType::WIDE: + return ".mrk2"; + case MergeTreeDataPartType::COMPACT: + return ".mrk3"; + default: + throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); + } +} + +size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num) +{ + switch(part_type) + { + case MergeTreeDataPartType::WIDE: + return sizeof(UInt64) * 3; + case MergeTreeDataPartType::COMPACT: + return sizeof(UInt64) * (columns_num * 2 + 1); + default: + throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); + } +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index ae91d604b34..a51bed5bf69 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -63,30 +63,7 @@ private: constexpr inline auto getNonAdaptiveMrkExtension() { return ".mrk"; } constexpr inline auto getNonAdaptiveMrkSize() { return sizeof(MarkInCompressedFile) * 2; } -inline std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) -{ - switch(part_type) - { - case MergeTreeDataPartType::WIDE: - return ".mrk2"; - case MergeTreeDataPartType::COMPACT: - return ".mrk3"; - default: - throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); - } -} - -inline size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num) -{ - switch(part_type) - { - case MergeTreeDataPartType::WIDE: - return sizeof(UInt64) * 3; - case MergeTreeDataPartType::COMPACT: - return sizeof(UInt64) * (columns_num * 2 + 1); - default: - throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); - } -} +std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type); +size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num); } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp index 8476dc0794a..84617dfb687 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -77,7 +77,7 @@ bool MergeTreePartsMover::selectPartsForMove( const AllowedMovingPredicate & can_move, const std::lock_guard & /* moving_parts_lock */) { - auto data_parts = data->getDataPartsVector(); + MergeTreeData::DataPartsVector data_parts = data->getDataPartsVector(); if (data_parts.empty()) return false; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index 7093bef21e4..3bbefa3ade7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -33,7 +33,7 @@ MergeTreeReadPool::MergeTreeReadPool( for (auto & part_ranges : parts_ranges) std::reverse(std::begin(part_ranges.ranges), std::end(part_ranges.ranges)); - /// parts don't contain duplicateIMergeTreeDataPart's. + /// parts don't contain duplicate MergeTreeDataPart's. const auto per_part_sum_marks = fillPerPartInfo(parts_, check_columns_); fillPerThreadInfo(threads_, sum_marks_, per_part_sum_marks, parts_, min_marks_for_concurrent_read_); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index ef51555bee4..bd47d13e5ea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -41,8 +41,7 @@ private: void initMarksLoader(); void seekToStart(); - void seekToMark(size_t row, size_t col); - const MarkInCompressedFile & getMark(size_t row, size_t col); + void seekToMark(size_t row_index, size_t column_index); void readData(const String & name, const IDataType & type, IColumn & column, size_t from_mark, size_t column_position, size_t rows_to_read); diff --git a/dbms/src/Storages/MergeTree/MergeTreeWriter.h b/dbms/src/Storages/MergeTree/MergeTreeWriter.h deleted file mode 100644 index 5176b153a6d..00000000000 --- a/dbms/src/Storages/MergeTree/MergeTreeWriter.h +++ /dev/null @@ -1,13 +0,0 @@ -#include - -namespace DB -{ - -class MergeTreeWriter -{ - -}; - -using MergeTreeWriterPtr = std::shared_ptr; - -} diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index a3f621f621e..daffbbd149f 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -307,6 +307,7 @@ void StorageMergeTree::alter( context.getDatabase(current_database_name)->alterTable(context, current_table_name, new_columns, new_indices, new_constraints, storage_modifier); + /// Reinitialize primary key because primary key column types might have changed. setProperties(new_order_by_ast, new_primary_key_ast, new_columns, new_indices, new_constraints); From 258e8d61fb66a561034cc4a5faf1b9fa030c4080 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 18 Dec 2019 19:41:11 +0300 Subject: [PATCH 0082/2007] polymorphic parts (development) cleanup --- dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp | 10 +++++----- dbms/src/Storages/MergeTree/IMergeTreeDataPart.h | 4 ++-- .../Storages/MergeTree/IMergeTreeDataPartWriter.cpp | 10 +++++----- .../Storages/MergeTree/IMergeTreeDataPartWriter.h | 8 ++++---- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 12 ++++++------ dbms/src/Storages/MergeTree/MergeTreeData.h | 6 +++--- .../MergeTree/MergeTreeDataMergerMutator.cpp | 10 +++++----- .../Storages/MergeTree/MergeTreeDataPartCompact.cpp | 10 +++++----- .../Storages/MergeTree/MergeTreeDataPartCompact.h | 4 ++-- .../src/Storages/MergeTree/MergeTreeDataPartWide.cpp | 4 ++-- .../MergeTree/MergeTreeDataPartWriterCompact.cpp | 12 ++++++------ .../MergeTree/MergeTreeDataPartWriterCompact.h | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 4 ++-- .../Storages/MergeTree/MergeTreeDataPartWriterWide.h | 4 ++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeIOSettings.h | 2 +- .../MergeTree/MergeTreeIndexGranularityInfo.cpp | 6 +++--- dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp | 6 +++--- dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h | 2 +- dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp | 2 +- .../Storages/MergeTree/MergeTreeReaderCompact.cpp | 4 ++-- .../src/Storages/MergeTree/MergeTreeReaderStream.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp | 4 ++-- dbms/src/Storages/MergeTree/MergeTreeReaderWide.h | 2 +- .../MergeTree/MergeTreeSelectBlockInputStream.cpp | 2 +- .../MergeTreeSequentialBlockInputStream.cpp | 2 +- .../Storages/MergeTree/MergedBlockOutputStream.cpp | 4 +--- .../MergeTree/MergedColumnOnlyOutputStream.cpp | 2 +- dbms/src/Storages/MergeTree/checkDataPart.cpp | 2 +- dbms/src/Storages/MutationCommands.h | 2 +- 33 files changed, 76 insertions(+), 78 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 79c8584f5fd..ba9c52b59dd 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -184,8 +184,8 @@ std::optional IMergeTreeDataPart::getColumnPosition(const String & colum { if (!sample_block.has(column_name)) return {}; - return sample_block.getPositionByName(column_name); -} + return sample_block.getPositionByName(column_name); +} DayNum IMergeTreeDataPart::getMinDate() const { @@ -583,7 +583,7 @@ void IMergeTreeDataPart::loadColumns(bool require) { is_frozen = !poco_file_path.canWrite(); ReadBufferFromFile file = openForReading(path); - columns.readText(file); + columns.readText(file); } index_granularity_info.initialize(storage, getType(), columns.size()); @@ -597,7 +597,7 @@ void IMergeTreeDataPart::loadColumnSizes() if (columns_num == 0) throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - + auto column_sizes_path = getFullPath() + "columns_sizes.txt"; auto columns_sizes_file = Poco::File(column_sizes_path); if (!columns_sizes_file.exists()) @@ -772,7 +772,7 @@ void IMergeTreeDataPart::checkConsistency(bool /* require_part_metadata */) cons String IMergeTreeDataPart::typeToString(Type type) { - switch(type) + switch (type) { case Type::WIDE: return "Wide"; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index dc267569607..6e7c43f02be 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -64,7 +64,7 @@ public: const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & writer_settings, const MergeTreeIndexGranularity & computed_index_granularity = {}) const = 0; - + virtual bool isStoredOnDisk() const = 0; @@ -168,7 +168,7 @@ public: std::atomic bytes_on_disk {0}; /// 0 - if not counted; /// Is used from several threads without locks (it is changed with ALTER). /// May not contain size of checksums.txt and columns.txt - + time_t modification_time = 0; /// When the part is removed from the working set. Changes once. mutable std::atomic remove_time { std::numeric_limits::max() }; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 0a0ec3d6422..1605beadeaf 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -61,7 +61,7 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, - const std::vector & indices_to_recalc_, + const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, @@ -81,7 +81,7 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( { if (settings.blocks_are_granules_size && !index_granularity.empty()) throw Exception("Can't take information about index granularity from blocks, when non empty index_granularity array specified", ErrorCodes::LOGICAL_ERROR); - + Poco::File part_dir(part_path); if (!part_dir.exists()) part_dir.createDirectories(); @@ -126,12 +126,12 @@ void fillIndexGranularityImpl( /// We should be less or equal than fixed index granularity index_granularity_for_block = std::min(fixed_index_granularity_rows, index_granularity_for_block); - + size_t current_row; for (current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) { size_t rows_left_in_block = rows_in_block - current_row; - + // if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block) // { // /// If enough rows are left, create a new granule. Otherwise, extend previous granule. @@ -342,7 +342,7 @@ void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::Da } - std::cerr << "(finishPrimaryIndexSerialization) marks_count: " << index_granularity.getMarksCount() << "\n"; + std::cerr << "(finishPrimaryIndexSerialization) marks_count: " << index_granularity.getMarksCount() << "\n"; index_stream->next(); checksums.files["primary.idx"].file_size = index_stream->count(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index b958376ba43..645a44f8238 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -60,7 +60,7 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, - const std::vector & indices_to_recalc, + const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, const MergeTreeWriterSettings & settings, @@ -78,7 +78,7 @@ public: void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, size_t rows); /// Shift mark and offset to prepare read next mark. - /// You must call it after calling write method and optionally + /// You must call it after calling write method and optionally /// calling calculations of primary and skip indices. void next(); @@ -102,8 +102,8 @@ public: void initSkipIndices(); void initPrimaryIndex(); - - virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) = 0; + + virtual void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync = false) = 0; void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums); void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 65c055b8b14..2dab4ebcc2c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -818,7 +818,7 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) MergeTreePartInfo part_info; if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return; - + auto part = createPart(part_name, part_info, part_disk_ptr, part_name); bool broken = false; @@ -1485,7 +1485,7 @@ MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_uncompressed, s const auto settings = getSettings(); if (bytes_uncompressed < settings->min_bytes_for_wide_part || rows_count < settings->min_rows_for_wide_part) return MergeTreeDataPartType::COMPACT; - + return MergeTreeDataPartType::WIDE; } @@ -1497,7 +1497,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, if (type == MergeTreeDataPartType::COMPACT) return std::make_shared(*this, name, part_info, disk, relative_path); else if (type == MergeTreeDataPartType::WIDE) - return std::make_shared(*this, name, part_info, disk, relative_path); + return std::make_shared(*this, name, part_info, disk, relative_path); else throw Exception("Unknown part type", ErrorCodes::LOGICAL_ERROR); } @@ -1513,7 +1513,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( { auto part = createPart(name, choosePartType(bytes_uncompressed, rows_count), part_info, disk, relative_path); part->setColumns(columns_list); - /// Don't save rows_count count here as it can change later + /// Don't save rows_count count here as it can change later return part; } @@ -1552,7 +1552,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( } return createPart(name, type, part_info, disk, relative_path); -} +} void MergeTreeData::alterDataPart( const NamesAndTypesList & new_columns, @@ -1639,7 +1639,7 @@ void MergeTreeData::alterDataPart( } DataPart::Checksums add_checksums; - + if (transaction->rename_map.empty() && !res.force_update_metadata) { transaction->clear(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 942b44444cf..83ce69f6529 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -187,7 +187,7 @@ public: const MergeTreePartInfo & part_info,const DiskSpace::DiskPtr & disk, const NamesAndTypesList & columns, size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; - + MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, const String & relative_path) const; @@ -196,7 +196,7 @@ public: /// FIXME make this inside this function MutableDataPartPtr createPart(const String & name, const DiskSpace::DiskPtr & disk, const String & relative_path) const; - + MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, const DiskSpace::DiskPtr & disk, const String & relative_path) const; @@ -910,7 +910,7 @@ protected: const NamesAndTypesList & new_columns, const IndicesASTs & old_indices, const IndicesASTs & new_indices) const; - + /// Expression for column type conversion. /// If no conversions are needed, out_expression=nullptr. /// out_rename_map maps column files for the out_expression onto new table files. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 29e5606fbe3..f27420745fc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -568,7 +568,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor size_t estimated_bytes_uncompressed = 0; for (const auto & part : parts) estimated_bytes_uncompressed += part->getTotalColumnsSize().data_uncompressed; - + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( future_part.name, future_part.part_info, @@ -622,7 +622,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor for (const MergeTreeData::DataPartPtr & part : parts) part->accumulateColumnSizes(merged_column_to_size); - + column_sizes = ColumnSizeEstimator(merged_column_to_size, merging_column_names, gathering_column_names); } else @@ -946,7 +946,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor future_part.parts[0]->info.partition_id == data.getPartitionIDFromQuery( command.partition, context_for_reading); }); - + if (isCompactPart(source_part)) commands_for_part.additional_columns = source_part->columns.getNames(); @@ -959,7 +959,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor } else LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation); - + auto in = mutations_interpreter.execute(table_lock_holder); const auto & updated_header = mutations_interpreter.getUpdatedHeader(); @@ -980,7 +980,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor source_part->bytes_on_disk, source_part->rows_count, "tmp_mut_" + future_part.name); - + new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 8ec6997523d..85a78352f25 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -52,7 +52,7 @@ namespace ErrorCodes // return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); // } -MergeTreeDataPartCompact::MergeTreeDataPartCompact( +MergeTreeDataPartCompact::MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, const DiskSpace::DiskPtr & disk_, @@ -154,7 +154,7 @@ String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const if (!minimum_size_column) throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); - + return *minimum_size_column; } @@ -192,7 +192,7 @@ void MergeTreeDataPartCompact::loadIndexGranularity() if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); - index_granularity.setInitialized(); + index_granularity.setInitialized(); } bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const IDataType &) const @@ -246,9 +246,9 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( projection.emplace_back(column.name, ""); } } - + analysis_result.expression->add(ExpressionAction::project(projection)); - + String data_temp_name = String(DATA_FILE_NAME) + "_converting"; rename_map[data_temp_name + DATA_FILE_EXTENSION] = String(DATA_FILE_NAME) + DATA_FILE_EXTENSION; rename_map[data_temp_name + part_mrk_file_extension] = DATA_FILE_NAME + part_mrk_file_extension; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 797f5e1d1e0..636391647cb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -44,7 +44,7 @@ public: const DiskSpace::DiskPtr & disk_, const std::optional & relative_path_ = {}); - MergeTreeDataPartCompact( + MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, const DiskSpace::DiskPtr & disk_, @@ -58,7 +58,7 @@ public: const MergeTreeReaderSettings & reader_settings_, const ValueSizeMap & avg_value_size_hints = ValueSizeMap{}, const ReadBufferFromFileBase::ProfileCallback & profile_callback = ReadBufferFromFileBase::ProfileCallback{}) const override; - + MergeTreeWriterPtr getWriter( const NamesAndTypesList & columns_list, const std::vector & indices_to_recalc, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index e6ac23f386f..c644aea4716 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -49,7 +49,7 @@ namespace ErrorCodes // return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); // } -MergeTreeDataPartWide::MergeTreeDataPartWide( +MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, const DiskSpace::DiskPtr & disk_, @@ -92,7 +92,7 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( getFullPath(), storage, columns_list, indices_to_recalc, index_granularity_info.marks_file_extension, default_codec, writer_settings, computed_index_granularity); -} +} /// Takes into account the fact that several columns can e.g. share their .size substreams. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 70bf634a0bd..fa9d1e279f0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -14,7 +14,7 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, - const std::vector & indices_to_recalc_, + const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, @@ -78,7 +78,7 @@ void MergeTreeDataPartWriterCompact::write( auto result = squashing.add(result_block.mutateColumns()); if (!result.ready) return; - + result_block = header.cloneWithColumns(std::move(result.columns)); writeBlock(result_block); @@ -86,7 +86,7 @@ void MergeTreeDataPartWriterCompact::write( void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) { - size_t total_rows = block.rows(); + size_t total_rows = block.rows(); size_t from_mark = current_mark; size_t current_row = 0; @@ -100,7 +100,7 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) /// There could already be enough data to compress into the new block. if (stream->compressed.offset() >= settings.min_compress_block_size) stream->compressed.next(); - + size_t next_row = 0; writeIntBinary(rows_to_write, stream->marks); for (const auto & it : columns_list) @@ -139,13 +139,13 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith /// We can't calculate compressed size by single column in compact format. size_t uncompressed_size = stream->compressed.count(); - columns_sizes[column.name].add(ColumnSize{0, 0, uncompressed_size - old_uncompressed_size}); + columns_sizes[column.name].add(ColumnSize{0, 0, uncompressed_size - old_uncompressed_size}); return from_row + number_of_rows; } void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) -{ +{ auto result = squashing.add({}); if (result.ready && !result.columns.empty()) writeBlock(header.cloneWithColumns(std::move(result.columns))); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index f7b6ba394e5..8037b876884 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -11,7 +11,7 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, - const std::vector & indices_to_recalc, + const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, const MergeTreeWriterSettings & settings, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 23ebdd4eea4..b3d06bb946f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -12,7 +12,7 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( const String & part_path_, const MergeTreeData & storage_, const NamesAndTypesList & columns_list_, - const std::vector & indices_to_recalc_, + const std::vector & indices_to_recalc_, const String & marks_file_extension_, const CompressionCodecPtr & default_codec_, const MergeTreeWriterSettings & settings_, @@ -80,7 +80,7 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( }; } -void MergeTreeDataPartWriterWide::write(const Block & block, +void MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 6a8715f9ea8..ad8a91c3ec8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -13,7 +13,7 @@ public: const String & part_path, const MergeTreeData & storage, const NamesAndTypesList & columns_list, - const std::vector & indices_to_recalc, + const std::vector & indices_to_recalc, const String & marks_file_extension, const CompressionCodecPtr & default_codec, const MergeTreeWriterSettings & settings, @@ -55,7 +55,7 @@ private: WrittenOffsetColumns & offset_columns, size_t number_of_rows, DB::IDataType::SubstreamPath & path); - + void writeFinalMark( const std::string & column_name, const DataTypePtr column_type, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index abb15415004..69a70717a2e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -567,7 +567,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts( BlockInputStreams res; - MergeTreeReaderSettings reader_settings = + MergeTreeReaderSettings reader_settings = { .min_bytes_to_use_direct_io = settings.min_bytes_to_use_direct_io, .max_read_buffer_size = settings.max_read_buffer_size, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 5eea3f38c46..107285f1b7e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -206,7 +206,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa NamesAndTypesList columns = data.getColumns().getAllPhysical().filter(block.getNames()); auto new_data_part = data.createPart( - part_name, new_part_info, + part_name, new_part_info, reservation->getDisk(), columns, expected_size, diff --git a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h index a6f6203e3ff..ff2b1362bca 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -1,5 +1,5 @@ #pragma once -#include +#include #include namespace DB diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 460ed4df3ed..d441f10d753 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -21,7 +21,7 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( for (Poco::DirectoryIterator part_it(path_to_part); part_it != end; ++part_it) { const auto & ext = "." + part_it.path().getExtension(); - if (ext == getNonAdaptiveMrkExtension() + if (ext == getNonAdaptiveMrkExtension() || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::WIDE) || ext == getAdaptiveMrkExtension(MergeTreeDataPartType::COMPACT)) return ext; @@ -83,7 +83,7 @@ void MergeTreeIndexGranularityInfo::setNonAdaptive() std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) { - switch(part_type) + switch (part_type) { case MergeTreeDataPartType::WIDE: return ".mrk2"; @@ -96,7 +96,7 @@ std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num) { - switch(part_type) + switch (part_type) { case MergeTreeDataPartType::WIDE: return sizeof(UInt64) * 3; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 30e9fa050af..709b370c33d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -8,7 +8,7 @@ MergeTreeIndexReader::MergeTreeIndexReader( MergeTreeIndexPtr index_, MergeTreeData::DataPartPtr part_, size_t marks_count_, const MarkRanges & all_mark_ranges_) : index(index_), stream( part_->getFullPath() + index->getFileName(), ".idx", marks_count_, - all_mark_ranges_, + all_mark_ranges_, { 0, DBMS_DEFAULT_BUFFER_SIZE, false}, nullptr, nullptr, part_->getFileSizeOrZero(index->getFileName() + ".idx"), &part_->index_granularity_info, diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 087cada6762..8bdc8fc8307 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -20,9 +20,9 @@ const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, siz if (!marks) loadMarks(); if (column_index >= columns_num) - throw Exception("Column index: " + toString(column_index) + throw Exception("Column index: " + toString(column_index) + " is out of range [0, " + toString(columns_num) + ")", ErrorCodes::LOGICAL_ERROR); - + return (*marks)[row_index * columns_num + column_index]; } @@ -47,7 +47,7 @@ void MergeTreeMarksLoader::loadMarks() marks = load(); if (!marks) - throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); + throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h index aefcaff7aa1..316be9d051d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -16,7 +16,7 @@ public: const LoadFunc & load_func_, bool save_marks_in_cache_, size_t columns_num_ = 1); - + const MarkInCompressedFile & getMark(size_t row_index, size_t column_index = 0); bool initialized() const { return marks != nullptr; } diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp index 84617dfb687..e364d5b8825 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -109,7 +109,7 @@ bool MergeTreePartsMover::selectPartsForMove( /// Don't report message to log, because logging is excessive if (!can_move(part, &reason)) continue; - + auto to_insert = need_to_move.find(part->disk); if (to_insert != need_to_move.end()) to_insert->second.add(part); diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 46f9188beb8..3a90fed05a9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -483,7 +483,7 @@ size_t MergeTreeRangeReader::Stream::ceilRowsToCompleteGranules(size_t rows_num) size_t from_mark = current_mark; while (result < rows_num && from_mark < last_mark) result += index_granularity->getMarkRows(from_mark++); - + return result; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 4708c7eb4d8..389a21e0ebd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -95,7 +95,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column < rows_to_read) - throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) + + throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) + ". Rows expected: " + toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA); /// For elements of Nested, column_size_before_reading may be greater than column size @@ -142,7 +142,7 @@ void MergeTreeReaderCompact::readData( /// FIXME seek only if needed seekToMark(from_mark, column_position); - + IDataType::DeserializeBinaryBulkSettings deserialize_settings; deserialize_settings.getter = [&](IDataType::SubstreamPath) -> ReadBuffer * { return data_buffer; }; // deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 58830493bb7..9bf1ac7a36c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -117,11 +117,11 @@ void MergeTreeReaderStream::initMarksLoader() /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - std::cerr << "data_file_extension: " << data_file_extension << '\n'; + std::cerr << "data_file_extension: " << data_file_extension << '\n'; size_t file_size = Poco::File(mrk_path).getSize(); size_t mark_size = mode == ReadingMode::INDEX - ? index_granularity_info->skip_index_mark_size_in_bytes + ? index_granularity_info->skip_index_mark_size_in_bytes : index_granularity_info->mark_size_in_bytes; size_t expected_file_size = mark_size * marks_count; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index ff12cd1f277..e03b75f152f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -158,10 +158,10 @@ void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type */ if (!data_file_exists) return; - + std::cerr << "(addStreams) part: " << path << '\n'; std::cerr << "(addStreams) marks count: " << data_part->getMarksCount() << "\n"; - + streams.emplace(stream_name, std::make_unique( path + stream_name, DATA_FILE_EXTENSION, data_part->getMarksCount(), all_mark_ranges, settings, mark_cache, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h index 83ef594fce9..f9941d42bca 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -36,7 +36,7 @@ private: FileStreams streams; /// Columns that are read. - + void addStreams(const String & name, const IDataType & type, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp index 909076c751f..3bb54169790 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp @@ -30,7 +30,7 @@ MergeTreeSelectBlockInputStream::MergeTreeSelectBlockInputStream( bool quiet) : MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, use_uncompressed_cache_, virt_column_names_}, required_columns{required_columns_}, data_part{owned_data_part_}, diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index b4e4bea8128..75fc32bf3f2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -63,7 +63,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( .save_marks_in_cache = false }; - reader = data_part->getReader(columns_for_reader, + reader = data_part->getReader(columns_for_reader, MarkRanges{MarkRange(0, data_part->getMarksCount())}, /* uncompressed_cache = */ nullptr, mark_cache.get(), reader_settings); } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 110b6134c14..bdb2fef5e5a 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -26,7 +26,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( bool blocks_are_granules_size) : IMergedBlockOutputStream(data_part) , columns_list(columns_list_) -{ +{ MergeTreeWriterSettings writer_settings(data_part->storage.global_context.getSettings(), data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); @@ -168,8 +168,6 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); new_part->index_granularity = writer->getIndexGranularity(); new_part->columns_sizes = columns_sizes; - std::cerr << "(writeSuffixAndFinalizePart) part: " << new_part->getFullPath() << "\n"; - std::cerr << "(writeSuffixAndFinalizePart) marks_count: " << new_part->index_granularity.getMarksCount() << "\n"; } void MergedBlockOutputStream::init() diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 6f834e4ac83..518321dca9c 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -44,7 +44,7 @@ void MergedColumnOnlyOutputStream::write(const Block & block) if (!rows) return; - std::cerr << "(MergedColumnOnlyOutputStream::write) writing rows: " << rows << "\n"; + std::cerr << "(MergedColumnOnlyOutputStream::write) writing rows: " << rows << "\n"; writer->write(block); writer->calculateAndSerializeSkipIndices(skip_indexes_block, rows); diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index 0474dcb9b03..2303ec38efa 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -201,7 +201,7 @@ MergeTreeData::DataPart::Checksums checkDataPart( ReadBufferFromFile buf(path + "columns.txt"); columns.readText(buf); assertEOF(buf); - } + } /// Checksums from file checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums. MergeTreeData::DataPart::Checksums checksums_txt; diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index b465740bfde..13c775121b7 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -48,7 +48,7 @@ public: void writeText(WriteBuffer & out) const; void readText(ReadBuffer & in); - /// Columns that we need to read except ones needed for expressions. + /// Extra columns that we need to read except ones needed for expressions. Names additional_columns; }; From ba2a630a13ad616afe2bb6c53dd12e80889c141b Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 19 Dec 2019 16:10:57 +0300 Subject: [PATCH 0083/2007] merging with master --- .../src/Interpreters/MutationsInterpreter.cpp | 2 +- dbms/src/Interpreters/MutationsInterpreter.h | 11 +---- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 10 ++--- .../Storages/MergeTree/IMergeTreeDataPart.h | 8 ++-- .../Storages/MergeTree/IMergeTreeReader.cpp | 4 +- .../src/Storages/MergeTree/IMergeTreeReader.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 16 ++++---- dbms/src/Storages/MergeTree/MergeTreeData.h | 16 ++++---- .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 4 +- .../MergeTree/MergeTreeDataPartCompact.h | 6 ++- .../MergeTree/MergeTreeDataPartWide.cpp | 14 ++++++- .../MergeTree/MergeTreeDataPartWide.h | 6 ++- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 6 +-- .../MergeTree/MergeTreeDataWriter.cpp | 2 +- .../MergeTree/MergeTreeIndexGranularity.h | 2 +- .../Storages/MergeTree/MergeTreePartsMover.h | 4 +- .../Storages/MergeTree/MergeTreeReadPool.cpp | 2 +- .../MergeTree/MergeTreeReaderCompact.cpp | 30 ++++++-------- .../MergeTree/MergeTreeReaderCompact.h | 2 +- .../MergeTree/MergeTreeReaderWide.cpp | 40 +------------------ .../Storages/MergeTree/MergeTreeReaderWide.h | 2 +- .../MergeTreeReverseSelectProcessor.cpp | 21 +--------- .../MergeTree/MergeTreeSelectProcessor.cpp | 11 +---- .../Storages/MergeTree/MergeTreeSettings.h | 4 +- dbms/src/Storages/StorageMergeTree.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 2 +- 27 files changed, 86 insertions(+), 145 deletions(-) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 7e7ab2f0030..ab1b147b472 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -164,7 +164,7 @@ bool isStorageTouchedByMutations( MutationsInterpreter::MutationsInterpreter( StoragePtr storage_, - std::vector commands_, + MutationCommands commands_, const Context & context_, bool can_execute_) : storage(std::move(storage_)) diff --git a/dbms/src/Interpreters/MutationsInterpreter.h b/dbms/src/Interpreters/MutationsInterpreter.h index 2dde67d3596..4b6686d77ae 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.h +++ b/dbms/src/Interpreters/MutationsInterpreter.h @@ -21,18 +21,9 @@ bool isStorageTouchedByMutations(StoragePtr storage, const std::vector commands_, const Context & context_, bool can_execute_); ->>>>>>> upstream/master + MutationsInterpreter(StoragePtr storage_, MutationCommands commands_, const Context & context_, bool can_execute_); void validate(TableStructureReadLockHolder & table_lock_holder); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 69e7798dfb5..bce4e51dd49 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -138,7 +138,7 @@ void IMergeTreeDataPart::MinMaxIndex::merge(const MinMaxIndex & other) IMergeTreeDataPart::IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_) : storage(storage_) , name(name_) @@ -152,7 +152,7 @@ IMergeTreeDataPart::IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_) : storage(storage_) , name(name_) @@ -567,7 +567,7 @@ void IMergeTreeDataPart::loadColumns(bool require) Poco::File poco_file_path{path}; if (!poco_file_path.exists()) { - if (require) + if (require || isCompactPart(shared_from_this())) throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. @@ -836,10 +836,10 @@ void IMergeTreeDataPart::makeCloneInDetached(const String & prefix) const localBackup(src, dst, 0); } -void IMergeTreeDataPart::makeCloneOnDiskDetached(const DiskSpace::ReservationPtr & reservation) const +void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservation) const { assertOnDisk(); - auto & reserved_disk = reservation->getDisk(); + auto reserved_disk = reservation->getDisk(); if (reserved_disk->getName() == disk->getName()) throw Exception("Can not clone data part " + name + " to same disk " + disk->getName(), ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 89b8996c65b..e03fa09b63d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -82,6 +82,8 @@ public: /// If no checksums are present returns the name of the first physically existing column. virtual String getColumnNameWithMinumumCompressedSize() const { return columns.front().name; } + virtual String getFileNameForColumn(const NameAndTypePair & column) const = 0; + virtual NameToNameMap createRenameMapForAlter( AlterAnalysisResult & /* analysis_result */, const NamesAndTypesList & /* old_columns */) const @@ -114,13 +116,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskSpace::DiskPtr & disk = {}, + const DiskPtr & disk = {}, const std::optional & relative_path = {}); IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, - const DiskSpace::DiskPtr & disk = {}, + const DiskPtr & disk = {}, const std::optional & relative_path = {}); void assertOnDisk() const; @@ -160,7 +162,7 @@ public: MergeTreePartInfo info; MergeTreeIndexGranularityInfo index_granularity_info; - DiskSpace::DiskPtr disk; + DiskPtr disk; mutable String relative_path; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp index a4c000411e6..51ac7486eb8 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -66,7 +66,7 @@ static bool arrayHasNoElementsRead(const IColumn & column) } -void MergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows) +void IMergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_evaluate_missing_defaults, size_t num_rows) { try { @@ -149,7 +149,7 @@ void MergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_ev } } -void MergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns & res_columns) +void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns & res_columns) { try { diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 8c51aa08741..1a750940b5e 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -29,7 +29,7 @@ public: /// Return the number of rows has been read or zero if there is no columns to read. /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark - virtual size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) = 0; + virtual size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) = 0; virtual bool canReadIncompleteGranules() const = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index ab2a0e0a5d6..f8e1147cadd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1521,7 +1521,7 @@ MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_uncompressed, s MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, const String & relative_path) const + const DiskPtr & disk, const String & relative_path) const { if (type == MergeTreeDataPartType::COMPACT) return std::make_shared(*this, name, part_info, disk, relative_path); @@ -1534,7 +1534,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const String & name, const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, + const DiskPtr & disk, const NamesAndTypesList & columns_list, size_t bytes_uncompressed, size_t rows_count, @@ -1559,14 +1559,14 @@ static MergeTreeDataPartType getPartTypeFromMarkExtension(const String & mrk_ext } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( - const String & name, const DiskSpace::DiskPtr & disk, const String & relative_path) const + const String & name, const DiskPtr & disk, const String & relative_path) const { return createPart(name, MergeTreePartInfo::fromPartName(name, format_version), disk, relative_path); } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const String & name, const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, const String & relative_path) const + const DiskPtr & disk, const String & relative_path) const { auto type = MergeTreeDataPartType::UNKNOWN; auto full_path = getFullPathOnDisk(disk) + relative_path + "/"; @@ -3169,7 +3169,7 @@ ReservationPtr MergeTreeData::tryReserveSpace(UInt64 expected_size, SpacePtr spa } ReservationPtr MergeTreeData::reserveSpacePreferringTTLRules(UInt64 expected_size, - const MergeTreeDataPart::TTLInfos & ttl_infos, + const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const { expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); @@ -3180,7 +3180,7 @@ ReservationPtr MergeTreeData::reserveSpacePreferringTTLRules(UInt64 expected_siz } ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules(UInt64 expected_size, - const MergeTreeDataPart::TTLInfos & ttl_infos, + const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const { expected_size = std::max(RESERVATION_MIN_ESTIMATION_SIZE, expected_size); @@ -3225,7 +3225,7 @@ SpacePtr MergeTreeData::TTLEntry::getDestination(const StoragePolicyPtr & policy return {}; } -bool MergeTreeData::TTLEntry::isPartInDestination(const StoragePolicyPtr & policy, const MergeTreeDataPart & part) const +bool MergeTreeData::TTLEntry::isPartInDestination(const StoragePolicyPtr & policy, const IMergeTreeDataPart & part) const { if (destination_type == PartDestinationType::VOLUME) { @@ -3239,7 +3239,7 @@ bool MergeTreeData::TTLEntry::isPartInDestination(const StoragePolicyPtr & polic } const MergeTreeData::TTLEntry * MergeTreeData::selectTTLEntryForTTLInfos( - const MergeTreeDataPart::TTLInfos & ttl_infos, + const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const { const MergeTreeData::TTLEntry * result = nullptr; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 9021e3102a7..f413092c039 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -185,21 +185,21 @@ public: /// After this methods setColumns must be called /// FIXME make this inside this function MutableDataPartPtr createPart(const String & name, - const MergeTreePartInfo & part_info,const DiskSpace::DiskPtr & disk, + const MergeTreePartInfo & part_info,const DiskPtr & disk, const NamesAndTypesList & columns, size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, const String & relative_path) const; + const DiskPtr & disk, const String & relative_path) const; /// After this methods loadColumnsChecksumsIndexes must be called /// FIXME make this inside this function MutableDataPartPtr createPart(const String & name, - const DiskSpace::DiskPtr & disk, const String & relative_path) const; + const DiskPtr & disk, const String & relative_path) const; MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info, - const DiskSpace::DiskPtr & disk, const String & relative_path) const; + const DiskPtr & disk, const String & relative_path) const; /// Auxiliary object to add a set of parts into the working set in two steps: /// * First, as PreCommitted parts (the parts are ready, but not yet in the active set). @@ -710,10 +710,10 @@ public: /// Reserves space at least 1MB preferring best destination according to `ttl_infos`. ReservationPtr reserveSpacePreferringTTLRules(UInt64 expected_size, - const MergeTreeDataPart::TTLInfos & ttl_infos, + const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const; ReservationPtr tryReserveSpacePreferringTTLRules(UInt64 expected_size, - const MergeTreeDataPart::TTLInfos & ttl_infos, + const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const; /// Choose disk with max available free space /// Reserves 0 bytes @@ -769,10 +769,10 @@ public: SpacePtr getDestination(const StoragePolicyPtr & policy) const; /// Checks if given part already belongs destination disk or volume for this rule. - bool isPartInDestination(const StoragePolicyPtr & policy, const MergeTreeDataPart & part) const; + bool isPartInDestination(const StoragePolicyPtr & policy, const IMergeTreeDataPart & part) const; }; - const TTLEntry * selectTTLEntryForTTLInfos(const MergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const; + const TTLEntry * selectTTLEntryForTTLInfos(const IMergeTreeDataPart::TTLInfos & ttl_infos, time_t time_of_move) const; using TTLEntriesByName = std::unordered_map; TTLEntriesByName column_ttl_entries_by_name; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index ead4f8cc56e..5ef367ba2fb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -963,7 +963,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (isCompactPart(source_part)) commands_for_part.additional_columns = source_part->columns.getNames(); - MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading); + MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading, true); if (!isStorageTouchedByMutations(storage_from_source_part, commands_for_part, context_for_reading)) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 85a78352f25..0008bc23e7a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -55,7 +55,7 @@ namespace ErrorCodes MergeTreeDataPartCompact::MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_) : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) { @@ -65,7 +65,7 @@ MergeTreeDataPartCompact::MergeTreeDataPartCompact( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_) : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 636391647cb..c1637b94d9b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -41,13 +41,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_ = {}); MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_ = {}); MergeTreeReaderPtr getReader( @@ -82,6 +82,8 @@ public: bool hasColumnFiles(const String & column_name, const IDataType & type) const override; + String getFileNameForColumn(const NameAndTypePair & /* column */) const override { return DATA_FILE_NAME; } + NameToNameMap createRenameMapForAlter( AlterAnalysisResult & analysis_result, const NamesAndTypesList & old_columns) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index c644aea4716..bc5e8924242 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -52,7 +52,7 @@ namespace ErrorCodes MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_) : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) { @@ -62,7 +62,7 @@ MergeTreeDataPartWide::MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskSpace::DiskPtr & disk_, + const DiskPtr & disk_, const std::optional & relative_path_) : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) { @@ -428,4 +428,14 @@ NameToNameMap MergeTreeDataPartWide::createRenameMapForAlter( return rename_map; } +String MergeTreeDataPartWide::getFileNameForColumn(const NameAndTypePair & column) const +{ + String filename; + column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { + if (filename.empty()) + filename = IDataType::getFileNameForStream(column.name, substream_path); + }); + return filename; +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 25a42fbfad1..8eb572a6d36 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -37,13 +37,13 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskSpace::DiskPtr & disk, + const DiskPtr & disk, const std::optional & relative_path = {}); MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, - const DiskSpace::DiskPtr & disk, + const DiskPtr & disk, const std::optional & relative_path = {}); MergeTreeReaderPtr getReader( @@ -68,6 +68,8 @@ public: void accumulateColumnSizes(ColumnToSize & column_to_size) const override; + String getFileNameForColumn(const NameAndTypePair & column) const override; + /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 3c4fe7346e1..c23b5b6efc6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -783,7 +783,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( auto source = std::make_shared( i, pool, min_marks_for_concurrent_read, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, data, use_uncompressed_cache, - query_info.prewhere_info, reader_settings, virt_columns)); + query_info.prewhere_info, reader_settings, virt_columns); if (i == 0) { @@ -1047,7 +1047,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, ranges_to_get_from_part, use_uncompressed_cache, query_info.prewhere_info, true, reader_settings, - virt_columns, part.part_index_in_query); + virt_columns, part.part_index_in_query)); } else { @@ -1055,7 +1055,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, ranges_to_get_from_part, use_uncompressed_cache, query_info.prewhere_info, true, reader_settings, - virt_columns, part.part_index_in_query); + virt_columns, part.part_index_in_query)); pipes.back().addSimpleTransform(std::make_shared(pipes.back().getHeader())); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index b5fd526f0f7..2688879b1c8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -225,7 +225,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa /// Size of part would not be greater than block.bytes() + epsilon size_t expected_size = block.bytes(); - DB::MergeTreeDataPart::TTLInfos move_ttl_infos; + DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; for (const auto & ttl_entry : data.move_ttl_entries) updateTTL(ttl_entry, move_ttl_infos, move_ttl_infos.moves_ttl[ttl_entry.result_column], block, false); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h index 5e25ab0e2b6..2a9cbcf654c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -5,7 +5,7 @@ namespace DB { -/// Class contains information about index granularity in rows of MergeTreeDataPart +/// Class contains information about index granularity in rows of IMergeTreeDataPart /// Inside it contains vector of partial sums of rows after mark: /// |-----|---|----|----| /// | 5 | 8 | 12 | 16 | diff --git a/dbms/src/Storages/MergeTree/MergeTreePartsMover.h b/dbms/src/Storages/MergeTree/MergeTreePartsMover.h index 78a5a9f9bee..6bb436ab4c3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartsMover.h +++ b/dbms/src/Storages/MergeTree/MergeTreePartsMover.h @@ -15,10 +15,10 @@ namespace DB /// it have to be moved. struct MergeTreeMoveEntry { - std::shared_ptr part; + std::shared_ptr part; ReservationPtr reserved_space; - MergeTreeMoveEntry(const std::shared_ptr & part_, ReservationPtr reservation_) + MergeTreeMoveEntry(const std::shared_ptr & part_, ReservationPtr reservation_) : part(part_), reserved_space(std::move(reservation_)) { } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index b0485c57eda..86ae4e2f83a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -33,7 +33,7 @@ MergeTreeReadPool::MergeTreeReadPool( for (auto & part_ranges : parts_ranges) std::reverse(std::begin(part_ranges.ranges), std::end(part_ranges.ranges)); - /// parts don't contain duplicate MergeTreeDataPart's. + /// parts don't contain duplicate IMergeTreeDataPart's. const auto per_part_sum_marks = fillPerPartInfo(parts_, check_columns_); fillPerThreadInfo(threads_, sum_marks_, per_part_sum_marks, parts_, min_marks_for_concurrent_read_); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 389a21e0ebd..dbe5736597d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -52,13 +52,8 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr column_positions.push_back(data_part->getColumnPosition(column.name)); } -size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) +size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) { - UNUSED(from_mark); - UNUSED(continue_reading); - UNUSED(max_rows_to_read); - UNUSED(res); - /// FIXME compute correct granularity std::cerr << "(MergeTreeReaderCompact::readRows) max_rows_to_read: " << max_rows_to_read << "\n"; @@ -75,23 +70,24 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, { size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); - auto it = columns.begin(); - for (size_t i = 0; i < num_columns; ++i, ++it) + auto name_and_type = columns.begin(); + for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) { - if (!column_positions[i]) + if (!column_positions[pos]) continue; - - bool append = res.has(it->name); + + auto & [name, type] = *name_and_type; + bool append = res_columns[pos] != nullptr; if (!append) - res.insert(ColumnWithTypeAndName(it->type->createColumn(), it->type, it->name)); + res_columns[pos] = name_and_type->type->createColumn(); /// To keep offsets shared. TODO Very dangerous. Get rid of this. - MutableColumnPtr column = res.getByName(it->name).column->assumeMutable(); + MutableColumnPtr column = res_columns[pos]->assumeMutable(); try { size_t column_size_before_reading = column->size(); - readData(it->name, *it->type, *column, from_mark, *column_positions[i], rows_to_read); + readData(name, *type, *column, from_mark, *column_positions[pos], rows_to_read); size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column < rows_to_read) @@ -107,14 +103,14 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, catch (Exception & e) { /// Better diagnostics. - e.addMessage("(while reading column " + it->name + ")"); + e.addMessage("(while reading column " + name + ")"); throw; } if (column->size()) - res.getByName(it->name).column = std::move(column); + res_columns[pos] = std::move(column); else - res.erase(it->name); + res_columns[pos] = nullptr; } ++from_mark; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index bd47d13e5ea..719516a39cd 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -24,7 +24,7 @@ public: /// Return the number of rows has been read or zero if there is no columns to read. /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark - size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) override; + size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) override; bool canReadIncompleteGranules() const override { return false; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 949a169fc70..567d9442743 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -25,7 +25,6 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; } -<<<<<<< HEAD:dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, @@ -34,36 +33,6 @@ MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data : IMergeTreeReader(data_part_, columns_ , uncompressed_cache_, mark_cache_, mark_ranges_ , settings_, avg_value_size_hints_) -======= - -MergeTreeReader::~MergeTreeReader() = default; - - -MergeTreeReader::MergeTreeReader( - String path_, - MergeTreeData::DataPartPtr data_part_, - NamesAndTypesList columns_, - UncompressedCache * uncompressed_cache_, - MarkCache * mark_cache_, - bool save_marks_in_cache_, - const MergeTreeData & storage_, - MarkRanges all_mark_ranges_, - size_t aio_threshold_, - size_t max_read_buffer_size_, - ValueSizeMap avg_value_size_hints_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_, - clockid_t clock_type_) - : data_part(std::move(data_part_)) - , avg_value_size_hints(std::move(avg_value_size_hints_)) - , path(std::move(path_)), columns(std::move(columns_)) - , uncompressed_cache(uncompressed_cache_) - , mark_cache(mark_cache_) - , save_marks_in_cache(save_marks_in_cache_) - , storage(storage_) - , all_mark_ranges(std::move(all_mark_ranges_)) - , aio_threshold(aio_threshold_) - , max_read_buffer_size(max_read_buffer_size_) ->>>>>>> upstream/master:dbms/src/Storages/MergeTree/MergeTreeReader.cpp { try { @@ -77,16 +46,9 @@ MergeTreeReader::MergeTreeReader( } } -size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) -{ - std::cerr << "(MergeTreeReaderWide::readRows) from_mark: " << from_mark << "\n"; - std::cerr << "(MergeTreeReaderWide::readRows) continue_reading: " << continue_reading << "\n"; -<<<<<<< HEAD:dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp -======= -size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) +size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) { ->>>>>>> upstream/master:dbms/src/Storages/MergeTree/MergeTreeReader.cpp size_t read_rows = 0; try { diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h index f9941d42bca..c9ce5dddf23 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -26,7 +26,7 @@ public: /// Return the number of rows has been read or zero if there is no columns to read. /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark - size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Block & res) override; + size_t readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) override; bool canReadIncompleteGranules() const override { return true; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 7602c60dc16..02636a34690 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -47,19 +47,12 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( size_t part_index_in_query_, bool quiet) : -<<<<<<< HEAD:dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp - MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, reader_settings_, - use_uncompressed_cache_, virt_column_names_}, - required_columns{required_columns_}, -======= MergeTreeBaseSelectProcessor{ replaceTypes(storage_.getSampleBlockForColumns(required_columns_), owned_data_part_), storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, min_bytes_to_use_direct_io_, - max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names_}, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, + reader_settings_, use_uncompressed_cache_, virt_column_names_}, required_columns{std::move(required_columns_)}, ->>>>>>> upstream/master:dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp data_part{owned_data_part_}, part_columns_lock(data_part->columns_lock), all_mark_ranges(std::move(mark_ranges_)), @@ -103,17 +96,7 @@ MergeTreeReverseSelectProcessor::MergeTreeReverseSelectProcessor( owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings); } -<<<<<<< HEAD:dbms/src/Storages/MergeTree/MergeTreeReverseSelectBlockInputStream.cpp -Block MergeTreeReverseSelectBlockInputStream::getHeader() const -{ - return header; -} - - -bool MergeTreeReverseSelectBlockInputStream::getNewTask() -======= bool MergeTreeReverseSelectProcessor::getNewTask() ->>>>>>> upstream/master:dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp try { if ((chunks.empty() && all_mark_ranges.empty()) || total_marks_count == 0) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 6931638961a..9cdde2a36a1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -47,19 +47,12 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( size_t part_index_in_query_, bool quiet) : -<<<<<<< HEAD:dbms/src/Storages/MergeTree/MergeTreeSelectBlockInputStream.cpp - MergeTreeBaseSelectBlockInputStream{storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, - reader_settings_, use_uncompressed_cache_, virt_column_names_}, - required_columns{required_columns_}, -======= MergeTreeBaseSelectProcessor{ replaceTypes(storage_.getSampleBlockForColumns(required_columns_), owned_data_part_), storage_, prewhere_info_, max_block_size_rows_, - preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, min_bytes_to_use_direct_io_, - max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names_}, + preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, + reader_settings_, use_uncompressed_cache_, virt_column_names_}, required_columns{std::move(required_columns_)}, ->>>>>>> upstream/master:dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp data_part{owned_data_part_}, part_columns_lock(data_part->columns_lock), all_mark_ranges(std::move(mark_ranges_)), diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 29c56b753f3..a3dbd81ff4d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -30,8 +30,8 @@ struct MergeTreeSettings : public SettingsCollection M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.", 0) \ \ /** Data storing format settigns. */ \ - M(SettingUInt64, min_bytes_for_wide_part, 0, "") \ - M(SettingUInt64, min_rows_for_wide_part, 0, "") \ + M(SettingUInt64, min_bytes_for_wide_part, 0, "", 0) \ + M(SettingUInt64, min_rows_for_wide_part, 0, "", 0) \ \ /** Merge settings. */ \ M(SettingUInt64, merge_max_block_size, DEFAULT_MERGE_BLOCK_SIZE, "How many rows in blocks should be formed for merge operations.", 0) \ diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 41c9335de1e..780e28fb79a 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -353,7 +353,7 @@ public: reserved_space = storage.tryReserveSpace(total_size, future_part_.parts[0]->disk); else { - MergeTreeDataPart::TTLInfos ttl_infos; + IMergeTreeDataPart::TTLInfos ttl_infos; for (auto & part_ptr : future_part_.parts) ttl_infos.update(part_ptr->ttl_infos); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 55b83c308a5..1ae192d921b 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1006,7 +1006,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) size_t estimated_space_for_merge = MergeTreeDataMergerMutator::estimateNeededDiskSpace(parts); /// Can throw an exception while reserving space. - MergeTreeDataPart::TTLInfos ttl_infos; + IMergeTreeDataPart::TTLInfos ttl_infos; for (auto & part_ptr : parts) { ttl_infos.update(part_ptr->ttl_infos); From 206cb1afa9bbf93666448ce1b638697a898662ff Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 19 Dec 2019 17:05:26 +0300 Subject: [PATCH 0084/2007] fix broken by refactoring functionality with wide parts --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 17 ++-------- .../MergeTree/IMergeTreeDataPartWriter.h | 8 ++--- .../MergeTreeBaseSelectProcessor.cpp | 2 -- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 14 +++----- .../MergeTree/MergeTreeDataMergerMutator.cpp | 5 ++- .../MergeTree/MergeTreeDataPartCompact.cpp | 4 --- .../MergeTree/MergeTreeDataPartWide.cpp | 5 +-- .../MergeTree/MergeTreeDataPartWide.h | 4 +-- .../MergeTreeDataPartWriterCompact.cpp | 6 ---- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 19 +++-------- .../MergeTree/MergeTreeDataPartWriterWide.h | 2 -- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 6 ---- .../MergeTree/MergeTreeDataWriter.cpp | 1 - .../Storages/MergeTree/MergeTreeIOSettings.h | 1 + .../MergeTree/MergeTreeIndexGranularityInfo.h | 2 +- .../MergeTree/MergeTreeIndexReader.cpp | 1 - .../MergeTree/MergeTreeRangeReader.cpp | 3 -- .../MergeTree/MergeTreeReaderCompact.cpp | 33 ++----------------- .../MergeTree/MergeTreeReaderStream.cpp | 4 --- .../MergeTree/MergeTreeReaderWide.cpp | 7 ---- .../MergeTreeSequentialBlockInputStream.cpp | 6 ---- ...rgeTreeThreadSelectBlockInputProcessor.cpp | 4 --- .../MergedColumnOnlyOutputStream.cpp | 13 ++------ ...est_aux_funcs_for_adaptive_granularity.cpp | 2 +- .../convert-month-partitioned-parts/main.cpp | 4 +-- 25 files changed, 31 insertions(+), 142 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 1605beadeaf..2cad6ee1930 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -90,7 +90,7 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter( IMergeTreeDataPartWriter::~IMergeTreeDataPartWriter() = default; -void fillIndexGranularityImpl( +static void fillIndexGranularityImpl( const Block & block, size_t index_granularity_bytes, size_t fixed_index_granularity_rows, @@ -98,7 +98,7 @@ void fillIndexGranularityImpl( size_t index_offset, MergeTreeIndexGranularity & index_granularity, bool can_use_adaptive_index_granularity, - bool need_finish_last_granule) + bool need_finish_last_granule = false) { /// FIXME correct index granularity for compact size_t rows_in_block = block.rows(); @@ -151,9 +151,6 @@ void fillIndexGranularityImpl( else index_granularity.appendMark(index_granularity_for_block); } - - for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) - std::cerr << "marks: " << index_granularity.getMarkRows(i) << "\n"; } void IMergeTreeDataPartWriter::fillIndexGranularity(const Block & block) @@ -226,7 +223,6 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p /// Write index. The index contains Primary Key value for each `index_granularity` row. - std::cerr << "writing index...\n"; for (size_t i = index_offset; i < rows;) { if (storage.hasPrimaryKey()) @@ -239,10 +235,6 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p } } - std::cerr << "(index) i: " << i << "\n"; - std::cerr << "(index) current_mark: " << current_mark << "\n"; - std::cerr << "(index) rows in mark: " << index_granularity.getMarkRows(current_mark) << "\n"; - i += index_granularity.getMarkRows(current_mark++); if (current_mark >= index_granularity.getMarksCount()) break; @@ -322,8 +314,6 @@ void IMergeTreeDataPartWriter::calculateAndSerializeSkipIndices( void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums) { - std::cerr << "finishPrimaryIndexSerialization called...\n"; - bool write_final_mark = (with_final_mark && data_written); if (write_final_mark && compute_granularity) index_granularity.appendMark(0); @@ -341,9 +331,6 @@ void IMergeTreeDataPartWriter::finishPrimaryIndexSerialization(MergeTreeData::Da last_index_row.clear(); } - - std::cerr << "(finishPrimaryIndexSerialization) marks_count: " << index_granularity.getMarksCount() << "\n"; - index_stream->next(); checksums.files["primary.idx"].file_size = index_stream->count(); checksums.files["primary.idx"].file_hash = index_stream->getHash(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 645a44f8238..96c5b7807e1 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -94,12 +94,14 @@ public: const MergeTreeData::ColumnSizeByName & getColumnsSizes() const { return columns_sizes; } - void setOffsetColumns(WrittenOffsetColumns * written_offset_columns_, bool skip_offsets_) + void setWrittenOffsetColumns(WrittenOffsetColumns * written_offset_columns_) { written_offset_columns = written_offset_columns_; - skip_offsets = skip_offsets_; } + using SkipIndices = std::vector; + const SkipIndices & getSkipIndices() { return skip_indices; } + void initSkipIndices(); void initPrimaryIndex(); @@ -160,8 +162,6 @@ protected: /// To correctly write Nested elements column-by-column. WrittenOffsetColumns * written_offset_columns = nullptr; - bool skip_offsets = false; - }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 47d9d89a2c9..b7ee14d141c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -155,8 +155,6 @@ Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() UInt64 recommended_rows = estimateNumRows(*task, task->range_reader); UInt64 rows_to_read = std::max(UInt64(1), std::min(current_max_block_size_rows, recommended_rows)); - // std::cerr << "(readFromPartImpl) rows_to_read: " << rows_to_read << "\n"; - auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); /// All rows were filtered. Repeat. diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index f8e1147cadd..79acbd5a683 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -296,16 +296,13 @@ void MergeTreeData::setProperties( Names new_primary_key_columns; Names new_sorting_key_columns; - NameSet sorting_key_columns_set; + NameSet primary_key_columns_set; for (size_t i = 0; i < sorting_key_size; ++i) { String sorting_key_column = new_sorting_key_expr_list->children[i]->getColumnName(); new_sorting_key_columns.push_back(sorting_key_column); - if (!sorting_key_columns_set.emplace(sorting_key_column).second) - throw Exception("Sorting key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); - if (i < primary_key_size) { String pk_column = new_primary_key_expr_list->children[i]->getColumnName(); @@ -314,6 +311,9 @@ void MergeTreeData::setProperties( + toString(i) + " its column is " + pk_column + ", not " + sorting_key_column, ErrorCodes::BAD_ARGUMENTS); + if (!primary_key_columns_set.emplace(pk_column).second) + throw Exception("Primary key contains duplicate columns", ErrorCodes::BAD_ARGUMENTS); + new_primary_key_columns.push_back(pk_column); } } @@ -840,8 +840,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) const auto & part_name = part_names_with_disks[i].first; const auto part_disk_ptr = part_names_with_disks[i].second; - std::cerr << "(loadDataParts) loading part: " << part_name << "\n"; - MergeTreePartInfo part_info; if (!MergeTreePartInfo::tryParsePartName(part_name, &part_info, format_version)) return; @@ -1678,7 +1676,6 @@ void MergeTreeData::alterDataPart( /// Apply the expression and write the result to temporary files. if (res.expression) { - std::cerr << "(alterDataPart) expression: " << res.expression->dumpActions() << "\n"; BlockInputStreamPtr part_in = std::make_shared( *this, part, res.expression->getRequiredColumns(), false, /* take_column_types_from_storage = */ false); @@ -1687,8 +1684,6 @@ void MergeTreeData::alterDataPart( static_cast(part->bytes_on_disk) / this->getTotalActiveSizeInBytes()); ExpressionBlockInputStream in(part_in, res.expression); - std::cerr << "im.header: " << in.getHeader().dumpStructure() << "\n"; - /** Don't write offsets for arrays, because ALTER never change them * (MODIFY COLUMN could only change types of elements but never modify array sizes). * Also note that they does not participate in 'rename_map'. @@ -1696,7 +1691,6 @@ void MergeTreeData::alterDataPart( * temporary column name ('converting_column_name') created in 'createConvertExpression' method * will have old name of shared offsets for arrays. */ - IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( part, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 5ef367ba2fb..ba025f3fa82 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -668,7 +668,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor } MergeStageProgress horizontal_stage_progress( - column_sizes ? 1.0 : column_sizes->keyColumnsWeight()); + column_sizes ? column_sizes->keyColumnsWeight() : 1.0); for (const auto & part : parts) { @@ -963,8 +963,6 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (isCompactPart(source_part)) commands_for_part.additional_columns = source_part->columns.getNames(); - MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading, true); - if (!isStorageTouchedByMutations(storage_from_source_part, commands_for_part, context_for_reading)) { LOG_TRACE(log, "Part " << source_part->name << " doesn't change up to mutation version " << future_part.part_info.mutation); @@ -973,6 +971,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor else LOG_TRACE(log, "Mutating part " << source_part->name << " to mutation version " << future_part.part_info.mutation); + MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading, true); auto in = mutations_interpreter.execute(table_lock_holder); const auto & updated_header = mutations_interpreter.getUpdatedHeader(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 0008bc23e7a..0977d18b6f8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -185,10 +185,6 @@ void MergeTreeDataPartCompact::loadIndexGranularity() buffer.seek(columns.size() * sizeof(MarkInCompressedFile), SEEK_CUR); } - std::cerr << "(loadIndexGranularity) marks: " << index_granularity.getMarksCount() << "\n"; - std::cerr << "(loadIndexGranularity) mark size: " << index_granularity_info.mark_size_in_bytes << "\n"; - std::cerr << "(loadIndexGranularity) marks file size: " << marks_file_size << "\n"; - if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index bc5e8924242..d8c705a874c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -181,7 +181,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// We can use any column, it doesn't matter - std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + escapeForFileName(columns.front().name)); + std::string marks_file_path = index_granularity_info.getMarksFilePath(full_path + getFileNameForColumn(columns.front())); if (!Poco::File(marks_file_path).exists()) throw Exception("Marks file '" + marks_file_path + "' doesn't exist", ErrorCodes::NO_FILE_IN_DATA_PART); @@ -431,7 +431,8 @@ NameToNameMap MergeTreeDataPartWide::createRenameMapForAlter( String MergeTreeDataPartWide::getFileNameForColumn(const NameAndTypePair & column) const { String filename; - column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) { + column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { if (filename.empty()) filename = IDataType::getFileNameForStream(column.name, substream_path); }); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 8eb572a6d36..ae747382dba 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -33,7 +33,7 @@ public: using Checksums = MergeTreeDataPartChecksums; using Checksum = MergeTreeDataPartChecksums::Checksum; - MergeTreeDataPartWide( + MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, @@ -86,7 +86,7 @@ public: ~MergeTreeDataPartWide() override; - bool hasColumnFiles(const String & column, const IDataType & type) const override; + bool hasColumnFiles(const String & column, const IDataType & type) const override; protected: void checkConsistency(bool require_part_metadata) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index fa9d1e279f0..d46d556d7b9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -40,8 +40,6 @@ void MergeTreeDataPartWriterCompact::write( const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) { - std::cerr << "(MergeTreeDataPartWriterCompact::write) block111: " << block.dumpStructure() << "\n"; - /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, /// but not in case of vertical merge) @@ -117,10 +115,6 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) { - std::cerr << "(writeColumnSingleGranule) writing column: " << column.name << "\n"; - std::cerr << "(writeColumnSingleGranule) from_row: " << from_row << "\n"; - std::cerr << "(writeColumnSingleGranule) number_of_rows: " << number_of_rows << "\n"; - size_t old_uncompressed_size = stream->compressed.count(); writeIntBinary(stream->plain_hashing.count(), stream->marks); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index b3d06bb946f..92c21e3166f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -20,7 +20,6 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( : IMergeTreeDataPartWriter(part_path_, storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_, false) - , can_use_adaptive_granularity(storage_.canUseAdaptiveGranularity()) { const auto & columns = storage.getColumns(); for (const auto & it : columns_list) @@ -35,11 +34,10 @@ void MergeTreeDataPartWriterWide::addStreams( { IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path) { - if (skip_offsets && !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) + if (settings.skip_offsets && !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes) return; String stream_name = IDataType::getFileNameForStream(name, substream_path); - /// Shared offsets for Nested type. if (column_streams.count(stream_name)) return; @@ -54,8 +52,6 @@ void MergeTreeDataPartWriterWide::addStreams( settings.aio_threshold); }; - std::cerr << "(addStreams) name: " << name << "\n"; - IDataType::SubstreamPath stream_path; type.enumerateStreams(callback, stream_path); } @@ -67,7 +63,7 @@ IDataType::OutputStreamGetter MergeTreeDataPartWriterWide::createStreamGetter( return [&, this] (const IDataType::SubstreamPath & substream_path) -> WriteBuffer * { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && skip_offsets) + if (is_offsets && settings.skip_offsets) return nullptr; String stream_name = IDataType::getFileNameForStream(name, substream_path); @@ -104,9 +100,6 @@ void MergeTreeDataPartWriterWide::write(const Block & block, if (compute_granularity) fillIndexGranularity(block); - std::cerr << "(MergeTreeDataPartWriterWide::write) marks_count: " << index_granularity.getMarksCount() << "\n"; - std::cerr << "(MergeTreeDataPartWriterWide::write) current_mark: " << current_mark << "\n"; - auto offset_columns = written_offset_columns ? *written_offset_columns : WrittenOffsetColumns{}; auto it = columns_list.begin(); @@ -150,7 +143,7 @@ void MergeTreeDataPartWriterWide::writeSingleMark( type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && skip_offsets) + if (is_offsets && settings.skip_offsets) return; String stream_name = IDataType::getFileNameForStream(name, substream_path); @@ -192,7 +185,7 @@ size_t MergeTreeDataPartWriterWide::writeSingleGranule( type.enumerateStreams([&] (const IDataType::SubstreamPath & substream_path) { bool is_offsets = !substream_path.empty() && substream_path.back().type == IDataType::Substream::ArraySizes; - if (is_offsets && skip_offsets) + if (is_offsets && settings.skip_offsets) return; String stream_name = IDataType::getFileNameForStream(name, substream_path); @@ -222,10 +215,6 @@ void MergeTreeDataPartWriterWide::writeColumn( type.serializeBinaryBulkStatePrefix(serialize_settings, it->second); } - std::cerr << "(writeColumn) table: " << storage.getTableName() << "\n"; - std::cerr << "(writeColumn) column: " << name << "\n"; - std::cerr << "(writeColumn) index_offset: " << index_offset << "\n"; - const auto & global_settings = storage.global_context.getSettingsRef(); IDataType::SerializeBinaryBulkSettings serialize_settings; serialize_settings.getter = createStreamGetter(name, offset_columns); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index ad8a91c3ec8..10260fb7a90 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -69,9 +69,7 @@ private: size_t estimated_size); SerializationStates serialization_states; - bool can_use_adaptive_granularity; ColumnStreams column_streams; - }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index c23b5b6efc6..8e355e8f55b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -718,12 +718,6 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( const Settings & settings, const MergeTreeReaderSettings & reader_settings) const { - std::cerr << "marks to read: "; - for (const auto & part : parts) - for (auto range : part.ranges) - std::cerr << "(" << range.begin << ", " << range.end << ") "; - - /// Count marks for each part. std::vector sum_marks_in_parts(parts.size()); size_t sum_marks = 0; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 2688879b1c8..86a10b35ca2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -190,7 +190,6 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(const Block & block MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPartition & block_with_partition) { Block & block = block_with_partition.block; - // std::cerr << "(MergeTreeDataWriter::writeTempPart) block.rows(): " << block.rows() << "\n"; static const String TMP_PREFIX = "tmp_insert_"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h index ff2b1362bca..e967c7248b2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -28,6 +28,7 @@ struct MergeTreeWriterSettings bool blocks_are_granules_size; String filename_suffix = ""; size_t estimated_size = 0; + bool skip_offsets = false; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index a51bed5bf69..d72ef5f7917 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -61,7 +61,7 @@ private: }; constexpr inline auto getNonAdaptiveMrkExtension() { return ".mrk"; } -constexpr inline auto getNonAdaptiveMrkSize() { return sizeof(MarkInCompressedFile) * 2; } +constexpr inline auto getNonAdaptiveMrkSize() { return sizeof(UInt64) * 2; } std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type); size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num); diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 709b370c33d..f3fcf1b6cfe 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -27,7 +27,6 @@ MergeTreeIndexGranulePtr MergeTreeIndexReader::read() { auto granule = index->createIndexGranule(); granule->deserializeBinary(*stream.data_buffer); - std::cerr << "(MergeTreeIndexReader) granule.empty(): " << granule->empty() << "\n"; return granule; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 81209fd31dc..2fe557040cb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -231,9 +231,6 @@ void MergeTreeRangeReader::ReadResult::addGranule(size_t num_rows_) void MergeTreeRangeReader::ReadResult::adjustLastGranule() { - std::cerr << "(adjustLastGranule) num_read_rows: " << num_read_rows << "\n"; - std::cerr << "(adjustLastGranule) total_rows_per_granule: " << total_rows_per_granule << "\n"; - size_t num_rows_to_subtract = total_rows_per_granule - num_read_rows; if (rows_per_granule.empty()) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index dbe5736597d..95db984ba43 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -55,10 +55,6 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) { /// FIXME compute correct granularity - std::cerr << "(MergeTreeReaderCompact::readRows) max_rows_to_read: " << max_rows_to_read << "\n"; - - std::cerr << "(MergeTreeReaderCompact::readRows) from_mark: " << from_mark << "\n"; - std::cerr << "(MergeTreeReaderCompact::readRows) continue_reading: " << continue_reading << "\n"; if (continue_reading) from_mark = next_mark; @@ -75,7 +71,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, { if (!column_positions[pos]) continue; - + auto & [name, type] = *name_and_type; bool append = res_columns[pos] != nullptr; if (!append) @@ -115,10 +111,6 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, ++from_mark; read_rows += rows_to_read; - - std::cerr << "(MergeTreeReaderCompact::readRows) cur mark: " << from_mark << "\n"; - std::cerr << "(MergeTreeReaderCompact::readRows) read_rows: " << read_rows << "\n"; - std::cerr << "(MergeTreeReaderCompact::readRows) rows_to_read: " << rows_to_read << "\n"; } next_mark = from_mark; @@ -128,14 +120,9 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, void MergeTreeReaderCompact::readData( - const String & name, const IDataType & type, IColumn & column, + const String & /* name */, const IDataType & type, IColumn & column, size_t from_mark, size_t column_position, size_t rows_to_read) { - std::cerr << "(MergeTreeReaderCompact::readData) from_mark: " << from_mark << "\n"; - std::cerr << "(MergeTreeReaderCompact::readData) column_position: " << column_position << "\n"; - std::cerr << "(MergeTreeReaderCompact::readData) rows_to_read: " << rows_to_read << "\n"; - std::cerr << "(MergeTreeReaderCompact::readData) start reading column: " << name << "\n"; - /// FIXME seek only if needed seekToMark(from_mark, column_position); @@ -147,9 +134,6 @@ void MergeTreeReaderCompact::readData( IDataType::DeserializeBinaryBulkStatePtr state; type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); type.deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state); - - // std::cerr << "(MergeTreeReaderCompact::readData) end reading column rows: " << column.size() << "\n"; - // std::cerr << "(MergeTreeReaderCompact::readData) end reading column: " << name << "\n"; } @@ -166,9 +150,6 @@ void MergeTreeReaderCompact::initMarksLoader() size_t marks_count = data_part->getMarksCount(); size_t mark_size_in_bytes = data_part->index_granularity_info.mark_size_in_bytes; - std::cerr << "(initMarksLoader) marks_count: " << marks_count << "\n"; - std::cerr << "() mark_size_in_bytes: " << mark_size_in_bytes << "\n"; - size_t expected_file_size = mark_size_in_bytes * marks_count; if (expected_file_size != file_size) throw Exception( @@ -180,8 +161,6 @@ void MergeTreeReaderCompact::initMarksLoader() auto res = std::make_shared(marks_count * columns_num); - // std::cerr << "(MergeTreeReaderCompact::loadMarks) marks_count: " << marks_count << "\n"; - ReadBufferFromFile buffer(mrk_path, file_size); size_t i = 0; @@ -189,14 +168,9 @@ void MergeTreeReaderCompact::initMarksLoader() { buffer.seek(sizeof(size_t), SEEK_CUR); buffer.readStrict(reinterpret_cast(res->data() + i * columns_num), sizeof(MarkInCompressedFile) * columns_num); - // std::cerr << "(MergeTreeReaderCompact::loadMarks) i: " << i << "\n"; - // std::cerr << "(MergeTreeReaderCompact::loadMarks) buffer pos in file: " << buffer.getPositionInFile() << "\n"; ++i; } - // std::cerr << "(MergeTreeReaderCompact::loadMarks) file_size: " << file_size << "\n"; - // std::cerr << "(MergeTreeReaderCompact::loadMarks) correct file size: " << i * mark_size_in_bytes << "\n"; - if (i * mark_size_in_bytes != file_size) throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); @@ -211,9 +185,6 @@ void MergeTreeReaderCompact::initMarksLoader() void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) { MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); - - std::cerr << "(MergeTreeReaderCompact::seekToMark) mark: (" << mark.offset_in_compressed_file << ", " << mark.offset_in_decompressed_block << "\n"; - try { if (cached_buffer) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 9bf1ac7a36c..839be3104e0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -112,13 +112,9 @@ void MergeTreeReaderStream::initMarksLoader() auto load = [this](const String & mrk_path) -> MarkCache::MappedPtr { - std::cerr << "reading marks from path: " << mrk_path << "\n"; - std::cerr << "marks: " << marks_count << "\n"; /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - std::cerr << "data_file_extension: " << data_file_extension << '\n'; - size_t file_size = Poco::File(mrk_path).getSize(); size_t mark_size = mode == ReadingMode::INDEX ? index_granularity_info->skip_index_mark_size_in_bytes diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 567d9442743..6b9e15fe443 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -146,8 +146,6 @@ size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, si throw; } - // std::cerr << "(MergeTreeReaderWide::readRows) read_rows: " << read_rows << "\n"; - return read_rows; } @@ -169,9 +167,6 @@ void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type if (!data_file_exists) return; - std::cerr << "(addStreams) part: " << path << '\n'; - std::cerr << "(addStreams) marks count: " << data_part->getMarksCount() << "\n"; - streams.emplace(stream_name, std::make_unique( path + stream_name, DATA_FILE_EXTENSION, data_part->getMarksCount(), all_mark_ranges, settings, mark_cache, @@ -191,8 +186,6 @@ void MergeTreeReaderWide::readData( size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) { - std::cerr << "(MergeTreeReaderWide::readData) name: " << name << "\n"; - std::cerr << "(MergeTreeReaderWide::readData) max_rows_to_read: " << max_rows_to_read << "\n"; auto get_stream_getter = [&](bool stream_for_prefix) -> IDataType::InputStreamGetter { return [&, stream_for_prefix](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index c1a8d72bbaa..9caf605262a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -51,10 +51,6 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( columns_for_reader = data_part->columns.addTypes(columns_to_read); } - std::cerr << "(MergeTreeSequentialBlockInputStream) table: " << storage.getTableName() << "\n"; - std::cerr << "(MergeTreeSequentialBlockInputStream) part: " << data_part_->getFullPath() << "\n"; - std::cerr << "(MergeTreeSequentialBlockInputStream) columns_for_reader: " << columns_for_reader.toString() << "\n"; - MergeTreeReaderSettings reader_settings = { /// This is hack @@ -136,8 +132,6 @@ try finish(); } - std::cerr << "(MergeTreeSequentialBlockInputStream::readImpl) block: " << res.dumpStructure() << "\n"; - return res; } catch (...) diff --git a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp index 27b13d5a934..537ccaba2c1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeThreadSelectBlockInputProcessor.cpp @@ -70,10 +70,6 @@ bool MergeTreeThreadSelectBlockInputProcessor::getNewTask() owned_uncompressed_cache = storage.global_context.getUncompressedCache(); owned_mark_cache = storage.global_context.getMarkCache(); - std::cerr << "In Part: " << task->data_part->getFullPath() << "\n"; - std::cerr << "task->columns: " << task->columns.toString() << "\n"; - std::cerr << "part->columns: " << task->data_part->columns.toString() << "\n"; - reader = task->data_part->getReader(task->columns, rest_mark_ranges, owned_uncompressed_cache.get(), owned_mark_cache.get(), reader_settings, IMergeTreeReader::ValueSizeMap{}, profile_callback); diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 518321dca9c..5b3da800d99 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -14,26 +14,21 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( : IMergedBlockOutputStream(data_part), header(header_), sync(sync_) { - // std::cerr << "(MergedColumnOnlyOutputStream) storage: " << storage.getTableName() << "\n"; - // std::cerr << "(MergedColumnOnlyOutputStream) can_use_adaptive_granularity: " << can_use_adaptive_granularity << "\n"; - // std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info: " << !!index_granularity_info_ << "\n"; - // if (index_granularity_info_) - // std::cerr << "(MergedColumnOnlyOutputStream) index_granularity_info->isAdaptive(): " << index_granularity_info_->is_adaptive << "\n"; - MergeTreeWriterSettings writer_settings( data_part->storage.global_context.getSettings(), index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity()); writer_settings.filename_suffix = filename_suffix; + writer_settings.skip_offsets = skip_offsets_; writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, default_codec, writer_settings, index_granularity); - writer->setOffsetColumns(offset_columns_, skip_offsets_); + writer->setWrittenOffsetColumns(offset_columns_); writer->initSkipIndices(); } void MergedColumnOnlyOutputStream::write(const Block & block) { std::set skip_indexes_column_names_set; - for (const auto & index : storage.skip_indices) /// FIXME save local indices + for (const auto & index : writer->getSkipIndices()) std::copy(index->columns.cbegin(), index->columns.cend(), std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end()); @@ -44,8 +39,6 @@ void MergedColumnOnlyOutputStream::write(const Block & block) if (!rows) return; - std::cerr << "(MergedColumnOnlyOutputStream::write) writing rows: " << rows << "\n"; - writer->write(block); writer->calculateAndSerializeSkipIndices(skip_indexes_block, rows); writer->next(); diff --git a/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp b/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp index 42544c192ae..0579fd05f5d 100644 --- a/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp +++ b/dbms/src/Storages/tests/gtest_aux_funcs_for_adaptive_granularity.cpp @@ -3,7 +3,7 @@ #include // I know that inclusion of .cpp is not good at all -#include +#include using namespace DB; static Block getBlockWithSize(size_t required_size_in_bytes, size_t size_of_row_in_bytes) diff --git a/utils/convert-month-partitioned-parts/main.cpp b/utils/convert-month-partitioned-parts/main.cpp index d0b4d7571fa..c1b89f2251e 100644 --- a/utils/convert-month-partitioned-parts/main.cpp +++ b/utils/convert-month-partitioned-parts/main.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -80,7 +80,7 @@ void run(String part_path, String date_column, String dest_path) checksums.files["count.txt"].file_size = count_out_hashing.count(); checksums.files["count.txt"].file_hash = count_out_hashing.getHash(); - MergeTreeDataPart::MinMaxIndex minmax_idx(min_date, max_date); + IMergeTreeDataPart::MinMaxIndex minmax_idx(min_date, max_date); Names minmax_idx_columns = {date_column}; DataTypes minmax_idx_column_types = {std::make_shared()}; minmax_idx.store(minmax_idx_columns, minmax_idx_column_types, new_tmp_part_path_str, checksums); From c8393f2c8bbbd527168d7bb5fbdf1e39d04326b5 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 25 Dec 2019 16:03:59 +0300 Subject: [PATCH 0085/2007] fix mutations with mixed-granularity parts --- dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp | 3 +-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 10 +++++++--- dbms/src/Storages/MergeTree/MergeTreeData.h | 4 +++- .../MergeTree/MergeTreeDataMergerMutator.cpp | 6 ++---- .../Storages/MergeTree/MergeTreeDataPartCompact.cpp | 2 +- .../MergeTree/MergeTreeIndexGranularityInfo.cpp | 12 ++---------- .../Storages/MergeTree/MergedBlockOutputStream.cpp | 1 - 7 files changed, 16 insertions(+), 22 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index bce4e51dd49..196f73484c8 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -227,7 +227,6 @@ void IMergeTreeDataPart::setColumns(const NamesAndTypesList & columns_) sample_block.clear(); for (const auto & column : columns) sample_block.insert({column.type, column.name}); - index_granularity_info.initialize(storage, getType(), columns.size()); } IMergeTreeDataPart::~IMergeTreeDataPart() = default; @@ -591,7 +590,7 @@ void IMergeTreeDataPart::loadColumns(bool require) columns.readText(file); } - index_granularity_info.initialize(storage, getType(), columns.size()); + index_granularity_info = MergeTreeIndexGranularityInfo{storage, getType(), columns.size()}; for (const auto & it : columns) sample_block.insert({it.type, it.name}); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 79acbd5a683..0a628c3c692 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1536,11 +1536,15 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const NamesAndTypesList & columns_list, size_t bytes_uncompressed, size_t rows_count, - const String & relative_path) const + const String & relative_path, + const MergeTreeIndexGranularityInfo * index_granularity_info) const { - auto part = createPart(name, choosePartType(bytes_uncompressed, rows_count), part_info, disk, relative_path); + auto type = choosePartType(bytes_uncompressed, rows_count); + auto part = createPart(name, type, part_info, disk, relative_path); + part->index_granularity_info = index_granularity_info ? *index_granularity_info + : MergeTreeIndexGranularityInfo{*this, type, columns_list.size()}; part->setColumns(columns_list); - /// Don't save rows_count count here as it can change later + /// Don't save rows_count count here as it can be changed later return part; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index f413092c039..906f9e72859 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -187,7 +187,9 @@ public: MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info,const DiskPtr & disk, const NamesAndTypesList & columns, - size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; + size_t bytes_on_disk, size_t rows_num, + const String & relative_path, + const MergeTreeIndexGranularityInfo * index_granularity_info = nullptr) const; MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 496d8c00a1b..4a7bfa3883b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -992,14 +992,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor std::move(new_columns), source_part->bytes_on_disk, source_part->rows_count, - "tmp_mut_" + future_part.name); + "tmp_mut_" + future_part.name, + &source_part->index_granularity_info); /// Granularity info can't be changed during mutation. new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; - /// FIXME Now it's wrong code. Check if nothing will break - // new_data_part->index_granularity_info = source_part->index_granularity_info; - String new_part_tmp_path = new_data_part->getFullPath(); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 0977d18b6f8..ede5ee474c4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -160,7 +160,7 @@ String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const void MergeTreeDataPartCompact::loadIndexGranularity() { - index_granularity_info.initialize(storage, getType(), columns.size()); + index_granularity_info = MergeTreeIndexGranularityInfo{storage, getType(), columns.size()}; String full_path = getFullPath(); if (columns.empty()) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index d441f10d753..d56c69b9173 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -33,14 +33,6 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo( const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num) { - initialize(storage, part_type, columns_num); -} - -void MergeTreeIndexGranularityInfo::initialize(const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num) -{ - if (initialized) - return; - const auto storage_settings = storage.getSettings(); fixed_index_granularity = storage_settings->index_granularity; @@ -53,12 +45,12 @@ void MergeTreeIndexGranularityInfo::initialize(const MergeTreeData & storage, Me } else setAdaptive(storage_settings->index_granularity_bytes, part_type, columns_num); -} + initialized = true; +} void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const std::string & path_to_part) { - /// FIXME check when we cant create compact part auto mrk_ext = getMrkExtensionFromFS(path_to_part); if (mrk_ext && *mrk_ext == getNonAdaptiveMrkExtension()) setNonAdaptive(); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 7059f67148b..f3541d5a351 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -162,7 +162,6 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); - // new_part->setColumns(*total_column_list); new_part->index = writer->releaseIndexColumns(); new_part->checksums = checksums; new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); From aadb948c09852faa48211dea92b29ab9bf89a7ca Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 25 Dec 2019 20:08:12 +0300 Subject: [PATCH 0086/2007] temporarly store all parts in compact format --- dbms/src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index a3dbd81ff4d..7387cd3c9da 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -31,7 +31,7 @@ struct MergeTreeSettings : public SettingsCollection \ /** Data storing format settigns. */ \ M(SettingUInt64, min_bytes_for_wide_part, 0, "", 0) \ - M(SettingUInt64, min_rows_for_wide_part, 0, "", 0) \ + M(SettingUInt64, min_rows_for_wide_part, 10000000000, "", 0) \ \ /** Merge settings. */ \ M(SettingUInt64, merge_max_block_size, DEFAULT_MERGE_BLOCK_SIZE, "How many rows in blocks should be formed for merge operations.", 0) \ From c298616eacd39d2cde94ec1e7845217a4758fef4 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 25 Dec 2019 20:34:23 +0300 Subject: [PATCH 0087/2007] reduce number of seeks in ReaderCompact --- .../MergeTree/MergeTreeReaderCompact.cpp | 21 +++++++++++-------- .../MergeTree/MergeTreeReaderCompact.h | 6 ++++-- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 95db984ba43..4a183710ab9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -83,7 +83,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, try { size_t column_size_before_reading = column->size(); - readData(name, *type, *column, from_mark, *column_positions[pos], rows_to_read); + readData(*column, *type, from_mark, *column_positions[pos], rows_to_read); size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column < rows_to_read) @@ -120,11 +120,11 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, void MergeTreeReaderCompact::readData( - const String & /* name */, const IDataType & type, IColumn & column, + IColumn & column, const IDataType & type, size_t from_mark, size_t column_position, size_t rows_to_read) { - /// FIXME seek only if needed - seekToMark(from_mark, column_position); + if (!isContinuousReading(from_mark, column_position)) + seekToMark(from_mark, column_position); IDataType::DeserializeBinaryBulkSettings deserialize_settings; deserialize_settings.getter = [&](IDataType::SubstreamPath) -> ReadBuffer * { return data_buffer; }; @@ -134,6 +134,8 @@ void MergeTreeReaderCompact::readData( IDataType::DeserializeBinaryBulkStatePtr state; type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); type.deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state); + + last_read_granule.emplace(from_mark, column_position); } @@ -203,12 +205,13 @@ void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) } -void MergeTreeReaderCompact::seekToStart() +bool MergeTreeReaderCompact::isContinuousReading(size_t mark, size_t column_position) { - if (cached_buffer) - cached_buffer->seek(0, 0); - if (non_cached_buffer) - non_cached_buffer->seek(0, 0); + if (!last_read_granule) + return false; + const auto & [last_mark, last_column] = *last_read_granule; + return (mark == last_mark && column_position == last_column + 1) + || (mark == last_mark + 1 && column_position == 0 && last_column == data_part->columns.size() - 1); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 719516a39cd..9aef896c272 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -29,6 +29,8 @@ public: bool canReadIncompleteGranules() const override { return false; } private: + bool isContinuousReading(size_t mark, size_t column_position); + ReadBuffer * data_buffer; std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; @@ -38,12 +40,12 @@ private: std::vector> column_positions; size_t next_mark = 0; + std::optional> last_read_granule; void initMarksLoader(); - void seekToStart(); void seekToMark(size_t row_index, size_t column_index); - void readData(const String & name, const IDataType & type, IColumn & column, + void readData(IColumn & column, const IDataType & type, size_t from_mark, size_t column_position, size_t rows_to_read); /// Columns that are read. From 0b99df9aed0a8536c9ecece80fd90048254c84bb Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 25 Dec 2019 23:06:16 +0300 Subject: [PATCH 0088/2007] better column initialization in data parts --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 20 +++++++++++-------- .../Storages/MergeTree/IMergeTreeDataPart.h | 14 ++++++------- .../MergeTree/MergeTreeDataPartCompact.cpp | 7 +++++++ .../MergeTree/MergeTreeDataPartCompact.h | 2 ++ 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 196f73484c8..270c574df83 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -182,9 +182,10 @@ String IMergeTreeDataPart::getNewName(const MergeTreePartInfo & new_part_info) c std::optional IMergeTreeDataPart::getColumnPosition(const String & column_name) const { - if (!sample_block.has(column_name)) + auto it = column_name_to_position.find(column_name); + if (it == column_name_to_position.end()) return {}; - return sample_block.getPositionByName(column_name); + return it->second; } DayNum IMergeTreeDataPart::getMinDate() const @@ -221,12 +222,14 @@ time_t IMergeTreeDataPart::getMaxTime() const return 0; } -void IMergeTreeDataPart::setColumns(const NamesAndTypesList & columns_) +void IMergeTreeDataPart::setColumns(const NamesAndTypesList & new_columns) { - columns = columns_; - sample_block.clear(); + columns = new_columns; + column_name_to_position.clear(); + column_name_to_position.reserve(new_columns.size()); + size_t pos = 0; for (const auto & column : columns) - sample_block.insert({column.type, column.name}); + column_name_to_position.emplace(column.name, pos++); } IMergeTreeDataPart::~IMergeTreeDataPart() = default; @@ -591,8 +594,9 @@ void IMergeTreeDataPart::loadColumns(bool require) } index_granularity_info = MergeTreeIndexGranularityInfo{storage, getType(), columns.size()}; - for (const auto & it : columns) - sample_block.insert({it.type, it.name}); + size_t pos = 0; + for (const auto & column : columns) + column_name_to_position.emplace(column.name, pos++); } void IMergeTreeDataPart::loadColumnSizes() diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index e03fa09b63d..323dd21df6d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -49,6 +49,7 @@ public: using MergeTreeWriterPtr = std::unique_ptr; using ColumnSizeByName = std::unordered_map; + using NameToPosition = std::unordered_map; virtual MergeTreeReaderPtr getReader( const NamesAndTypesList & columns_, @@ -84,12 +85,11 @@ public: virtual String getFileNameForColumn(const NameAndTypePair & column) const = 0; + virtual void setColumns(const NamesAndTypesList & columns_); + virtual NameToNameMap createRenameMapForAlter( AlterAnalysisResult & /* analysis_result */, - const NamesAndTypesList & /* old_columns */) const - { - return {}; - } + const NamesAndTypesList & /* old_columns */) const { return {}; } virtual ~IMergeTreeDataPart(); @@ -111,7 +111,6 @@ public: static String typeToString(Type type); String getTypeName() { return typeToString(getType()); } - IMergeTreeDataPart( const MergeTreeData & storage_, const String & name_, @@ -152,8 +151,6 @@ public: time_t getMinTime() const; time_t getMaxTime() const; - void setColumns(const NamesAndTypesList & columns_); - bool isEmpty() const { return rows_count == 0; } const MergeTreeData & storage; @@ -331,7 +328,8 @@ protected: virtual void checkConsistency(bool require_part_metadata) const; private: - Block sample_block; + /// In compact parts order of columns is necessary + NameToPosition column_name_to_position; /// Reads columns names and types from columns.txt void loadColumns(bool require); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index ede5ee474c4..95f4f9a818d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -253,6 +253,13 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( return rename_map; } +void MergeTreeDataPartCompact::setColumns(const NamesAndTypesList & new_columns) +{ + if (new_columns.size() != columns.size()) + index_granularity_info = MergeTreeIndexGranularityInfo{storage, Type::COMPACT, new_columns.size()}; + IMergeTreeDataPart::setColumns(new_columns); +} + void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { UNUSED(require_part_metadata); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index c1637b94d9b..4afed5c8785 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -88,6 +88,8 @@ public: AlterAnalysisResult & analysis_result, const NamesAndTypesList & old_columns) const override; + void setColumns(const NamesAndTypesList & new_columns) override; + ~MergeTreeDataPartCompact() override; private: From 145f0702e57ec6740d516b3fa3131b5555ce0128 Mon Sep 17 00:00:00 2001 From: Aleksei Levushkin Date: Fri, 27 Dec 2019 16:33:08 +0300 Subject: [PATCH 0089/2007] added grpc as submodule --- .gitmodules | 3 +++ contrib/CMakeLists.txt | 2 ++ contrib/grpc | 1 + 3 files changed, 6 insertions(+) create mode 160000 contrib/grpc diff --git a/.gitmodules b/.gitmodules index 0b80743cadb..97c2b27c364 100644 --- a/.gitmodules +++ b/.gitmodules @@ -106,3 +106,6 @@ [submodule "contrib/sparsehash-c11"] path = contrib/sparsehash-c11 url = https://github.com/sparsehash/sparsehash-c11.git +[submodule "contrib/grpc"] + path = contrib/grpc + url = https://github.com/grpc/grpc.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index b0a271b21ac..e434bdb3c07 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -332,6 +332,8 @@ if (USE_FASTOPS) add_subdirectory (fastops-cmake) endif() +add_subdirectory(grpc-cmake) + #if (USE_INTERNAL_ORC_LIBRARY) # add_subdirectory(orc-cmake) #endif () diff --git a/contrib/grpc b/contrib/grpc new file mode 160000 index 00000000000..c1d176528fd --- /dev/null +++ b/contrib/grpc @@ -0,0 +1 @@ +Subproject commit c1d176528fd8da9dd4066d16554bcd216d29033f From bbd368cdac18742e32830d16c72cc199d841180d Mon Sep 17 00:00:00 2001 From: Aleksei Levushkin Date: Fri, 27 Dec 2019 16:33:42 +0300 Subject: [PATCH 0090/2007] added grpc-cmake --- contrib/grpc-cmake/CMakeLists.txt | 1482 +++++++++++++++++++++++++++++ 1 file changed, 1482 insertions(+) create mode 100644 contrib/grpc-cmake/CMakeLists.txt diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt new file mode 100644 index 00000000000..85f3e6c6737 --- /dev/null +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -0,0 +1,1482 @@ +cmake_minimum_required(VERSION 3.5.1) + +set(GRPC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../grpc) +set(GRPC_INCLUDE_DIR ${GRPC_SOURCE_DIR}/include/) + +if(UNIX) + if(${CMAKE_SYSTEM_NAME} MATCHES "Linux") + set(_gRPC_PLATFORM_LINUX ON) + elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") + set(_gRPC_PLATFORM_MAC ON) + else() + set(_gRPC_PLATFORM_POSIX ON) + endif() +endif() + +set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_gRPC_C_CXX_FLAGS}") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_gRPC_C_CXX_FLAGS}") + +set(_gRPC_PROTOBUF_LIBRARY_NAME "libprotobuf") + +if(gRPC_BACKWARDS_COMPATIBILITY_MODE) + add_definitions(-DGPR_BACKWARDS_COMPATIBILITY_MODE) + if (_gRPC_PLATFORM_MAC) + # some C++11 constructs not supported before OS X 10.9 + set(CMAKE_OSX_DEPLOYMENT_TARGET 10.9) + endif() +endif() + +if (_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC) + set(_gRPC_CORE_NOSTDCXX_FLAGS -fno-exceptions -fno-rtti) +else() + set(_gRPC_CORE_NOSTDCXX_FLAGS "") +endif() + +# address_sorting.cmake +include(${GRPC_SOURCE_DIR}/cmake/address_sorting.cmake) +set(_gRPC_ADDRESS_SORTING_INCLUDE_DIR "${GRPC_SOURCE_DIR}/third_party/address_sorting/include") +set(_gRPC_ADDRESS_SORTING_LIBRARIES address_sorting) + +# cares.cmake +set(CARES_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/cares/cares) +set(CARES_SHARED OFF CACHE BOOL "disable shared library") +set(CARES_STATIC ON CACHE BOOL "link cares statically") +if(gRPC_BACKWARDS_COMPATIBILITY_MODE) + # See https://github.com/grpc/grpc/issues/17255 + set(HAVE_LIBNSL OFF CACHE BOOL "avoid cares dependency on libnsl") +endif() +add_subdirectory(${CARES_ROOT_DIR} ${CARES_ROOT_DIR}) +if(TARGET c-ares) + set(_gRPC_CARES_LIBRARIES c-ares) +endif() + +# protobuf.cmake +set(PROTOBUF_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../protobuf) +if(NOT protobuf_BUILD_TESTS) + set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests") +endif() +if(NOT protobuf_WITH_ZLIB) + set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build protobuf with zlib.") +endif() +set(protobuf_MSVC_STATIC_RUNTIME OFF CACHE BOOL "Link static runtime libraries") +if(NOT USE_INTERNAL_PROTOBUF_LIBRARY) + add_subdirectory(${PROTOBUF_ROOT_DIR}/cmake ${GRPC_SOURCE_DIR}/third_party/protobuf) +endif() +if(TARGET ${_gRPC_PROTOBUF_LIBRARY_NAME}) + set(_gRPC_PROTOBUF_LIBRARIES ${_gRPC_PROTOBUF_LIBRARY_NAME}) +endif() +if(TARGET libprotoc) + set(_gRPC_PROTOBUF_PROTOC_LIBRARIES libprotoc) +endif() +if(TARGET protoc) + set(_gRPC_PROTOBUF_PROTOC protoc) + set(_gRPC_PROTOBUF_PROTOC_EXECUTABLE $) +endif() +# For well-known .proto files distributed with protobuf +set(_gRPC_PROTOBUF_WELLKNOWN_INCLUDE_DIR "${PROTOBUF_ROOT_DIR}/src") +if(gRPC_INSTALL) + message(WARNING "gRPC_INSTALL will be forced to FALSE because gRPC_PROTOBUF_PROVIDER is \"module\"") + set(gRPC_INSTALL FALSE) +endif() + +# ssl.cmake +set(BORINGSSL_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../ssl) +if(TARGET ssl) + set(_gRPC_SSL_LIBRARIES ssl) + set(_gRPC_SSL_INCLUDE_DIR ${BORINGSSL_ROOT_DIR}/include) +endif() + +# upb.cmake +set(UPB_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/upb) +set(_gRPC_UPB_INCLUDE_DIR "${UPB_ROOT_DIR}") +set(_gRPC_UPB_GRPC_GENERATED_DIR "${GRPC_SOURCE_DIR}/src/core/ext/upb-generated") +set(_gRPC_UPB_LIBRARIES upb) + +# zlib.cmake +set(ZLIB_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/zlib) +include_directories("${ZLIB_ROOT_DIR}") +add_subdirectory(${ZLIB_ROOT_DIR} ${ZLIB_ROOT_DIR}) +set(_gRPC_ZLIB_LIBRARIES zlibstatic) +set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}") + + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c99") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + +if(_gRPC_PLATFORM_MAC) + set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} m pthread) +elseif(UNIX) + set(_gRPC_ALLTARGETS_LIBRARIES ${CMAKE_DL_LIBS} rt m pthread) +endif() + +add_library(address_sorting + ${GRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting.c + ${GRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting_posix.c + ${GRPC_SOURCE_DIR}/third_party/address_sorting/address_sorting_windows.c +) + +target_include_directories(address_sorting + PUBLIC $ $ + PRIVATE ${GRPC_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(address_sorting + ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} +) + +add_library(gpr + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/alloc.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/atm.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/cpu_iphone.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/cpu_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/cpu_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/cpu_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/env_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/env_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/env_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/log.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/log_android.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/log_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/log_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/log_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/murmur_hash.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/string_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/string_util_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/string_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/sync.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/sync_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/sync_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/time.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/time_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/time_precise.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/time_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/tls_pthread.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/tmpfile_msys.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/tmpfile_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/tmpfile_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gpr/wrap_memcpy.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gprpp/arena.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gprpp/fork.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gprpp/global_config_env.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gprpp/host_port.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gprpp/mpscq.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gprpp/thd_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/gprpp/thd_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/profiling/basic_timers.cc + ${GRPC_SOURCE_DIR}/src/core/lib/profiling/stap_timers.cc +) + +target_include_directories(gpr + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${GRPC_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(gpr + ${_gRPC_ALLTARGETS_LIBRARIES} +) + +add_library(grpc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/init.cc + ${GRPC_SOURCE_DIR}/src/core/lib/avl/avl.cc + ${GRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker_registry.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_args.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_gzip.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_identity.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/parser.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epollex_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/mpmcqueue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/threadpool.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/is_epollexclusive_available.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resource_quota.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/time_averaged_stats.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/udp_server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_intern.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/version.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/byte_stream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/static_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/secure/server_secure_chttp2.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/context_list.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_table.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/incoming_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_map.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/message_compress_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/alts_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/credentials_generic.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/google_default_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/iam/iam_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/json_token.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_verifier.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/local/local_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/ssl/ssl_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/spiffe_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/alts/alts_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/local/local_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils_config.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/tls/spiffe_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/target_authority_table.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/init_secure.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/aes_gcm.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/gsec.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_counter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_crypter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_frame_protector.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/frame_handler.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_handshaker_client.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_shared_resource.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_utils.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/decode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/encode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/msg.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/port.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/table.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/upb.c + ${GRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/authority.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/connector.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/health/health_check_client.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_connect_handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/parse_address.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_result_parsing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolving_lb_policy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/server_address.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/data/orca/v1/orca_load_report.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/gogoproto/gogo.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c + ${GRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_cache.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel_secure.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_api.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_bootstrap.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_channel_secure.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_client.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_client_stats.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/auth/cert.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cds.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cluster/circuit_breaker.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cluster/filter.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cluster/outlier_detection.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/discovery.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/eds.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/endpoint/endpoint.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/endpoint/load_report.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/discovery/v2/ads.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/load_stats/v2/lrs.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/address.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/base.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/config_source.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/grpc_service.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/health_check.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/http_uri.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/protocol.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/percent.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/range.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_libuv.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/census/grpc_context.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_idle/client_idle_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/max_age/max_age_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/message_size/message_size_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/workarounds/workaround_utils.cc + ${GRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc +) + +target_compile_options(grpc PUBLIC -fpermissive) + +target_include_directories(grpc + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${GRPC_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc + ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_SSL_LIBRARIES} + ${_gRPC_ZLIB_LIBRARIES} + ${_gRPC_CARES_LIBRARIES} + ${_gRPC_ADDRESS_SORTING_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + gpr +) +if (_gRPC_PLATFORM_MAC) + target_link_libraries(grpc "-framework CoreFoundation") +endif() + +add_library(grpc_cronet + ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/plugin_registry/grpc_cronet_plugin_registry.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/init.cc + ${GRPC_SOURCE_DIR}/src/core/lib/avl/avl.cc + ${GRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker_registry.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_args.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_gzip.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_identity.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/parser.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epollex_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/mpmcqueue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/threadpool.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/is_epollexclusive_available.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resource_quota.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/time_averaged_stats.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/udp_server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_intern.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/version.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/byte_stream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/static_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/client/secure/cronet_channel_create.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/transport/cronet_api_dummy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/cronet/transport/cronet_transport.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/secure/secure_channel_create.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/context_list.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_table.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/incoming_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_map.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/message_compress_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/connector.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/health/health_check_client.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_connect_handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/parse_address.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_result_parsing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolving_lb_policy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/server_address.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/decode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/encode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/msg.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/port.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/table.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/data/orca/v1/orca_load_report.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/gogoproto/gogo.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c + ${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/context/security_context.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/alts_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/composite/composite_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/credentials_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/fake/fake_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/credentials_generic.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/google_default/google_default_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/iam/iam_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/json_token.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/jwt/jwt_verifier.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/local/local_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/oauth2/oauth2_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/plugin/plugin_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/ssl/ssl_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/grpc_tls_credentials_options.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/tls/spiffe_credentials.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/alts/alts_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/fake/fake_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_fallback.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/load_system_roots_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/local/local_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl/ssl_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/ssl_utils_config.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/security_connector/tls/spiffe_security_connector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/client_auth_filter.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/secure_endpoint.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/security_handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/server_auth_filter.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/target_authority_table.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/transport/tsi_error.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/util/json_util.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/init_secure.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/aes_gcm.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/crypt/gsec.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_counter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_crypter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_frame_protector.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_record_protocol_crypter_common.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_seal_privacy_integrity_crypter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/alts_unseal_privacy_integrity_crypter.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/frame_protector/frame_handler.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_handshaker_client.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_shared_resource.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_integrity_only_record_protocol.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_privacy_integrity_record_protocol.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_grpc_record_protocol_common.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_iovec_record_protocol.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/zero_copy_frame_protector/alts_zero_copy_grpc_protector.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_no_op.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/check_gcp_environment_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_client_options.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_options.cc + ${GRPC_SOURCE_DIR}/src/core/lib/security/credentials/alts/grpc_alts_credentials_server_options.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/alts_tsi_utils.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/alts/handshaker/transport_security_common_api.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/altscontext.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/handshaker.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/gcp/transport_security_common.upb.c + ${GRPC_SOURCE_DIR}/src/core/tsi/transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/authority.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/fake_transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/local_transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_boringssl.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_cache.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl/session_cache/ssl_session_openssl.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/ssl_transport_security.cc + ${GRPC_SOURCE_DIR}/src/core/tsi/transport_security_grpc.cc +) + +target_include_directories(grpc_cronet + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${GRPC_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc_cronet + ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_SSL_LIBRARIES} + ${_gRPC_ZLIB_LIBRARIES} + ${_gRPC_CARES_LIBRARIES} + ${_gRPC_ADDRESS_SORTING_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + gpr +) +if (_gRPC_PLATFORM_MAC) + target_link_libraries(grpc_cronet "-framework CoreFoundation") +endif() + +add_library(grpc_unsecure + ${GRPC_SOURCE_DIR}/src/core/lib/surface/init.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/init_unsecure.cc + ${GRPC_SOURCE_DIR}/src/core/lib/avl/avl.cc + ${GRPC_SOURCE_DIR}/src/core/lib/backoff/backoff.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_args.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_stack_builder.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channel_trace.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/channelz_registry.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/connected_channel.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/handshaker_registry.cc + ${GRPC_SOURCE_DIR}/src/core/lib/channel/status_util.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_args.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/compression_internal.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/message_compress.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_gzip.cc + ${GRPC_SOURCE_DIR}/src/core/lib/compression/stream_compression_identity.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/stats_data.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/format_request.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/httpcli.cc + ${GRPC_SOURCE_DIR}/src/core/lib/http/parser.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/buffer_list.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/call_combiner.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/cfstream_handle.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/combiner.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/endpoint_pair_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/error_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epoll1_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_epollex_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_poll_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/ev_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/exec_ctx.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/mpmcqueue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/executor/threadpool.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/fork_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_fallback.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_host_name_max.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/gethostname_sysconf.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/grpc_if_nametoindex_unsupported.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/internal_errqueue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iocp_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_internal.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_posix_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/iomgr_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/is_epollexclusive_available.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/load_file.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/lockfree_event.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/polling_entity.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/resource_quota.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_common_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_linux.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_utils_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_cfstream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_client_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_common.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_ifaddrs.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_utils_posix_noifaddrs.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_server_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/tcp_windows.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/time_averaged_stats.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_custom.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_generic.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_heap.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_manager.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/timer_uv.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/udp_server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/unix_sockets_posix_noop.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_eventfd.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_nospecial.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_pipe.cc + ${GRPC_SOURCE_DIR}/src/core/lib/iomgr/wakeup_fd_posix.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_reader.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/json/json_writer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/b64.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_buffer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_intern.cc + ${GRPC_SOURCE_DIR}/src/core/lib/slice/slice_string_helpers.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/api_trace.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/byte_buffer_reader.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_details.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/call_log_batch.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_init.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_ping.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/channel_stack_type.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/completion_queue_factory.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/event_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/lame_client.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/metadata_array.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/server.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/validate_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/surface/version.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/bdp_estimator.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/byte_stream.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/connectivity_state.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/error_utils.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/metadata_batch.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/pid_controller.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/static_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_conversion.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/status_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/timeout_encoding.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport.cc + ${GRPC_SOURCE_DIR}/src/core/lib/transport/transport_op_string.cc + ${GRPC_SOURCE_DIR}/src/core/lib/uri/uri_parser.cc + ${GRPC_SOURCE_DIR}/src/core/lib/debug/trace.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/insecure/server_chttp2.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/insecure/server_chttp2_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/context_list.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/flow_control.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_data.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_goaway.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_ping.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_rst_stream.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_settings.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/frame_window_update.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_encoder.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_parser.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/hpack_table.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/http2_settings.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/huffsyms.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/incoming_metadata.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/parsing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_map.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/writing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/alpn/alpn.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/client/http_client_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/http_filters_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/message_compress/message_compress_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/server/http_server_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/insecure/channel_create_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/authority.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backend_metric.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/backup_poller.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/channel_connectivity.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_channelz.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_factory.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/client_channel_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/connector.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/global_subchannel_pool.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/health/health_check_client.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_connect_handshaker.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/http_proxy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/local_subchannel_pool.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/parse_address.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/proxy_mapper_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_registry.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver_result_parsing.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolving_lb_policy.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/retry_throttle.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/server_address.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/service_config.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/subchannel_pool_interface.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/deadline/deadline_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/decode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/encode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/msg.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/port.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/table.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/udpa/data/orca/v1/orca_load_report.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/gogoproto/gogo.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/validate/validate.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/annotations.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/api/http.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/any.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/descriptor.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/duration.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/empty.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/struct.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/timestamp.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/protobuf/wrappers.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/google/rpc/status.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_plugin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/transport/inproc/inproc_transport.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_libuv.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_fallback.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_libuv.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/dns_resolver_selection.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/sockaddr/sockaddr_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/fake/fake_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/resolver/xds/xds_resolver.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_api.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_bootstrap.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_channel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_client.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/xds/xds_client_stats.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/auth/cert.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cds.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cluster/circuit_breaker.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cluster/filter.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/cluster/outlier_detection.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/discovery.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/eds.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/endpoint/endpoint.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/endpoint/load_report.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/discovery/v2/ads.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/service/load_stats/v2/lrs.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/address.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/base.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/config_source.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/grpc_service.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/health_check.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/http_uri.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/api/v2/core/protocol.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/percent.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/envoy/type/range.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/client_load_reporting_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_channel.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb_client_stats.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/grpclb/load_balancer_api.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/lb/v1/load_balancer.upb.c + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/xds/xds.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/census/grpc_context.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/client_idle/client_idle_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/max_age/max_age_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/message_size/message_size_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/http/client_authority_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/workarounds/workaround_cronet_compression_filter.cc + ${GRPC_SOURCE_DIR}/src/core/ext/filters/workarounds/workaround_utils.cc + ${GRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_unsecure_plugin_registry.cc +) + + +target_include_directories(grpc_unsecure + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${GRPC_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(grpc_unsecure + ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_ZLIB_LIBRARIES} + ${_gRPC_CARES_LIBRARIES} + ${_gRPC_ADDRESS_SORTING_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + gpr +) +if (_gRPC_PLATFORM_MAC) + target_link_libraries(grpc_unsecure "-framework CoreFoundation") +endif() + +add_library(grpc++ + ${GRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/secure_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/auth_property_iterator.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/secure_auth_context.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/secure_channel_arguments.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/secure_create_auth_context.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/tls_credentials_options.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/tls_credentials_options_util.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/insecure_server_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/secure_server_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/client_context.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/client_interceptor.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/create_channel.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/create_channel_internal.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/create_channel_posix.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/credentials_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/generic_stub.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/alarm.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/channel_arguments.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/channel_filter.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/completion_queue_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/core_codegen.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/resource_quota_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/rpc_method.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/validate_service_config.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/version_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/async_generic_service.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/channel_argument_option.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/create_default_thread_pool.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/dynamic_thread_pool.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/external_connection_acceptor_impl.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/health/default_health_check_service.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service_server_builder_option.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_builder.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_context.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_posix.cc + ${GRPC_SOURCE_DIR}/src/cpp/thread_manager/thread_manager.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/byte_buffer_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/status.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/string_ref.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/time_cc.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/decode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/encode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/msg.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/port.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/table.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/upb.c + ${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc +) + +target_compile_options(grpc++ PUBLIC -Wno-error=shadow) + +target_include_directories(grpc++ + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${GRPC_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) +target_link_libraries(grpc++ + ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_SSL_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc + gpr +) + +add_library(grpc++_unsecure + ${GRPC_SOURCE_DIR}/src/cpp/client/insecure_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/insecure_create_auth_context.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/insecure_server_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/channel_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/client_context.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/client_interceptor.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/create_channel.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/create_channel_internal.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/create_channel_posix.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/credentials_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/client/generic_stub.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/alarm.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/channel_arguments.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/channel_filter.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/completion_queue_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/core_codegen.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/resource_quota_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/rpc_method.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/validate_service_config.cc + ${GRPC_SOURCE_DIR}/src/cpp/common/version_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/async_generic_service.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/channel_argument_option.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/create_default_thread_pool.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/dynamic_thread_pool.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/external_connection_acceptor_impl.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/health/default_health_check_service.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/health/health_check_service_server_builder_option.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_builder.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_context.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_credentials.cc + ${GRPC_SOURCE_DIR}/src/cpp/server/server_posix.cc + ${GRPC_SOURCE_DIR}/src/cpp/thread_manager/thread_manager.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/byte_buffer_cc.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/status.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/string_ref.cc + ${GRPC_SOURCE_DIR}/src/cpp/util/time_cc.cc + ${GRPC_SOURCE_DIR}/src/core/ext/upb-generated/src/proto/grpc/health/v1/health.upb.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/decode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/encode.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/msg.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/port.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/table.c + ${GRPC_SOURCE_DIR}/third_party/upb/upb/upb.c + ${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc +) + +target_compile_options(grpc++_unsecure PUBLIC -Wno-error=shadow) + +target_include_directories(grpc++_unsecure + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${GRPC_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) +target_link_libraries(grpc++_unsecure + ${_gRPC_BASELIB_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + gpr + grpc_unsecure +) + +add_library(grpc_plugin_support + ${GRPC_SOURCE_DIR}/src/compiler/cpp_generator.cc +) + +target_include_directories(grpc_plugin_support + PRIVATE ${GRPC_SOURCE_DIR} + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) +target_link_libraries(grpc_plugin_support + ${_gRPC_PROTOBUF_PROTOC_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} +) + +add_executable(grpc_cpp_plugin + ${GRPC_SOURCE_DIR}/src/compiler/cpp_plugin.cc +) + +target_include_directories(grpc_cpp_plugin + PRIVATE ${GRPC_SOURCE_DIR} + PUBLIC ${GRPC_INCLUDE_DIR} + PRIVATE ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + PRIVATE ${_gRPC_BENCHMARK_INCLUDE_DIR} + PRIVATE ${_gRPC_CARES_INCLUDE_DIR} + PRIVATE ${_gRPC_GFLAGS_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTOBUF_INCLUDE_DIR} + PRIVATE ${_gRPC_SSL_INCLUDE_DIR} + PRIVATE ${_gRPC_UPB_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_GRPC_GENERATED_DIR} + PRIVATE ${_gRPC_UPB_INCLUDE_DIR} + PRIVATE ${_gRPC_ZLIB_INCLUDE_DIR} + PRIVATE ${_gRPC_PROTO_GENS_DIR} +) + +target_link_libraries(grpc_cpp_plugin + ${_gRPC_PROTOBUF_PROTOC_LIBRARIES} + ${_gRPC_PROTOBUF_LIBRARIES} + ${_gRPC_ALLTARGETS_LIBRARIES} + grpc_plugin_support +) \ No newline at end of file From 74d5c6edc7dea50ebc653ada774e60a18f1b9411 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Sat, 28 Dec 2019 00:17:53 +0300 Subject: [PATCH 0091/2007] better writer for compact parts --- .../MergeTree/IMergeTreeDataPartWriter.cpp | 36 ++++--- .../MergeTree/IMergeTreeDataPartWriter.h | 2 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 4 +- .../MergeTreeDataPartWriterCompact.cpp | 94 +++++++++++++------ .../MergeTreeDataPartWriterCompact.h | 21 ++++- .../MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 4 +- 7 files changed, 103 insertions(+), 60 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 2cad6ee1930..60ce5b6b947 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -100,7 +100,6 @@ static void fillIndexGranularityImpl( bool can_use_adaptive_index_granularity, bool need_finish_last_granule = false) { - /// FIXME correct index granularity for compact size_t rows_in_block = block.rows(); size_t index_granularity_for_block; if (!can_use_adaptive_index_granularity) @@ -131,25 +130,22 @@ static void fillIndexGranularityImpl( for (current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) { size_t rows_left_in_block = rows_in_block - current_row; - - // if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block) - // { - // /// If enough rows are left, create a new granule. Otherwise, extend previous granule. - // /// So,real size of granule differs from index_granularity_for_block not more than 50%. - // if (rows_left_in_block * 2 >= index_granularity_for_block) - // index_granularity.appendMark(rows_left_in_block); - // else - // index_granularity.addRowsToLastMark(rows_left_in_block); - // } - // else - // { - // index_granularity.appendMark(index_granularity_for_block); - // } - - if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block) - index_granularity.appendMark(rows_left_in_block); + + /// FIXME need comment + if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block + && (rows_in_block >= index_granularity_for_block || index_offset != 0)) + { + // If enough rows are left, create a new granule. Otherwise, extend previous granule. + // So, real size of granule differs from index_granularity_for_block not more than 50%. + if (rows_left_in_block * 2 >= index_granularity_for_block) + index_granularity.appendMark(rows_left_in_block); + else + index_granularity.addRowsToLastMark(rows_left_in_block); + } else + { index_granularity.appendMark(index_granularity_for_block); + } } } @@ -164,7 +160,7 @@ void IMergeTreeDataPartWriter::fillIndexGranularity(const Block & block) index_offset, index_granularity, settings.can_use_adaptive_granularity, - need_finish_last_granule); + need_finish_last_granule && index_offset != 0); } void IMergeTreeDataPartWriter::initPrimaryIndex() @@ -256,7 +252,7 @@ void IMergeTreeDataPartWriter::calculateAndSerializeSkipIndices( size_t skip_index_current_data_mark = 0; - /// Filling and writing skip indices like in IMergeTreeDataPartWriter::writeColumn + /// Filling and writing skip indices like in MergeTreeDataPartWriterWide::writeColumn for (size_t i = 0; i < skip_indices.size(); ++i) { const auto index = skip_indices[i]; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 96c5b7807e1..1408370417c 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -14,7 +14,7 @@ namespace DB { -class IMergeTreeDataPartWriter +class IMergeTreeDataPartWriter : private boost::noncopyable { public: using WrittenOffsetColumns = std::set; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 95f4f9a818d..09d87016911 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -178,11 +178,11 @@ void MergeTreeDataPartCompact::loadIndexGranularity() ReadBufferFromFile buffer(marks_file_path, marks_file_size); while (!buffer.eof()) { + /// Skip offsets for columns + buffer.seek(columns.size() * sizeof(MarkInCompressedFile), SEEK_CUR); size_t granularity; readIntBinary(granularity, buffer); index_granularity.appendMark(granularity); - /// Skip offsets for columns - buffer.seek(columns.size() * sizeof(MarkInCompressedFile), SEEK_CUR); } if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index d46d556d7b9..7896c0e1c61 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -23,7 +23,6 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( storage_, columns_list_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_, true) - , squashing(storage.getSettings()->index_granularity, storage.getSettings()->index_granularity_bytes) /// FIXME { String data_file_name = DATA_FILE_NAME + settings.filename_suffix; stream = std::make_unique( @@ -73,13 +72,18 @@ void MergeTreeDataPartWriterCompact::write( if (!header) header = result_block.cloneEmpty(); - auto result = squashing.add(result_block.mutateColumns()); - if (!result.ready) + columns_buffer.add(result_block.mutateColumns()); + size_t last_mark_rows = index_granularity.getLastMarkRows(); + size_t rows_in_buffer = columns_buffer.size(); + + if (rows_in_buffer < last_mark_rows) + { + /// FIXME need comment + next_index_offset = last_mark_rows - rows_in_buffer; return; + } - result_block = header.cloneWithColumns(std::move(result.columns)); - - writeBlock(result_block); + writeBlock(header.cloneWithColumns(columns_buffer.releaseColumns())); } void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) @@ -94,32 +98,43 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) if (rows_to_write) data_written = true; - /// There could already be enough data to compress into the new block. if (stream->compressed.offset() >= settings.min_compress_block_size) stream->compressed.next(); - size_t next_row = 0; - writeIntBinary(rows_to_write, stream->marks); - for (const auto & it : columns_list) - next_row = writeColumnSingleGranule(block.getByName(it.name), current_row, rows_to_write); + for (const auto & column : columns_list) + { + size_t old_uncompressed_size = stream->compressed.count(); + writeIntBinary(stream->plain_hashing.count(), stream->marks); + writeIntBinary(stream->compressed.offset(), stream->marks); + + writeColumnSingleGranule(block.getByName(column.name), current_row, rows_to_write); + + /// We can't calculate compressed size by single column in compact format. + size_t uncompressed_size = stream->compressed.count(); + columns_sizes[column.name].add(ColumnSize{0, 0, uncompressed_size - old_uncompressed_size}); + } ++from_mark; - current_row = next_row; + size_t rows_written = total_rows - current_row; + current_row += rows_to_write; + + if (current_row >= total_rows && rows_written != rows_to_write) + { + rows_to_write = rows_written; + index_granularity.popMark(); + index_granularity.appendMark(rows_written); + } + + writeIntBinary(rows_to_write, stream->marks); } + next_index_offset = 0; next_mark = from_mark; - next_index_offset = total_rows - current_row; } - -size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) +void MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWithTypeAndName & column, size_t from_row, size_t number_of_rows) const { - size_t old_uncompressed_size = stream->compressed.count(); - - writeIntBinary(stream->plain_hashing.count(), stream->marks); - writeIntBinary(stream->compressed.offset(), stream->marks); - IDataType::SerializeBinaryBulkStatePtr state; IDataType::SerializeBinaryBulkSettings serialize_settings; @@ -130,28 +145,21 @@ size_t MergeTreeDataPartWriterCompact::writeColumnSingleGranule(const ColumnWith column.type->serializeBinaryBulkStatePrefix(serialize_settings, state); column.type->serializeBinaryBulkWithMultipleStreams(*column.column, from_row, number_of_rows, serialize_settings, state); column.type->serializeBinaryBulkStateSuffix(serialize_settings, state); - - /// We can't calculate compressed size by single column in compact format. - size_t uncompressed_size = stream->compressed.count(); - columns_sizes[column.name].add(ColumnSize{0, 0, uncompressed_size - old_uncompressed_size}); - - return from_row + number_of_rows; } void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) { - auto result = squashing.add({}); - if (result.ready && !result.columns.empty()) - writeBlock(header.cloneWithColumns(std::move(result.columns))); + if (columns_buffer.size() != 0) + writeBlock(header.cloneWithColumns(columns_buffer.releaseColumns())); if (with_final_mark && data_written) { - writeIntBinary(0ULL, stream->marks); for (size_t i = 0; i < columns_list.size(); ++i) { writeIntBinary(stream->plain_hashing.count(), stream->marks); writeIntBinary(stream->compressed.offset(), stream->marks); } + writeIntBinary(0ULL, stream->marks); } size_t marks_size = stream->marks.count(); @@ -165,4 +173,30 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: stream.reset(); } +void MergeTreeDataPartWriterCompact::ColumnsBuffer::add(MutableColumns && columns) +{ + if (accumulated_columns.empty()) + accumulated_columns = std::move(columns); + else + { + for (size_t i = 0; i < columns.size(); ++i) + accumulated_columns[i]->insertRangeFrom(*columns[i], 0, columns[i]->size()); + } +} + +Columns MergeTreeDataPartWriterCompact::ColumnsBuffer::releaseColumns() +{ + Columns res(std::make_move_iterator(accumulated_columns.begin()), + std::make_move_iterator(accumulated_columns.end())); + accumulated_columns.clear(); + return res; +} + +size_t MergeTreeDataPartWriterCompact::ColumnsBuffer::size() const +{ + if (accumulated_columns.empty()) + return 0; + return accumulated_columns.at(0)->size(); +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 8037b876884..f2e9503c8bb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -24,17 +24,32 @@ public: private: /// Write single granule of one column (rows between 2 marks) - size_t writeColumnSingleGranule( + void writeColumnSingleGranule( const ColumnWithTypeAndName & column, size_t from_row, - size_t number_of_rows); + size_t number_of_rows) const; void writeBlock(const Block & block); ColumnStreamPtr stream; - SquashingTransform squashing; Block header; + + /** Simplified SquashingTransform. The original one isn't suitable in this case + * as it can return smaller block from buffer without merging it with larger block if last is enough size. + * But in compact parts we should guarantee, that written block is larger or equals than index_granularity. + */ + class ColumnsBuffer + { + public: + void add(MutableColumns && columns); + size_t size() const; + Columns releaseColumns(); + private: + MutableColumns accumulated_columns; + }; + + ColumnsBuffer columns_buffer; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 4a183710ab9..751681df6fb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -168,8 +168,8 @@ void MergeTreeReaderCompact::initMarksLoader() while (!buffer.eof()) { - buffer.seek(sizeof(size_t), SEEK_CUR); buffer.readStrict(reinterpret_cast(res->data() + i * columns_num), sizeof(MarkInCompressedFile) * columns_num); + buffer.seek(sizeof(size_t), SEEK_CUR); ++i; } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index f3541d5a351..f7b20bee00d 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -184,7 +184,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm if (!rows) return; - std::set skip_indexes_column_names_set; + std::unordered_set skip_indexes_column_names_set; for (const auto & index : storage.skip_indices) std::copy(index->columns.cbegin(), index->columns.cend(), std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); @@ -199,8 +199,6 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm writer->next(); rows_count += rows; - - // index_offset = new_index_offset; } } From ccb15e63322c4e05e368489e9fe471fe1ea14e10 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Sat, 28 Dec 2019 00:32:55 +0300 Subject: [PATCH 0092/2007] better granularity computing --- dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp | 4 ++-- .../src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp | 2 +- dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index 60ce5b6b947..c369430b954 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -130,7 +130,7 @@ static void fillIndexGranularityImpl( for (current_row = index_offset; current_row < rows_in_block; current_row += index_granularity_for_block) { size_t rows_left_in_block = rows_in_block - current_row; - + /// FIXME need comment if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block && (rows_in_block >= index_granularity_for_block || index_offset != 0)) @@ -160,7 +160,7 @@ void IMergeTreeDataPartWriter::fillIndexGranularity(const Block & block) index_offset, index_granularity, settings.can_use_adaptive_granularity, - need_finish_last_granule && index_offset != 0); + need_finish_last_granule); } void IMergeTreeDataPartWriter::initPrimaryIndex() diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 7896c0e1c61..d157d19baa4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -75,7 +75,7 @@ void MergeTreeDataPartWriterCompact::write( columns_buffer.add(result_block.mutateColumns()); size_t last_mark_rows = index_granularity.getLastMarkRows(); size_t rows_in_buffer = columns_buffer.size(); - + if (rows_in_buffer < last_mark_rows) { /// FIXME need comment diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index f2e9503c8bb..8cefde6cac9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -35,7 +35,7 @@ private: Block header; - /** Simplified SquashingTransform. The original one isn't suitable in this case + /** Simplified SquashingTransform. The original one isn't suitable in this case * as it can return smaller block from buffer without merging it with larger block if last is enough size. * But in compact parts we should guarantee, that written block is larger or equals than index_granularity. */ From 13643d825ba3192323d337bbda2f4c71fba759b9 Mon Sep 17 00:00:00 2001 From: Aleksei Levushkin Date: Sun, 29 Dec 2019 14:40:55 +0300 Subject: [PATCH 0093/2007] added PROTOBUF_GENERATE_GRPC_CPP for generating grpc proto --- CMakeLists.txt | 1 + cmake/find/grpc.cmake | 57 +++++++++++++++++++++++++++++++ contrib/grpc-cmake/CMakeLists.txt | 3 +- 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 cmake/find/grpc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 986096ba9e8..229b83c947d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -323,6 +323,7 @@ endif() include (cmake/find/libxml2.cmake) include (cmake/find/brotli.cmake) include (cmake/find/protobuf.cmake) +include (cmake/find/grpc.cmake) include (cmake/find/pdqsort.cmake) include (cmake/find/hdfs3.cmake) # uses protobuf include (cmake/find/consistent-hashing.cmake) diff --git a/cmake/find/grpc.cmake b/cmake/find/grpc.cmake new file mode 100644 index 00000000000..e19cac5cf5f --- /dev/null +++ b/cmake/find/grpc.cmake @@ -0,0 +1,57 @@ +set(_PROTOBUF_PROTOC $) +set(_GRPC_CPP_PLUGIN_EXECUTABLE $) + +function(PROTOBUF_GENERATE_GRPC_CPP SRCS HDRS) + if(NOT ARGN) + message(SEND_ERROR "Error: PROTOBUF_GENERATE_GRPC_CPP() called without any proto files") + return() + endif() + + if(PROTOBUF_GENERATE_CPP_APPEND_PATH) + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(ABS_PATH ${ABS_FIL} PATH) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + else() + set(_protobuf_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + endif() + + if(DEFINED PROTOBUF_IMPORT_DIRS) + foreach(DIR ${Protobuf_IMPORT_DIRS}) + get_filename_component(ABS_PATH ${DIR} ABSOLUTE) + list(FIND _protobuf_include_path ${ABS_PATH} _contains_already) + if(${_contains_already} EQUAL -1) + list(APPEND _protobuf_include_path -I ${ABS_PATH}) + endif() + endforeach() + endif() + + set(${SRCS}) + set(${HDRS}) + foreach(FIL ${ARGN}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h") + + add_custom_command( + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.cc" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.grpc.pb.h" + COMMAND ${_PROTOBUF_PROTOC} + ARGS --grpc_out=${CMAKE_CURRENT_BINARY_DIR} + --plugin=protoc-gen-grpc=${_GRPC_CPP_PLUGIN_EXECUTABLE} + ${_protobuf_include_path} ${ABS_FIL} + DEPENDS ${ABS_FIL} + COMMENT "Running gRPC C++ protocol buffer compiler on ${FIL}" + VERBATIM) + endforeach() + + set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) + set(${SRCS} ${${SRCS}} PARENT_SCOPE) + set(${HDRS} ${${HDRS}} PARENT_SCOPE) +endfunction() diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index 85f3e6c6737..cc1c2b52b51 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -1479,4 +1479,5 @@ target_link_libraries(grpc_cpp_plugin ${_gRPC_PROTOBUF_LIBRARIES} ${_gRPC_ALLTARGETS_LIBRARIES} grpc_plugin_support -) \ No newline at end of file +) + From 2495849cd8efeb379f7907a29f42f8626e27db33 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 9 Jan 2020 18:46:32 +0300 Subject: [PATCH 0094/2007] fix reading of nested columns in compact format --- .../Storages/MergeTree/IMergeTreeReader.cpp | 1 - .../MergeTree/MergeTreeReaderCompact.cpp | 56 +++++++++++++++++-- .../MergeTree/MergeTreeReaderCompact.h | 6 +- 3 files changed, 55 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp index 51ac7486eb8..d1a366ee809 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -16,7 +16,6 @@ namespace { using OffsetColumns = std::map; } - namespace ErrorCodes { extern const int LOGICAL_ERROR; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 751681df6fb..4a0ec018fe4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include #include namespace DB @@ -69,10 +71,11 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, auto name_and_type = columns.begin(); for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) { - if (!column_positions[pos]) + auto & [name, type] = *name_and_type; + + if (!column_positions[pos] && !typeid_cast(type.get())) continue; - auto & [name, type] = *name_and_type; bool append = res_columns[pos] != nullptr; if (!append) res_columns[pos] = name_and_type->type->createColumn(); @@ -83,9 +86,15 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, try { size_t column_size_before_reading = column->size(); - readData(*column, *type, from_mark, *column_positions[pos], rows_to_read); - size_t read_rows_in_column = column->size() - column_size_before_reading; + // If nested column is missing in current part, we need to read offsets anyway. + auto * array_column = typeid_cast(column.get()); + if (array_column && !column_positions[pos]) + readOffsets(name, *array_column, *type, from_mark, rows_to_read); + else + readData(*column, *type, from_mark, *column_positions[pos], rows_to_read); + + size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column < rows_to_read) throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) + ". Rows expected: " + toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA); @@ -118,16 +127,51 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, return read_rows; } +void MergeTreeReaderCompact::readOffsets( + String name, ColumnArray & column, + const IDataType & type, size_t from_mark, size_t rows_to_read) +{ + std::optional column_position; + String table_name = Nested::extractTableName(name); + auto name_and_type = data_part->columns.begin(); + for (size_t pos = 0; pos < data_part->columns.size(); ++pos, ++name_and_type) + { + const auto & [name, type] = *name_and_type; + auto current_position = data_part->getColumnPosition(name); + + if (typeid_cast(type.get()) + && current_position + && Nested::extractTableName(name) == table_name) + { + column_position.emplace(*current_position); + break; + } + } + + if (!column_position) + return; + + readData(column, type, from_mark, *column_position, rows_to_read, true); +} + void MergeTreeReaderCompact::readData( IColumn & column, const IDataType & type, - size_t from_mark, size_t column_position, size_t rows_to_read) + size_t from_mark, size_t column_position, size_t rows_to_read, bool only_offsets) { if (!isContinuousReading(from_mark, column_position)) seekToMark(from_mark, column_position); + auto buffer_getter = [&, this](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * + { + if (only_offsets && (substream_path.size() != 1 || substream_path[0].type != IDataType::Substream::ArraySizes)) + return nullptr; + + return data_buffer; + }; + IDataType::DeserializeBinaryBulkSettings deserialize_settings; - deserialize_settings.getter = [&](IDataType::SubstreamPath) -> ReadBuffer * { return data_buffer; }; + deserialize_settings.getter = buffer_getter; // deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; deserialize_settings.position_independent_encoding = true; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 9aef896c272..1e189d6c2e2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -2,6 +2,7 @@ #include #include +#include #include @@ -46,7 +47,10 @@ private: void seekToMark(size_t row_index, size_t column_index); void readData(IColumn & column, const IDataType & type, - size_t from_mark, size_t column_position, size_t rows_to_read); + size_t from_mark, size_t column_position, size_t rows_to_read, bool only_offsets = false); + + void readOffsets(String name, ColumnArray & column, const IDataType & type, + size_t from_mark, size_t rows_to_read); /// Columns that are read. From 97521800f049c78ed52a31a39ac033cbfb5ec89a Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 9 Jan 2020 20:06:34 +0300 Subject: [PATCH 0095/2007] fix reading of nested columns in compact format --- .../MergeTree/MergeTreeReaderCompact.cpp | 59 +++++++++---------- .../MergeTree/MergeTreeReaderCompact.h | 7 ++- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 4a0ec018fe4..fac32024659 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -49,9 +49,24 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr data_buffer = non_cached_buffer.get(); } - column_positions.reserve(columns.size()); - for (const auto & column : columns) - column_positions.push_back(data_part->getColumnPosition(column.name)); + size_t columns_num = columns.size(); + + column_positions.resize(columns_num); + read_only_offsets.resize(columns_num); + auto name_and_type = columns.begin(); + for (size_t i = 0; i < columns_num; ++i, ++name_and_type) + { + const auto & [name, type] = *name_and_type; + auto position = data_part->getColumnPosition(name); + if (!position && typeid_cast(type.get())) + { + position = findColumnForOffsets(name); + read_only_offsets[i] = (position != std::nullopt); + } + + column_positions[i] = std::move(position); + } + } size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) @@ -73,7 +88,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, { auto & [name, type] = *name_and_type; - if (!column_positions[pos] && !typeid_cast(type.get())) + if (!column_positions[pos]) continue; bool append = res_columns[pos] != nullptr; @@ -87,12 +102,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, { size_t column_size_before_reading = column->size(); - // If nested column is missing in current part, we need to read offsets anyway. - auto * array_column = typeid_cast(column.get()); - if (array_column && !column_positions[pos]) - readOffsets(name, *array_column, *type, from_mark, rows_to_read); - else - readData(*column, *type, from_mark, *column_positions[pos], rows_to_read); + readData(*column, *type, from_mark, *column_positions[pos], rows_to_read, read_only_offsets[pos]); size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column < rows_to_read) @@ -127,31 +137,20 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, return read_rows; } -void MergeTreeReaderCompact::readOffsets( - String name, ColumnArray & column, - const IDataType & type, size_t from_mark, size_t rows_to_read) +MergeTreeReaderCompact::ColumnPosition MergeTreeReaderCompact::findColumnForOffsets(const String & column_name) { - std::optional column_position; - String table_name = Nested::extractTableName(name); - auto name_and_type = data_part->columns.begin(); - for (size_t pos = 0; pos < data_part->columns.size(); ++pos, ++name_and_type) + String table_name = Nested::extractTableName(column_name); + for (const auto & part_column : data_part->columns) { - const auto & [name, type] = *name_and_type; - auto current_position = data_part->getColumnPosition(name); - - if (typeid_cast(type.get()) - && current_position - && Nested::extractTableName(name) == table_name) + if (typeid_cast(part_column.type.get())) { - column_position.emplace(*current_position); - break; + auto position = data_part->getColumnPosition(part_column.name); + if (position && Nested::extractTableName(part_column.name) == table_name) + return position; } } - if (!column_position) - return; - - readData(column, type, from_mark, *column_position, rows_to_read, true); + return {}; } @@ -162,7 +161,7 @@ void MergeTreeReaderCompact::readData( if (!isContinuousReading(from_mark, column_position)) seekToMark(from_mark, column_position); - auto buffer_getter = [&, this](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * + auto buffer_getter = [&](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * { if (only_offsets && (substream_path.size() != 1 || substream_path[0].type != IDataType::Substream::ArraySizes)) return nullptr; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 1e189d6c2e2..2d78eacb16b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -38,7 +38,9 @@ private: MergeTreeMarksLoader marks_loader; - std::vector> column_positions; + using ColumnPosition = std::optional; + std::vector column_positions; + std::vector read_only_offsets; size_t next_mark = 0; std::optional> last_read_granule; @@ -49,8 +51,7 @@ private: void readData(IColumn & column, const IDataType & type, size_t from_mark, size_t column_position, size_t rows_to_read, bool only_offsets = false); - void readOffsets(String name, ColumnArray & column, const IDataType & type, - size_t from_mark, size_t rows_to_read); + ColumnPosition findColumnForOffsets(const String & column_name); /// Columns that are read. From 6f09b5fc28ad2f79ab9dfc779569842e55791b73 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 9 Jan 2020 20:27:44 +0300 Subject: [PATCH 0096/2007] fix reading of nested columns in compact format --- dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index fac32024659..f6009a68200 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -58,6 +58,9 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr { const auto & [name, type] = *name_and_type; auto position = data_part->getColumnPosition(name); + + /// If array of Nested column is missing in part, + /// we have to read it's offsets if they exists. if (!position && typeid_cast(type.get())) { position = findColumnForOffsets(name); @@ -178,7 +181,11 @@ void MergeTreeReaderCompact::readData( type.deserializeBinaryBulkStatePrefix(deserialize_settings, state); type.deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, state); - last_read_granule.emplace(from_mark, column_position); + /// The buffer is left in inconsistent state after reading single offsets + if (only_offsets) + last_read_granule.reset(); + else + last_read_granule.emplace(from_mark, column_position); } From bae3aa3c6e07e231828066617e8d8b8202c6a131 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 13 Jan 2020 17:53:32 +0300 Subject: [PATCH 0097/2007] simplify data part checking --- .../Storages/MergeTree/IMergeTreeDataPart.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 4 +- .../MergeTree/MergeTreeDataPartCompact.h | 1 + .../ReplicatedMergeTreePartCheckThread.cpp | 2 - dbms/src/Storages/MergeTree/checkDataPart.cpp | 449 +++--------------- dbms/src/Storages/MergeTree/checkDataPart.h | 19 +- dbms/src/Storages/StorageMergeTree.cpp | 4 +- 8 files changed, 81 insertions(+), 402 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index dba2bdec882..f569ca75923 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -308,7 +308,7 @@ public: void makeCloneOnDiskDetached(const ReservationPtr & reservation) const; /// Checks that .bin and .mrk files exist - virtual bool hasColumnFiles(const String & /* column */, const IDataType & /* type */ ) const { return true; } + virtual bool hasColumnFiles(const String & /* column */, const IDataType & /* type */) const{ return false; } static UInt64 calculateTotalSizeOnDisk(const String & from); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 6aa7b4f168b..4f7c11130d7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -2714,7 +2714,7 @@ void MergeTreeData::loadPartAndFixMetadata(MutableDataPartPtr part) /// Check the data while we are at it. if (part->checksums.empty()) { - part->checksums = checkDataPart(part, false, primary_key_data_types, skip_indices); + part->checksums = checkDataPart(part, false); { WriteBufferFromFile out(full_part_path + "checksums.txt.tmp", 4096); part->checksums.write(out); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 09d87016911..754433a92f7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -196,7 +196,7 @@ bool MergeTreeDataPartCompact::hasColumnFiles(const String & column_name, const if (!getColumnPosition(column_name)) return false; - auto bin_checksum = checksums.files.find(String(DATA_FILE_NAME) + DATA_FILE_EXTENSION); + auto bin_checksum = checksums.files.find(DATA_FILE_NAME_WITH_EXTENSION); auto mrk_checksum = checksums.files.find(DATA_FILE_NAME + index_granularity_info.marks_file_extension); return (bin_checksum != checksums.files.end() && mrk_checksum != checksums.files.end()); @@ -246,7 +246,7 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( analysis_result.expression->add(ExpressionAction::project(projection)); String data_temp_name = String(DATA_FILE_NAME) + "_converting"; - rename_map[data_temp_name + DATA_FILE_EXTENSION] = String(DATA_FILE_NAME) + DATA_FILE_EXTENSION; + rename_map[data_temp_name + DATA_FILE_EXTENSION] = DATA_FILE_NAME_WITH_EXTENSION; rename_map[data_temp_name + part_mrk_file_extension] = DATA_FILE_NAME + part_mrk_file_extension; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 4afed5c8785..d03ab472513 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -36,6 +36,7 @@ public: static constexpr auto DATA_FILE_NAME = "data"; static constexpr auto DATA_FILE_EXTENSION = ".bin"; + static constexpr auto DATA_FILE_NAME_WITH_EXTENSION = "data.bin"; MergeTreeDataPartCompact( const MergeTreeData & storage_, diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index abb7e8e88ce..84fe62013bd 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -236,8 +236,6 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na checkDataPart( part, true, - storage.primary_key_data_types, - storage.skip_indices, [this] { return need_stop.load(); }); if (need_stop) diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index a2d6a836d6f..0317e904a09 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -23,172 +24,20 @@ namespace DB namespace ErrorCodes { extern const int CORRUPTED_DATA; - extern const int LOGICAL_ERROR; - extern const int INCORRECT_MARK; - extern const int EMPTY_LIST_OF_COLUMNS_PASSED; } -namespace -{ - -/** To read and checksum single stream (a pair of .bin, .mrk files) for a single column or secondary index. - */ -class Stream -{ -public: - String base_name; - String bin_file_extension; - String mrk_file_extension; - String bin_file_path; - String mrk_file_path; -private: - const MergeTreeIndexGranularity & index_granularity; - ReadBufferFromFile file_buf; - HashingReadBuffer compressed_hashing_buf; - CompressedReadBuffer uncompressing_buf; - size_t mark_position = 0; -public: - HashingReadBuffer uncompressed_hashing_buf; - -private: - ReadBufferFromFile mrk_file_buf; - - std::pair readMarkFromFile() - { - size_t mrk_rows; - MarkInCompressedFile mrk_mark; - readIntBinary(mrk_mark.offset_in_compressed_file, mrk_hashing_buf); - readIntBinary(mrk_mark.offset_in_decompressed_block, mrk_hashing_buf); - if (mrk_file_extension == ".mrk2") - readIntBinary(mrk_rows, mrk_hashing_buf); - else - mrk_rows = index_granularity.getMarkRows(mark_position); - - return {mrk_mark, mrk_rows}; - } -public: - HashingReadBuffer mrk_hashing_buf; - - Stream( - const String & path, - const String & base_name_, - const String & bin_file_extension_, - const String & mrk_file_extension_, - const MergeTreeIndexGranularity & index_granularity_) - : base_name(base_name_) - , bin_file_extension(bin_file_extension_) - , mrk_file_extension(mrk_file_extension_) - , bin_file_path(path + base_name + bin_file_extension) - , mrk_file_path(path + base_name + mrk_file_extension) - , index_granularity(index_granularity_) - , file_buf(bin_file_path) - , compressed_hashing_buf(file_buf) - , uncompressing_buf(compressed_hashing_buf) - , uncompressed_hashing_buf(uncompressing_buf) - , mrk_file_buf(mrk_file_path) - , mrk_hashing_buf(mrk_file_buf) - {} - - void assertMark(bool only_read=false) - { - - auto [mrk_mark, mrk_rows] = readMarkFromFile(); - bool has_alternative_mark = false; - MarkInCompressedFile alternative_data_mark = {}; - MarkInCompressedFile data_mark = {}; - - /// If the mark should be exactly at the border of blocks, we can also use a mark pointing to the end of previous block, - /// and the beginning of next. - if (!uncompressed_hashing_buf.hasPendingData()) - { - /// Get a mark pointing to the end of previous block. - has_alternative_mark = true; - alternative_data_mark.offset_in_compressed_file = compressed_hashing_buf.count() - uncompressing_buf.getSizeCompressed(); - alternative_data_mark.offset_in_decompressed_block = uncompressed_hashing_buf.offset(); - - if (mrk_mark == alternative_data_mark) - { - mark_position++; - return; - } - - uncompressed_hashing_buf.next(); - - /// At the end of file `compressed_hashing_buf.count()` points to the end of the file even before `calling next()`, - /// and the check you just performed does not work correctly. For simplicity, we will not check the last mark. - if (uncompressed_hashing_buf.eof()) - { - mark_position++; - return; - } - } - - data_mark.offset_in_compressed_file = compressed_hashing_buf.count() - uncompressing_buf.getSizeCompressed(); - data_mark.offset_in_decompressed_block = uncompressed_hashing_buf.offset(); - - if (!only_read && (mrk_mark != data_mark || mrk_rows != index_granularity.getMarkRows(mark_position))) - throw Exception("Incorrect mark: " + data_mark.toStringWithRows(index_granularity.getMarkRows(mark_position)) + - (has_alternative_mark ? " or " + alternative_data_mark.toString() : "") + " in data, " + - mrk_mark.toStringWithRows(mrk_rows) + " in " + mrk_file_path + " file", ErrorCodes::INCORRECT_MARK); - - mark_position++; - } - - void assertEnd() - { - if (!uncompressed_hashing_buf.eof()) - throw Exception("EOF expected in " + bin_file_path + " file" - + " at position " - + toString(compressed_hashing_buf.count()) + " (compressed), " - + toString(uncompressed_hashing_buf.count()) + " (uncompressed)", ErrorCodes::CORRUPTED_DATA); - - /// Maybe we have final mark. - if (index_granularity.hasFinalMark()) - { - auto final_mark_rows = readMarkFromFile().second; - if (final_mark_rows != 0) - throw Exception("Incorrect final mark at the end of " + mrk_file_path + " expected 0 rows, got " + toString(final_mark_rows), ErrorCodes::CORRUPTED_DATA); - } - - if (!mrk_hashing_buf.eof()) - throw Exception("EOF expected in " + mrk_file_path + " file" - + " at position " - + toString(mrk_hashing_buf.count()), ErrorCodes::CORRUPTED_DATA); - } - - void saveChecksums(MergeTreeData::DataPart::Checksums & checksums) - { - checksums.files[base_name + bin_file_extension] = MergeTreeData::DataPart::Checksums::Checksum( - compressed_hashing_buf.count(), compressed_hashing_buf.getHash(), - uncompressed_hashing_buf.count(), uncompressed_hashing_buf.getHash()); - - checksums.files[base_name + mrk_file_extension] = MergeTreeData::DataPart::Checksums::Checksum( - mrk_hashing_buf.count(), mrk_hashing_buf.getHash()); - } -}; - -} - - -MergeTreeData::DataPart::Checksums checkDataPart( +IMergeTreeDataPart::Checksums checkDataPart( const String & full_path, - const MergeTreeIndexGranularity & adaptive_index_granularity, - const String & mrk_file_extension, + const NamesAndTypesList & columns_list, + const MergeTreeDataPartType & part_type, bool require_checksums, - const DataTypes & primary_key_data_types, - const MergeTreeIndices & indices, std::function is_cancelled) { - Logger * log = &Logger::get("checkDataPart"); - /** Responsibility: * - read list of columns from columns.txt; * - read checksums if exist; - * - read (and validate checksum) of primary.idx; obtain number of marks; - * - read data files and marks for each stream of each column; calculate and validate checksums; - * - check that there are the same number of rows in each column; - * - check that all marks files have the same size; + * - validate list of columns and checksums */ CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedChecks}; @@ -197,16 +46,74 @@ MergeTreeData::DataPart::Checksums checkDataPart( if (!path.empty() && path.back() != '/') path += "/"; - NamesAndTypesList columns; + NamesAndTypesList columns_txt; { ReadBufferFromFile buf(path + "columns.txt"); - columns.readText(buf); + columns_txt.readText(buf); assertEOF(buf); } + if (columns_txt != columns_list) + throw Exception("Columns doesn't match in part " + path + + ". Expected: " + columns_list.toString() + + ". Found: " + columns_txt.toString(), ErrorCodes::CORRUPTED_DATA); + + /// Real checksums based on contents of data. Must correspond to checksums.txt. If not - it means the data is broken. + IMergeTreeDataPart::Checksums checksums_data; + + auto checksum_compressed_file = [](const String & file_path) + { + ReadBufferFromFile file_buf(file_path); + HashingReadBuffer compressed_hashing_buf(file_buf); + CompressedReadBuffer uncompressing_buf(compressed_hashing_buf); + HashingReadBuffer uncompressed_hashing_buf(uncompressing_buf); + + uncompressed_hashing_buf.tryIgnore(std::numeric_limits::max()); + return IMergeTreeDataPart::Checksums::Checksum + { + compressed_hashing_buf.count(), compressed_hashing_buf.getHash(), + uncompressed_hashing_buf.count(), uncompressed_hashing_buf.getHash() + }; + }; + + if (part_type == MergeTreeDataPartType::COMPACT) + { + const auto & file_name = MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; + checksums_data.files[file_name] = checksum_compressed_file(path + file_name); + } + else if (part_type == MergeTreeDataPartType::WIDE) + { + for (const auto & column : columns_list) + { + column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) + { + String file_name = IDataType::getFileNameForStream(column.name, substream_path) + ".bin"; + checksums_data.files[file_name] = checksum_compressed_file(path + file_name); + }, {}); + } + } + else + { + throw Exception("Unknown type in part " + path, ErrorCodes::UNKNOWN_PART_TYPE); + } + + Poco::DirectoryIterator dir_end; + for (Poco::DirectoryIterator dir_it(path); dir_it != dir_end; ++dir_it) + { + const String & file_name = dir_it.name(); + auto checksum_it = checksums_data.files.find(file_name); + if (checksum_it == checksums_data.files.end() && file_name != "checksums.txt" && file_name != "columns.txt") + { + ReadBufferFromFile file_buf(dir_it->path()); + HashingReadBuffer hashing_buf(file_buf); + hashing_buf.tryIgnore(std::numeric_limits::max()); + checksums_data.files[file_name] = IMergeTreeDataPart::Checksums::Checksum(hashing_buf.count(), hashing_buf.getHash()); + } + } + /// Checksums from file checksums.txt. May be absent. If present, they are subsequently compared with the actual data checksums. - MergeTreeData::DataPart::Checksums checksums_txt; + IMergeTreeDataPart::Checksums checksums_txt; if (require_checksums || Poco::File(path + "checksums.txt").exists()) { @@ -215,244 +122,26 @@ MergeTreeData::DataPart::Checksums checkDataPart( assertEOF(buf); } - /// Real checksums based on contents of data. Must correspond to checksums.txt. If not - it means the data is broken. - MergeTreeData::DataPart::Checksums checksums_data; - - size_t marks_in_primary_key = 0; - if (!primary_key_data_types.empty()) - { - ReadBufferFromFile file_buf(path + "primary.idx"); - HashingReadBuffer hashing_buf(file_buf); - - size_t key_size = primary_key_data_types.size(); - MutableColumns tmp_columns(key_size); - - for (size_t j = 0; j < key_size; ++j) - tmp_columns[j] = primary_key_data_types[j]->createColumn(); - - while (!hashing_buf.eof()) - { - if (is_cancelled()) - return {}; - - ++marks_in_primary_key; - for (size_t j = 0; j < key_size; ++j) - primary_key_data_types[j]->deserializeBinary(*tmp_columns[j].get(), hashing_buf); - } - - size_t primary_idx_size = hashing_buf.count(); - - checksums_data.files["primary.idx"] = MergeTreeData::DataPart::Checksums::Checksum(primary_idx_size, hashing_buf.getHash()); - } - - /// Optional files count.txt, partition.dat, minmax_*.idx, ttl.txt. Just calculate checksums for existing files. - Poco::DirectoryIterator dir_end; - for (Poco::DirectoryIterator dir_it(path); dir_it != dir_end; ++dir_it) - { - const String & file_name = dir_it.name(); - if (file_name == "count.txt" - || file_name == "partition.dat" - || (startsWith(file_name, "minmax_") && endsWith(file_name, ".idx")) - || file_name == "ttl.txt") - { - ReadBufferFromFile file_buf(dir_it->path()); - HashingReadBuffer hashing_buf(file_buf); - hashing_buf.tryIgnore(std::numeric_limits::max()); - checksums_data.files[file_name] = MergeTreeData::DataPart::Checksums::Checksum(hashing_buf.count(), hashing_buf.getHash()); - } - } - if (is_cancelled()) return {}; - /// If count.txt file exists, use it as source of truth for number of rows. Otherwise just check that all columns have equal amount of rows. - std::optional rows; - - if (Poco::File(path + "count.txt").exists()) - { - ReadBufferFromFile buf(path + "count.txt"); - size_t count = 0; - readText(count, buf); - assertEOF(buf); - rows = count; - } - - /// Read and check skip indices. - for (const auto & index : indices) - { - Stream stream(path, index->getFileName(), ".idx", mrk_file_extension, adaptive_index_granularity); - size_t mark_num = 0; - - while (!stream.uncompressed_hashing_buf.eof()) - { - if (stream.mrk_hashing_buf.eof()) - throw Exception("Unexpected end of mrk file while reading index " + index->name, - ErrorCodes::CORRUPTED_DATA); - try - { - stream.assertMark(); - } - catch (Exception & e) - { - e.addMessage("Cannot read mark " + toString(mark_num) - + " in file " + stream.mrk_file_path - + ", mrk file offset: " + toString(stream.mrk_hashing_buf.count())); - throw; - } - try - { - index->createIndexGranule()->deserializeBinary(stream.uncompressed_hashing_buf); - } - catch (Exception & e) - { - e.addMessage("Cannot read granule " + toString(mark_num) - + " in file " + stream.bin_file_path - + ", mrk file offset: " + toString(stream.mrk_hashing_buf.count())); - throw; - } - ++mark_num; - if (is_cancelled()) - return {}; - } - - stream.assertEnd(); - stream.saveChecksums(checksums_data); - } - - /// Read all columns, calculate checksums and validate marks. - for (const NameAndTypePair & name_type : columns) - { - LOG_DEBUG(log, "Checking column " + name_type.name + " in " + path); - - std::map streams; - size_t column_size = 0; - size_t mark_num = 0; - - IDataType::DeserializeBinaryBulkStatePtr state; - IDataType::DeserializeBinaryBulkSettings settings; - settings.getter = [&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); - auto & stream = streams.try_emplace(file_name, path, file_name, ".bin", mrk_file_extension, adaptive_index_granularity).first->second; - return &stream.uncompressed_hashing_buf; - }; - - /// Prefixes have to be read before data because first mark points after prefix - name_type.type->deserializeBinaryBulkStatePrefix(settings, state); - - while (true) - { - - /// Check that mark points to current position in file. - bool marks_eof = false; - name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); - - auto & stream = streams.try_emplace(file_name, path, file_name, ".bin", mrk_file_extension, adaptive_index_granularity).first->second; - try - { - /// LowCardinality dictionary column is not read monotonically, so marks maybe inconsistent with - /// offset position in file. So we just read data and marks file, but doesn't check marks equality. - bool only_read = !substream_path.empty() && substream_path.back().type == IDataType::Substream::DictionaryKeys; - if (!stream.mrk_hashing_buf.eof()) - stream.assertMark(only_read); - else - marks_eof = true; - } - catch (Exception & e) - { - e.addMessage("Cannot read mark " + toString(mark_num) + " at row " + toString(column_size) - + " in file " + stream.mrk_file_path - + ", mrk file offset: " + toString(stream.mrk_hashing_buf.count())); - throw; - } - }, settings.path); - - size_t rows_after_mark = adaptive_index_granularity.getMarkRows(mark_num); - ++mark_num; - - /// Read index_granularity rows from column. - /// NOTE Shared array sizes of Nested columns are read more than once. That's Ok. - - MutableColumnPtr tmp_column = name_type.type->createColumn(); - name_type.type->deserializeBinaryBulkWithMultipleStreams(*tmp_column, rows_after_mark, settings, state); - - size_t read_size = tmp_column->size(); - column_size += read_size; - - /// We already checked all marks except final (it will be checked in assertEnd()). - if (mark_num == adaptive_index_granularity.getMarksCountWithoutFinal()) - break; - else if (marks_eof) - throw Exception("Unexpected end of mrk file while reading column " + name_type.name, ErrorCodes::CORRUPTED_DATA); - - if (is_cancelled()) - return {}; - } - - /// Check that number of rows are equal in each column. - if (!rows) - rows = column_size; - else if (*rows != column_size) - throw Exception{"Unexpected number of rows in column " - + name_type.name + " (" + toString(column_size) + ", expected: " + toString(*rows) + ")", - ErrorCodes::SIZES_OF_COLUMNS_DOESNT_MATCH}; - - /// Save checksums for column. - name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - { - String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); - auto stream_it = streams.find(file_name); - if (stream_it == streams.end()) - throw Exception("Logical error: cannot find stream " + file_name, ErrorCodes::LOGICAL_ERROR); - - stream_it->second.assertEnd(); - stream_it->second.saveChecksums(checksums_data); - }, {}); - - if (is_cancelled()) - return {}; - } - - if (!rows) - throw Exception("No columns in data part", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); - - if (!primary_key_data_types.empty()) - { - size_t expected_marks = adaptive_index_granularity.getMarksCount(); - if (expected_marks != marks_in_primary_key) - { - throw Exception("Size of primary key doesn't match expected number of marks." - " Number of rows in columns: " + toString(*rows) - + ", expected number of marks: " + toString(expected_marks) - + ", size of primary key: " + toString(marks_in_primary_key), - ErrorCodes::CORRUPTED_DATA); - } - } - if (require_checksums || !checksums_txt.files.empty()) checksums_txt.checkEqual(checksums_data, true); return checksums_data; } -MergeTreeData::DataPart::Checksums checkDataPart( +IMergeTreeDataPart::Checksums checkDataPart( MergeTreeData::DataPartPtr data_part, bool require_checksums, - const DataTypes & primary_key_data_types, - const MergeTreeIndices & indices, std::function is_cancelled) { return checkDataPart( data_part->getFullPath(), - data_part->index_granularity, - data_part->index_granularity_info.marks_file_extension, + data_part->columns, + data_part->getType(), require_checksums, - primary_key_data_types, - indices, is_cancelled); } - } diff --git a/dbms/src/Storages/MergeTree/checkDataPart.h b/dbms/src/Storages/MergeTree/checkDataPart.h index 936eebd17b2..2f263065b5d 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.h +++ b/dbms/src/Storages/MergeTree/checkDataPart.h @@ -6,25 +6,16 @@ namespace DB { -/** Completely checks the part data - * - Calculates checksums and compares them with checksums.txt. - * - For arrays and strings, checks the correspondence of the size and amount of data. - * - Checks the correctness of marks. - * Throws an exception if the part is corrupted or if the check fails (TODO: you can try to separate these cases). - */ -MergeTreeData::DataPart::Checksums checkDataPart( +/// Calculates checksums and compares them with checksums.txt. +IMergeTreeDataPart::Checksums checkDataPart( MergeTreeData::DataPartPtr data_part, bool require_checksums, - const DataTypes & primary_key_data_types, - const MergeTreeIndices & indices = {}, /// Check skip indices std::function is_cancelled = []{ return false; }); -MergeTreeData::DataPart::Checksums checkDataPart( +IMergeTreeDataPart::Checksums checkDataPart( const String & full_path, - const MergeTreeIndexGranularity & index_granularity, - const String & marks_file_extension, + const NamesAndTypesList & columns_list, + const MergeTreeDataPartType & part_type, bool require_checksums, - const DataTypes & primary_key_data_types, - const MergeTreeIndices & indices = {}, /// Check skip indices std::function is_cancelled = []{ return false; }); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 01f197c209d..6f4f772fb43 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -1233,7 +1233,7 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, const Context & c { try { - auto calculated_checksums = checkDataPart(part, false, primary_key_data_types, skip_indices); + auto calculated_checksums = checkDataPart(part, false); calculated_checksums.checkEqual(part->checksums, true); WriteBufferFromFile out(tmp_checksums_path, 4096); part->checksums.write(out); @@ -1254,7 +1254,7 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, const Context & c { try { - checkDataPart(part, true, primary_key_data_types, skip_indices); + checkDataPart(part, true); results.emplace_back(part->name, true, ""); } catch (const Exception & ex) From 1011675b11149d1cbe6be31b8238e302e7246740 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 13 Jan 2020 19:28:29 +0300 Subject: [PATCH 0098/2007] avoid errors with compact non-adaptive parts --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 30 ++++++++++++++++++- dbms/src/Storages/MergeTree/MergeTreeData.h | 2 ++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 4f7c11130d7..37b5d07c5b6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -237,6 +237,8 @@ MergeTreeData::MergeTreeData( "MergeTree data format version on disk doesn't support custom partitioning", ErrorCodes::METADATA_MISMATCH); } + + checkCanUsePolymorphicParts(true); } @@ -1567,6 +1569,9 @@ AlterAnalysisResult MergeTreeData::analyzeAlterConversions( MergeTreeDataPartType MergeTreeData::choosePartType(size_t bytes_uncompressed, size_t rows_count) const { + if (!canUseAdaptiveGranularity()) + return MergeTreeDataPartType::WIDE; + const auto settings = getSettings(); if (bytes_uncompressed < settings->min_bytes_for_wide_part || rows_count < settings->min_rows_for_wide_part) return MergeTreeDataPartType::COMPACT; @@ -1584,7 +1589,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart(const String & name, else if (type == MergeTreeDataPartType::WIDE) return std::make_shared(*this, name, part_info, disk, relative_path); else - throw Exception("Unknown part type", ErrorCodes::LOGICAL_ERROR); + throw Exception("Unknown type in part " + relative_path, ErrorCodes::UNKNOWN_PART_TYPE); } MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( @@ -1818,6 +1823,7 @@ void MergeTreeData::changeSettings( copy.applyChanges(new_changes); storage_settings.set(std::make_unique(copy)); settings_ast = new_settings; + checkCanUsePolymorphicParts(false); } } @@ -3808,4 +3814,26 @@ bool MergeTreeData::moveParts(CurrentlyMovingPartsTagger && moving_tagger) return true; } +void MergeTreeData::checkCanUsePolymorphicParts(bool no_throw) +{ + const auto settings = getSettings(); + if (!canUseAdaptiveGranularity() && (settings->min_rows_for_wide_part != 0 || settings->min_bytes_for_wide_part != 0)) + { + std::ostringstream message; + message << "Table can't create parts with adaptive granularity, but settings min_rows_for_wide_part = " + << settings->min_rows_for_wide_part << ", min_bytes_for_wide_part = " << settings->min_bytes_for_wide_part + << ". Parts with non-adaptive granularity can be stored only in Wide (default) format."; + + if (no_throw) + { + message << " Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored further."; + LOG_WARNING(log, message.str()); + } + else + { + throw Exception(message.str(), ErrorCodes::NOT_IMPLEMENTED); + } + } +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 773e7b0f5ef..aef43bb5a2e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -1017,6 +1017,8 @@ private: /// Check selected parts for movements. Used by ALTER ... MOVE queries. CurrentlyMovingPartsTagger checkPartsForMove(const DataPartsVector & parts, SpacePtr space); + + void checkCanUsePolymorphicParts(bool no_throw); }; } From bc59e473e8be7ff4cc34b44c3f188cb6bb66466e Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 13 Jan 2020 19:39:20 +0300 Subject: [PATCH 0099/2007] Some thoughts on non blocking alter --- .../src/Interpreters/MutationsInterpreter.cpp | 4 + dbms/src/Storages/AlterCommands.cpp | 143 +++++- dbms/src/Storages/AlterCommands.h | 11 + dbms/src/Storages/MergeTree/MergeTreeData.cpp | 1 + .../MergeTree/MergeTreeDataMergerMutator.cpp | 18 +- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 18 + .../MergeTree/ReplicatedMergeTreeLogEntry.h | 2 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 20 + .../MergeTree/ReplicatedMergeTreeQueue.h | 2 + dbms/src/Storages/MutationCommands.h | 8 +- .../Storages/StorageReplicatedMergeTree.cpp | 438 ++++++++++-------- .../src/Storages/StorageReplicatedMergeTree.h | 6 + 12 files changed, 461 insertions(+), 210 deletions(-) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 8ff10e92dee..4a585cf424f 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -381,6 +381,10 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) const auto required_columns = syntax_result->requiredSourceColumns(); affected_indices_columns.insert(std::cbegin(required_columns), std::cend(required_columns)); } + else if (command.type == MutationCommand::CAST) + { + stages.back().column_to_updated.emplace(command.column_name, makeASTFunction("CAST", command.column_name, command.type_name)); + } else throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); } diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index fc7bf608b17..58e21caedd6 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -1,24 +1,30 @@ -#include -#include +#include +#include +#include +#include +#include #include +#include #include #include #include -#include -#include #include -#include -#include -#include -#include -#include -#include +#include +#include #include #include -#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include #include -#include + #include @@ -43,6 +49,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ if (command_ast->type == ASTAlterCommand::ADD_COLUMN) { AlterCommand command; + command.ast = command_ast; command.type = AlterCommand::ADD_COLUMN; const auto & ast_col_decl = command_ast->col_decl->as(); @@ -83,6 +90,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; + command.ast = command_ast; command.type = AlterCommand::DROP_COLUMN; command.column_name = getIdentifierName(command_ast->column); command.if_exists = command_ast->if_exists; @@ -91,6 +99,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_COLUMN) { AlterCommand command; + command.ast = command_ast; command.type = AlterCommand::MODIFY_COLUMN; const auto & ast_col_decl = command_ast->col_decl->as(); @@ -126,6 +135,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::COMMENT_COLUMN) { AlterCommand command; + command.ast = command_ast; command.type = COMMENT_COLUMN; command.column_name = getIdentifierName(command_ast->column); const auto & ast_comment = command_ast->comment->as(); @@ -136,6 +146,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_ORDER_BY) { AlterCommand command; + command.ast = command_ast; command.type = AlterCommand::MODIFY_ORDER_BY; command.order_by = command_ast->order_by; return command; @@ -143,6 +154,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::ADD_INDEX) { AlterCommand command; + command.ast = command_ast; command.index_decl = command_ast->index_decl; command.type = AlterCommand::ADD_INDEX; @@ -160,6 +172,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::ADD_CONSTRAINT) { AlterCommand command; + command.ast = command_ast; command.constraint_decl = command_ast->constraint_decl; command.type = AlterCommand::ADD_CONSTRAINT; @@ -177,6 +190,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; + command.ast = command_ast; command.if_exists = command_ast->if_exists; command.type = AlterCommand::DROP_CONSTRAINT; command.constraint_name = command_ast->constraint->as().name; @@ -189,6 +203,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ throw Exception("\"ALTER TABLE table CLEAR INDEX index\" queries are not supported yet. Use \"CLEAR INDEX index IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; + command.ast = command_ast; command.type = AlterCommand::DROP_INDEX; command.index_name = command_ast->index->as().name; command.if_exists = command_ast->if_exists; @@ -198,6 +213,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_TTL) { AlterCommand command; + command.ast = command_ast; command.type = AlterCommand::MODIFY_TTL; command.ttl = command_ast->ttl; return command; @@ -205,6 +221,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_SETTING) { AlterCommand command; + command.ast = command_ast; command.type = AlterCommand::MODIFY_SETTING; command.settings_changes = command_ast->settings_changes->as().changes; return command; @@ -423,6 +440,76 @@ bool AlterCommand::isSettingsAlter() const return type == MODIFY_SETTING; } +namespace +{ + +/// If true, then in order to ALTER the type of the column from the type from to the type to +/// we don't need to rewrite the data, we only need to update metadata and columns.txt in part directories. +/// The function works for Arrays and Nullables of the same structure. +bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) +{ + if (from->getName() == to->getName()) + return true; + + static const std::unordered_multimap ALLOWED_CONVERSIONS = + { + { typeid(DataTypeEnum8), typeid(DataTypeEnum8) }, + { typeid(DataTypeEnum8), typeid(DataTypeInt8) }, + { typeid(DataTypeEnum16), typeid(DataTypeEnum16) }, + { typeid(DataTypeEnum16), typeid(DataTypeInt16) }, + { typeid(DataTypeDateTime), typeid(DataTypeUInt32) }, + { typeid(DataTypeUInt32), typeid(DataTypeDateTime) }, + { typeid(DataTypeDate), typeid(DataTypeUInt16) }, + { typeid(DataTypeUInt16), typeid(DataTypeDate) }, + }; + + while (true) + { + auto it_range = ALLOWED_CONVERSIONS.equal_range(typeid(*from)); + for (auto it = it_range.first; it != it_range.second; ++it) + { + if (it->second == typeid(*to)) + return true; + } + + const auto * arr_from = typeid_cast(from); + const auto * arr_to = typeid_cast(to); + if (arr_from && arr_to) + { + from = arr_from->getNestedType().get(); + to = arr_to->getNestedType().get(); + continue; + } + + const auto * nullable_from = typeid_cast(from); + const auto * nullable_to = typeid_cast(to); + if (nullable_from && nullable_to) + { + from = nullable_from->getNestedType().get(); + to = nullable_to->getNestedType().get(); + continue; + } + + return false; + } +} + +} + + +bool AlterCommand::isRequireMutationStage(const StorageInMemoryMetadata & metadata) const +{ + if (type != MODIFY_COLUMN || data_type == nullptr) + return false; + + for (const auto & column : metadata.columns.getAllPhysical()) + { + if (column.name == column_name && !isMetadataOnlyConversion(column.type, data_type)) + return true; + } + return false; +} + bool AlterCommand::isCommentAlter() const { if (type == COMMENT_COLUMN) @@ -440,6 +527,21 @@ bool AlterCommand::isCommentAlter() const return false; } +std::optional AlterCommand::tryConvertToMutationCommand(const StorageInMemoryMetadata & metadata) const +{ + if (!isRequireMutationStage(metadata)) + return {}; + + MutationCommand result; + + result.type = MutationCommand::Type::CAST; + result.column_name = column_name; + result.data_type = data_type; + result.predicate = nullptr; + result.ast = ast; + return result; +} + String alterTypeToString(const AlterCommand::Type type) { @@ -635,6 +737,12 @@ void AlterCommands::prepare(const StorageInMemoryMetadata & metadata, const Cont command->default_expression = makeASTFunction("CAST", command->default_expression->clone(), std::make_shared(explicit_type->getName())); + + //TODO(alesap) + //command->ast = std::make_shared(); + //command->type = ASTAlterCommand::MODIFY_COLUMN; + //command->col_decl = std::make_shared(); + //command->col_decl->name = column.name; } } else @@ -725,4 +833,15 @@ bool AlterCommands::isCommentAlter() const { return std::all_of(begin(), end(), [](const AlterCommand & c) { return c.isCommentAlter(); }); } + + +MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata) const +{ + MutationCommands result; + for (const auto & alter_cmd : *this) + if (auto mutation_cmd = alter_cmd.tryConvertToMutationCommand(metadata); mutation_cmd) + result.push_back(*mutation_cmd); + return result; +} + } diff --git a/dbms/src/Storages/AlterCommands.h b/dbms/src/Storages/AlterCommands.h index e547752fa09..fbe29800b34 100644 --- a/dbms/src/Storages/AlterCommands.h +++ b/dbms/src/Storages/AlterCommands.h @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -18,6 +19,8 @@ class ASTAlterCommand; /// Adding Nested columns is not expanded to add individual columns. struct AlterCommand { + ASTPtr ast; /// The AST of the whole command + enum Type { ADD_COLUMN, @@ -96,11 +99,15 @@ struct AlterCommand /// in each part on disk (it's not lightweight alter). bool isModifyingData() const; + bool isRequireMutationStage(const StorageInMemoryMetadata & metadata) const; + /// Checks that only settings changed by alter bool isSettingsAlter() const; /// Checks that only comment changed by alter bool isCommentAlter() const; + + std::optional tryConvertToMutationCommand(const StorageInMemoryMetadata & metadata) const; }; /// Return string representation of AlterCommand::Type @@ -136,6 +143,10 @@ public: /// At least one command modify comments. bool isCommentAlter() const; + + MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata) const; }; + +MutationCommands extractMutationCommandsFromAlterCommands(const StorageInMemoryMetadata & metadata, AlterCommands & commands); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index a978c72caad..a9fb732e81d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1784,6 +1784,7 @@ void MergeTreeData::alterDataPart( new_checksums.files[it.second] = add_checksums.files[it.first]; } + /// NOTE(alesap) Don't miss this /// Write the checksums to the temporary file. if (!part->checksums.empty()) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 05db73ce215..3295c863ef9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -945,16 +945,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor context_for_reading.getSettingsRef().max_threads = 1; std::vector commands_for_part; - std::copy_if( - std::cbegin(commands), std::cend(commands), - std::back_inserter(commands_for_part), - [&] (const MutationCommand & command) - { - return command.partition == nullptr || - future_part.parts[0]->info.partition_id == data.getPartitionIDFromQuery( - command.partition, context_for_reading); - }); - + for (const auto & command : commands) + { + if (command.partition == nullptr || future_part.parts[0]->info.partition_id == data.getPartitionIDFromQuery( + command.partition, context_for_reading)) + commands_for_part.emplace_back(command); + } if (!isStorageTouchedByMutations(storage_from_source_part, commands_for_part, context_for_reading)) { @@ -1061,7 +1057,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor indices_recalc_syntax, context).getActions(false); /// We can update only one column, but some skip idx expression may depend on several - /// columns (c1 + c2 * c3). It works because in stream was created with help of + /// columns (c1 + c2 * c3). It works because this stream was created with help of /// MutationsInterpreter which knows about skip indices and stream 'in' already has /// all required columns. /// TODO move this logic to single place. diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 29878cc064e..0599a35905d 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -65,6 +65,12 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const << new_part_name; break; + case FINISH_ALTER: /// Just make local /metadata and /columns consistent with global + out << "alter\n"; + for (const String & s : source_parts) + out << s << '\n'; + out << "finish"; + break; default: throw Exception("Unknown log entry type: " + DB::toString(type), ErrorCodes::LOGICAL_ERROR); } @@ -152,6 +158,18 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) >> new_part_name; source_parts.push_back(source_part); } + else if (type_str == "alter") + { + type = FINISH_ALTER; + while (!in.eof()) + { + String s; + in >> s >> "\n"; + if (s == "finish") + break; + source_parts.push_back(s); + } + } in >> "\n"; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index ca8c9315fa9..7d63c20ce9b 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -37,6 +37,7 @@ struct ReplicatedMergeTreeLogEntryData CLEAR_INDEX, /// Drop specific index from specified partition. REPLACE_RANGE, /// Drop certain range of partitions and replace them by new ones MUTATE_PART, /// Apply one or several mutations to the part. + FINISH_ALTER, /// Apply one or several alter modifications to part }; static String typeToString(Type type) @@ -50,6 +51,7 @@ struct ReplicatedMergeTreeLogEntryData case ReplicatedMergeTreeLogEntryData::CLEAR_INDEX: return "CLEAR_INDEX"; case ReplicatedMergeTreeLogEntryData::REPLACE_RANGE: return "REPLACE_RANGE"; case ReplicatedMergeTreeLogEntryData::MUTATE_PART: return "MUTATE_PART"; + case ReplicatedMergeTreeLogEntryData::FINISH_ALTER: return "FINISH_ALTER"; default: throw Exception("Unknown log entry type: " + DB::toString(type), ErrorCodes::LOGICAL_ERROR); } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 7fd08788704..41e1d408b52 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -557,6 +557,11 @@ static Names getPartNamesToMutate( } +Names getPartNamesToMutate(ReplicatedMergeTreeMutationEntry & entry) const +{ + return getPartNamesToMutate(entry, current_parts); +} + void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) { std::lock_guard lock(update_mutations_mutex); @@ -1001,6 +1006,21 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( } } + if (entry.type == LogEntry::FINISH_ALTER) + { + for (const auto & name : entry.source_parts) + { + if (future_parts.count(name)) + { + String reason = "Not altering storage because part " + name + + " is not ready yet (log entry for that part is being processed)."; + LOG_TRACE(log, reason); + out_postpone_reason = reason; + return false; + } + } + } + return true; } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 4e199068667..6ed2f1889ed 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -339,6 +339,8 @@ public: /// Adds a subscriber SubscriberHandler addSubscriber(SubscriberCallBack && callback); + Names getPartNamesToMutate(ReplicatedMergeTreeMutationEntry & entry) const; + struct Status { UInt32 future_parts; diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index 96ebd30f254..ce84c08e631 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -25,7 +26,8 @@ struct MutationCommand EMPTY, /// Not used. DELETE, UPDATE, - MATERIALIZE_INDEX + MATERIALIZE_INDEX, + CAST /// for ALTER MODIFY column }; Type type = EMPTY; @@ -40,6 +42,10 @@ struct MutationCommand String index_name; ASTPtr partition; + /// For cast + String column_name; + DataTypePtr data_type; + static std::optional parse(ASTAlterCommand * command); }; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 67b9cbd5ca4..0fa52b28ba1 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -977,6 +977,10 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) { do_fetch = !tryExecutePartMutation(entry); } + else if (entry.type == LogEntry::FINISH_ALTER) + { + tryFinishAlter(entry); + } else { throw Exception("Unexpected log entry type: " + toString(static_cast(entry.type)), ErrorCodes::LOGICAL_ERROR); @@ -1152,6 +1156,72 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } +bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree::LogEntry & entry) +{ + auto zookeeper = getZooKeeper(); + + String columns_path = zookeeper_path + "/columns"; + auto columns_znode = zookeeper->get(columns_path); + if (!columns_znode.exists) + throw Exception(columns_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); + int32_t columns_version = columns_znode.stat.version; + + String metadata_path = zookeeper_path + "/metadata"; + auto metadata_znode = zookeeper->get(metadata_path); + if (!metadata_znode.exists) + throw Exception(metadata_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); + int32_t metadata_version = metadata_znode.stat.version; + + const bool changed_columns_version = (columns_version != storage.columns_version); + const bool changed_metadata_version = (metadata_version != storage.metadata_version); + + if (!(changed_columns_version || changed_metadata_version)) + return; + + const String & columns_str = columns_znode.contents; + auto columns_in_zk = ColumnsDescription::parse(columns_str); + + const String & metadata_str = metadata_znode.contents; + auto metadata_in_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); + auto metadata_diff = ReplicatedMergeTreeTableMetadata(storage).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); + + MergeTreeData::DataParts parts; + + /// If metadata nodes have changed, we will update table structure locally. + if (changed_columns_version || changed_metadata_version) + { + LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); + + auto table_lock = storage.lockExclusively(RWLockImpl::NO_QUERY); + + if (columns_in_zk == storage.getColumns() && metadata_diff.empty()) + { + LOG_INFO( + log, + "Metadata nodes changed in ZooKeeper, but their contents didn't change. " + "Most probably it is a cyclic ALTER."); + } + else + { + LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); + + storage.setTableStructure(std::move(columns_in_zk), metadata_diff); + + LOG_INFO(log, "Applied changes to the metadata of the table."); + } + + columns_version = columns_version; + metadata_version = metadata_version; + + recalculateColumnSizes(); + /// Update metadata ZK nodes for a specific replica. + if (changed_columns_version) + zookeeper->set(replica_path + "/columns", columns_str); + if (changed_metadata_version) + zookeeper->set(replica_path + "/metadata", metadata_str); + } +} + bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedMergeTree::LogEntry & entry) { const String & source_part_name = entry.source_parts.at(0); @@ -3199,6 +3269,7 @@ void StorageReplicatedMergeTree::alter( const String current_database_name = getDatabaseName(); const String current_table_name = getTableName(); + auto maybe_mutation_commands = params.getMutationCommands(getInMemoryMetadata()); /// We cannot check this alter commands with method isModifyingData() /// because ReplicatedMergeTree stores both columns and metadata for @@ -3218,31 +3289,6 @@ void StorageReplicatedMergeTree::alter( return; } - /// Alter is done by modifying the metadata nodes in ZK that are shared between all replicas - /// (/columns, /metadata). We set contents of the shared nodes to the new values and wait while - /// replicas asynchronously apply changes (see ReplicatedMergeTreeAlterThread.cpp) and modify - /// their respective replica metadata nodes (/replicas//columns, /replicas//metadata). - - struct ChangedNode - { - ChangedNode(const String & table_path_, String name_, String new_value_) - : table_path(table_path_), name(std::move(name_)), shared_path(table_path + "/" + name) - , new_value(std::move(new_value_)) - {} - - const String & table_path; - String name; - - String shared_path; - - String getReplicaPath(const String & replica) const - { - return table_path + "/replicas/" + replica + "/" + name; - } - - String new_value; - int32_t new_version = -1; /// Initialization is to suppress (useless) false positive warning found by cppcheck. - }; auto ast_to_str = [](ASTPtr query) -> String { @@ -3251,9 +3297,6 @@ void StorageReplicatedMergeTree::alter( return queryToString(query); }; - /// /columns and /metadata nodes - std::vector changed_nodes; - { /// Just to read current structure. Alter will be done in separate thread. auto table_lock = lockStructureForShare(false, query_context.getCurrentQueryId()); @@ -3309,202 +3352,219 @@ void StorageReplicatedMergeTree::alter( table_lock_holder.release(); - /// Wait until all replicas will apply ALTER. + ReplicatedMergeTreeLogEntryData entry; + entry.type = LogEntry::FINISH_ALTER; + entry.source_replica = replica_name; - for (const auto & node : changed_nodes) + if (maybe_mutation_commands) { - Coordination::Stat stat; - /// Subscribe to change of shared ZK metadata nodes, to finish waiting if someone will do another ALTER. - if (!getZooKeeper()->exists(node.shared_path, &stat, alter_query_event)) - throw Exception(node.shared_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - - if (stat.version != node.new_version) - { - LOG_WARNING(log, node.shared_path + " changed before this ALTER finished; " + - "overlapping ALTER-s are fine but use caution with nontransitive changes"); - return; - } + ReplicatedMergeTreeMutationEntry entry = mutateImpl(*maybe_mutation_commands, context); + entry.source_parts = queue.getPartNamesToMutate(entry); } - Strings replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); + entry.new_part_name = new_part_name; + entry.create_time = time(nullptr); - std::set inactive_replicas; - std::set timed_out_replicas; + zookeeper->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); - time_t replication_alter_columns_timeout = query_context.getSettingsRef().replication_alter_columns_timeout; + waitForAllReplicasToProcessLogEntry(entry); - /// This code is quite similar with waitMutationToFinishOnReplicas - /// but contains more complicated details (versions manipulations, multiple nodes, etc.). - /// It will be removed soon in favor of alter-modify implementation on top of mutations. - /// TODO (alesap) - for (const String & replica : replicas) - { - LOG_DEBUG(log, "Waiting for " << replica << " to apply changes"); + ///// Wait until all replicas will apply ALTER. - while (!partial_shutdown_called) - { - auto zookeeper = getZooKeeper(); + //for (const auto & node : changed_nodes) + //{ + // Coordination::Stat stat; + // /// Subscribe to change of shared ZK metadata nodes, to finish waiting if someone will do another ALTER. + // if (!getZooKeeper()->exists(node.shared_path, &stat, alter_query_event)) + // throw Exception(node.shared_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - /// Replica could be inactive. - if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) - { - LOG_WARNING(log, "Replica " << replica << " is not active during ALTER query." - " ALTER will be done asynchronously when replica becomes active."); + // if (stat.version != node.new_version) + // { + // LOG_WARNING(log, node.shared_path + " changed before this ALTER finished; " + + // "overlapping ALTER-s are fine but use caution with nontransitive changes"); + // return; + // } + //} - inactive_replicas.emplace(replica); - break; - } + //Strings replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); - struct ReplicaNode - { - explicit ReplicaNode(String path_) : path(std::move(path_)) {} + //std::set inactive_replicas; + //std::set timed_out_replicas; - String path; - String value; - int32_t version = -1; - }; + //time_t replication_alter_columns_timeout = query_context.getSettingsRef().replication_alter_columns_timeout; - std::vector replica_nodes; - for (const auto & node : changed_nodes) - replica_nodes.emplace_back(node.getReplicaPath(replica)); + ///// This code is quite similar with waitMutationToFinishOnReplicas + ///// but contains more complicated details (versions manipulations, multiple nodes, etc.). + ///// It will be removed soon in favor of alter-modify implementation on top of mutations. + ///// TODO (alesap) + //for (const String & replica : replicas) + //{ + // LOG_DEBUG(log, "Waiting for " << replica << " to apply changes"); - bool replica_was_removed = false; - for (auto & node : replica_nodes) - { - Coordination::Stat stat; + // while (!partial_shutdown_called) + // { + // auto zookeeper = getZooKeeper(); - /// Replica could has been removed. - if (!zookeeper->tryGet(node.path, node.value, &stat)) - { - LOG_WARNING(log, replica << " was removed"); - replica_was_removed = true; - break; - } + // /// Replica could be inactive. + // if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) + // { + // LOG_WARNING(log, "Replica " << replica << " is not active during ALTER query." + // " ALTER will be done asynchronously when replica becomes active."); - node.version = stat.version; - } + // inactive_replicas.emplace(replica); + // break; + // } - if (replica_was_removed) - break; + // struct ReplicaNode + // { + // explicit ReplicaNode(String path_) : path(std::move(path_)) {} - bool alter_was_applied = true; - for (size_t i = 0; i < replica_nodes.size(); ++i) - { - if (replica_nodes[i].value != changed_nodes[i].new_value) - { - alter_was_applied = false; - break; - } - } + // String path; + // String value; + // int32_t version = -1; + // }; - /// The ALTER has been successfully applied. - if (alter_was_applied) - break; + // std::vector replica_nodes; + // for (const auto & node : changed_nodes) + // replica_nodes.emplace_back(node.getReplicaPath(replica)); - for (const auto & node : changed_nodes) - { - Coordination::Stat stat; - if (!zookeeper->exists(node.shared_path, &stat)) - throw Exception(node.shared_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); + // bool replica_was_removed = false; + // for (auto & node : replica_nodes) + // { + // Coordination::Stat stat; - if (stat.version != node.new_version) - { - LOG_WARNING(log, node.shared_path + " changed before this ALTER finished; " - "overlapping ALTER-s are fine but use caution with nontransitive changes"); - return; - } - } + // /// Replica could has been removed. + // if (!zookeeper->tryGet(node.path, node.value, &stat)) + // { + // LOG_WARNING(log, replica << " was removed"); + // replica_was_removed = true; + // break; + // } - bool replica_nodes_changed_concurrently = false; - for (const auto & replica_node : replica_nodes) - { - Coordination::Stat stat; - if (!zookeeper->exists(replica_node.path, &stat, alter_query_event)) - { - LOG_WARNING(log, replica << " was removed"); - replica_was_removed = true; - break; - } + // node.version = stat.version; + // } - if (stat.version != replica_node.version) - { - replica_nodes_changed_concurrently = true; - break; - } - } + // if (replica_was_removed) + // break; - if (replica_was_removed) - break; + // bool alter_was_applied = true; + // for (size_t i = 0; i < replica_nodes.size(); ++i) + // { + // if (replica_nodes[i].value != changed_nodes[i].new_value) + // { + // alter_was_applied = false; + // break; + // } + // } - if (replica_nodes_changed_concurrently) - continue; + // /// The ALTER has been successfully applied. + // if (alter_was_applied) + // break; - /// alter_query_event subscribed with zookeeper watch callback to /repliacs/{replica}/metadata - /// and /replicas/{replica}/columns nodes for current relica + shared nodes /columns and /metadata, - /// which is common for all replicas. If changes happen with this nodes (delete, set and create) - /// than event will be notified and wait will be interrupted. - /// - /// ReplicatedMergeTreeAlterThread responsible for local /replicas/{replica}/metadata and - /// /replicas/{replica}/columns changes. Shared /columns and /metadata nodes can be changed by *newer* - /// concurrent alter from other replica. First of all it will update shared nodes and we will have no - /// ability to identify, that our *current* alter finshed. So we cannot do anything better than just - /// return from *current* alter with success result. - if (!replication_alter_columns_timeout) - { - alter_query_event->wait(); - /// Everything is fine. - } - else if (alter_query_event->tryWait(replication_alter_columns_timeout * 1000)) - { - /// Everything is fine. - } - else - { - LOG_WARNING(log, "Timeout when waiting for replica " << replica << " to apply ALTER." - " ALTER will be done asynchronously."); + // for (const auto & node : changed_nodes) + // { + // Coordination::Stat stat; + // if (!zookeeper->exists(node.shared_path, &stat)) + // throw Exception(node.shared_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - timed_out_replicas.emplace(replica); - break; - } - } + // if (stat.version != node.new_version) + // { + // LOG_WARNING(log, node.shared_path + " changed before this ALTER finished; " + // "overlapping ALTER-s are fine but use caution with nontransitive changes"); + // return; + // } + // } - if (partial_shutdown_called) - throw Exception("Alter is not finished because table shutdown was called. Alter will be done after table restart.", - ErrorCodes::UNFINISHED); + // bool replica_nodes_changed_concurrently = false; + // for (const auto & replica_node : replica_nodes) + // { + // Coordination::Stat stat; + // if (!zookeeper->exists(replica_node.path, &stat, alter_query_event)) + // { + // LOG_WARNING(log, replica << " was removed"); + // replica_was_removed = true; + // break; + // } - if (!inactive_replicas.empty() || !timed_out_replicas.empty()) - { - std::stringstream exception_message; - exception_message << "Alter is not finished because"; + // if (stat.version != replica_node.version) + // { + // replica_nodes_changed_concurrently = true; + // break; + // } + // } - if (!inactive_replicas.empty()) - { - exception_message << " some replicas are inactive right now"; + // if (replica_was_removed) + // break; - for (auto it = inactive_replicas.begin(); it != inactive_replicas.end(); ++it) - exception_message << (it == inactive_replicas.begin() ? ": " : ", ") << *it; - } + // if (replica_nodes_changed_concurrently) + // continue; - if (!timed_out_replicas.empty() && !inactive_replicas.empty()) - exception_message << " and"; + // /// alter_query_event subscribed with zookeeper watch callback to /repliacs/{replica}/metadata + // /// and /replicas/{replica}/columns nodes for current relica + shared nodes /columns and /metadata, + // /// which is common for all replicas. If changes happen with this nodes (delete, set and create) + // /// than event will be notified and wait will be interrupted. + // /// + // /// ReplicatedMergeTreeAlterThread responsible for local /replicas/{replica}/metadata and + // /// /replicas/{replica}/columns changes. Shared /columns and /metadata nodes can be changed by *newer* + // /// concurrent alter from other replica. First of all it will update shared nodes and we will have no + // /// ability to identify, that our *current* alter finshed. So we cannot do anything better than just + // /// return from *current* alter with success result. + // if (!replication_alter_columns_timeout) + // { + // alter_query_event->wait(); + // /// Everything is fine. + // } + // else if (alter_query_event->tryWait(replication_alter_columns_timeout * 1000)) + // { + // /// Everything is fine. + // } + // else + // { + // LOG_WARNING(log, "Timeout when waiting for replica " << replica << " to apply ALTER." + // " ALTER will be done asynchronously."); - if (!timed_out_replicas.empty()) - { - exception_message << " timeout when waiting for some replicas"; + // timed_out_replicas.emplace(replica); + // break; + // } + // } - for (auto it = timed_out_replicas.begin(); it != timed_out_replicas.end(); ++it) - exception_message << (it == timed_out_replicas.begin() ? ": " : ", ") << *it; + // if (partial_shutdown_called) + // throw Exception("Alter is not finished because table shutdown was called. Alter will be done after table restart.", + // ErrorCodes::UNFINISHED); - exception_message << " (replication_alter_columns_timeout = " << replication_alter_columns_timeout << ")"; - } + // if (!inactive_replicas.empty() || !timed_out_replicas.empty()) + // { + // std::stringstream exception_message; + // exception_message << "Alter is not finished because"; - exception_message << ". Alter will be done asynchronously."; + // if (!inactive_replicas.empty()) + // { + // exception_message << " some replicas are inactive right now"; - throw Exception(exception_message.str(), ErrorCodes::UNFINISHED); - } - } + // for (auto it = inactive_replicas.begin(); it != inactive_replicas.end(); ++it) + // exception_message << (it == inactive_replicas.begin() ? ": " : ", ") << *it; + // } - LOG_DEBUG(log, "ALTER finished"); + // if (!timed_out_replicas.empty() && !inactive_replicas.empty()) + // exception_message << " and"; + + // if (!timed_out_replicas.empty()) + // { + // exception_message << " timeout when waiting for some replicas"; + + // for (auto it = timed_out_replicas.begin(); it != timed_out_replicas.end(); ++it) + // exception_message << (it == timed_out_replicas.begin() ? ": " : ", ") << *it; + + // exception_message << " (replication_alter_columns_timeout = " << replication_alter_columns_timeout << ")"; + // } + + // exception_message << ". Alter will be done asynchronously."; + + // throw Exception(exception_message.str(), ErrorCodes::UNFINISHED); + // } + //} + + //LOG_DEBUG(log, "ALTER finished"); } void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) @@ -4457,6 +4517,11 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const Context & query_context) +{ + mutateImpl(commands, context); +} + +StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & query_context) { /// Overview of the mutation algorithm. /// @@ -4572,6 +4637,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const waitMutationToFinishOnReplicas(replicas, entry.znode_name); } + return entry; } std::vector StorageReplicatedMergeTree::getMutationsStatus() const diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 60c2ea0b870..0ccdf046892 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -110,6 +110,7 @@ public: void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; void mutate(const MutationCommands & commands, const Context & context) override; + ReplicatedMergeTreeMutationEntry mutateImpl(const MutationCommands & commands, const Context & context); std::vector getMutationsStatus() const override; CancellationCode killMutation(const String & mutation_id) override; @@ -382,6 +383,8 @@ private: /// Do the merge or recommend to make the fetch instead of the merge bool tryExecuteMerge(const LogEntry & entry); + bool tryFinishAlter(const LogEntry & entry); + bool tryExecutePartMutation(const LogEntry & entry); @@ -431,6 +434,9 @@ private: /// Checks if some mutations are done and marks them as done. void mutationsFinalizingTask(); + /// finish alter after all heavy processes finished + void finishAlter(); + /** Write the selected parts to merge into the log, * Call when merge_selecting_mutex is locked. * Returns false if any part is not in ZK. From f15696246ee3f8aae8dc912ba575fa420d330777 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 13 Jan 2020 19:40:24 +0300 Subject: [PATCH 0100/2007] add part type to system.parts table --- dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp | 2 +- dbms/src/Storages/MergeTree/IMergeTreeDataPart.h | 2 +- dbms/src/Storages/System/StorageSystemParts.cpp | 2 ++ dbms/src/Storages/System/StorageSystemPartsColumns.cpp | 2 ++ 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 270c574df83..3d340e2e869 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -785,7 +785,7 @@ String IMergeTreeDataPart::typeToString(Type type) case Type::WIDE: return "Wide"; case Type::COMPACT: - return "Striped"; + return "Compact"; case Type::IN_MEMORY: return "InMemory"; case Type::UNKNOWN: diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index f569ca75923..1ad21ba8a27 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -109,7 +109,7 @@ public: virtual Type getType() const = 0; static String typeToString(Type type); - String getTypeName() { return typeToString(getType()); } + String getTypeName() const { return typeToString(getType()); } IMergeTreeDataPart( const MergeTreeData & storage_, diff --git a/dbms/src/Storages/System/StorageSystemParts.cpp b/dbms/src/Storages/System/StorageSystemParts.cpp index bb00efd8909..82ff63db012 100644 --- a/dbms/src/Storages/System/StorageSystemParts.cpp +++ b/dbms/src/Storages/System/StorageSystemParts.cpp @@ -19,6 +19,7 @@ StorageSystemParts::StorageSystemParts(const std::string & name_) { {"partition", std::make_shared()}, {"name", std::make_shared()}, + {"part_type", std::make_shared()}, {"active", std::make_shared()}, {"marks", std::make_shared()}, {"rows", std::make_shared()}, @@ -78,6 +79,7 @@ void StorageSystemParts::processNextStorage(MutableColumns & columns_, const Sto columns_[i++]->insert(out.str()); } columns_[i++]->insert(part->name); + columns_[i++]->insert(part->getTypeName()); columns_[i++]->insert(part_state == State::Committed); columns_[i++]->insert(part->getMarksCount()); columns_[i++]->insert(part->rows_count); diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 3aad49eb942..25565810227 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -20,6 +20,7 @@ StorageSystemPartsColumns::StorageSystemPartsColumns(const std::string & name_) { {"partition", std::make_shared()}, {"name", std::make_shared()}, + {"part_type", std::make_shared()}, {"active", std::make_shared()}, {"marks", std::make_shared()}, {"rows", std::make_shared()}, @@ -110,6 +111,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns_, co columns_[j++]->insert(out.str()); } columns_[j++]->insert(part->name); + columns_[j++]->insert(part->getTypeName()); columns_[j++]->insert(part_state == State::Committed); columns_[j++]->insert(part->getMarksCount()); From a90ab99282a2d7b6e3f3106aa407461eabd6ceb5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 13 Jan 2020 19:40:30 +0300 Subject: [PATCH 0101/2007] Maybe it will compile better --- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 0fa52b28ba1..b7048de6e34 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1192,7 +1192,7 @@ bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree { LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); - auto table_lock = storage.lockExclusively(RWLockImpl::NO_QUERY); + auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); if (columns_in_zk == storage.getColumns() && metadata_diff.empty()) { @@ -1205,7 +1205,7 @@ bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree { LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); - storage.setTableStructure(std::move(columns_in_zk), metadata_diff); + setTableStructure(std::move(columns_in_zk), metadata_diff); LOG_INFO(log, "Applied changes to the metadata of the table."); } From 18eacfe7211581fc697153162634e15b7061c5eb Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 13 Jan 2020 21:12:53 +0300 Subject: [PATCH 0102/2007] ignore compact parts in MergeTreeWhereOptimizer --- dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index dcca5baf311..699f2d0207f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -86,6 +86,8 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const ASTPtr & node) collectIdentifiersNoSubqueries(node, cond.identifiers); + cond.columns_size = getIdentifiersColumnSize(cond.identifiers); + cond.viable = /// Condition depend on some column. Constant expressions are not moved. !cond.identifiers.empty() @@ -95,13 +97,12 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const ASTPtr & node) /// Only table columns are considered. Not array joined columns. NOTE We're assuming that aliases was expanded. && isSubsetOfTableColumns(cond.identifiers) /// Do not move conditions involving all queried columns. - && cond.identifiers.size() < queried_columns.size(); + && cond.identifiers.size() < queried_columns.size() + /// Columns size of compact parts can't be counted. If all parts are compact do not move any condition. + && cond.columns_size > 0; if (cond.viable) - { - cond.columns_size = getIdentifiersColumnSize(cond.identifiers); cond.good = isConditionGood(node); - } res.emplace_back(std::move(cond)); } From ce914cbab8cff5a8437352e5df1d14f4c6a92f1f Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 14 Jan 2020 16:23:51 +0300 Subject: [PATCH 0103/2007] refactor code near MergeTreeDataPart --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 23 +++++--- .../Storages/MergeTree/IMergeTreeDataPart.h | 18 +++--- .../MergeTree/IMergeTreeDataPart_fwd.h | 22 -------- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 5 +- dbms/src/Storages/MergeTree/MergeTreeData.h | 3 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 3 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 14 +---- .../MergeTree/MergeTreeDataPartCompact.h | 4 -- .../MergeTree/MergeTreeDataPartType.h | 12 ++++ .../MergeTree/MergeTreeDataPartWide.cpp | 8 +-- .../MergeTree/MergeTreeDataPartWide.h | 2 - .../MergeTreeIndexGranularityInfo.cpp | 55 ++++++++----------- .../MergeTree/MergeTreeIndexGranularityInfo.h | 25 +++------ .../MergeTree/MergeTreeIndexReader.cpp | 1 - .../MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../MergeTree/MergeTreeReaderStream.cpp | 6 +- .../MergeTree/MergeTreeReaderStream.h | 8 --- .../MergeTree/MergeTreeReaderWide.cpp | 1 - 18 files changed, 78 insertions(+), 134 deletions(-) delete mode 100644 dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartType.h diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 3d340e2e869..affcb04b79f 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -52,12 +52,12 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Str { String file_name = part_path + "minmax_" + escapeForFileName(data.minmax_idx_columns[i]) + ".idx"; ReadBufferFromFile file = openForReading(file_name); - const DataTypePtr & type = data.minmax_idx_column_types[i]; + const DataTypePtr & data_type = data.minmax_idx_column_types[i]; Field min_val; - type->deserializeBinary(min_val, file); + data_type->deserializeBinary(min_val, file); Field max_val; - type->deserializeBinary(max_val, file); + data_type->deserializeBinary(max_val, file); parallelogram.emplace_back(min_val, true, max_val, true); } @@ -78,12 +78,12 @@ void IMergeTreeDataPart::MinMaxIndex::store(const Names & column_names, const Da for (size_t i = 0; i < column_names.size(); ++i) { String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; - const DataTypePtr & type = data_types.at(i); + const DataTypePtr & data_type = data_types.at(i); WriteBufferFromFile out(part_path + file_name); HashingWriteBuffer out_hashing(out); - type->serializeBinary(parallelogram[i].left, out_hashing); - type->serializeBinary(parallelogram[i].right, out_hashing); + data_type->serializeBinary(parallelogram[i].left, out_hashing); + data_type->serializeBinary(parallelogram[i].right, out_hashing); out_hashing.next(); out_checksums.files[file_name].file_size = out_hashing.count(); out_checksums.files[file_name].file_hash = out_hashing.getHash(); @@ -139,12 +139,15 @@ IMergeTreeDataPart::IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, const DiskPtr & disk_, - const std::optional & relative_path_) + const std::optional & relative_path_, + Type part_type_) : storage(storage_) , name(name_) , info(MergeTreePartInfo::fromPartName(name_, storage.format_version)) , disk(disk_) , relative_path(relative_path_.value_or(name_)) + , index_granularity_info(storage_, part_type_) + , part_type(part_type_) { } @@ -153,12 +156,15 @@ IMergeTreeDataPart::IMergeTreeDataPart( const String & name_, const MergeTreePartInfo & info_, const DiskPtr & disk_, - const std::optional & relative_path_) + const std::optional & relative_path_, + Type part_type_) : storage(storage_) , name(name_) , info(info_) , disk(disk_) , relative_path(relative_path_.value_or(name_)) + , index_granularity_info(storage_, part_type_) + , part_type(part_type_) { } @@ -593,7 +599,6 @@ void IMergeTreeDataPart::loadColumns(bool require) columns.readText(file); } - index_granularity_info = MergeTreeIndexGranularityInfo{storage, getType(), columns.size()}; size_t pos = 0; for (const auto & column : columns) column_name_to_position.emplace(column.name, pos++); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 1ad21ba8a27..5d7fca42f6c 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -85,7 +85,7 @@ public: virtual String getFileNameForColumn(const NameAndTypePair & column) const = 0; - virtual void setColumns(const NamesAndTypesList & columns_); + void setColumns(const NamesAndTypesList & columns_); virtual NameToNameMap createRenameMapForAlter( AlterAnalysisResult & /* analysis_result */, @@ -106,7 +106,7 @@ public: virtual void accumulateColumnSizes(ColumnToSize & column_to_size) const; using Type = MergeTreeDataPartType; - virtual Type getType() const = 0; + Type getType() const { return part_type; } static String typeToString(Type type); String getTypeName() const { return typeToString(getType()); } @@ -115,14 +115,16 @@ public: const MergeTreeData & storage_, const String & name_, const MergeTreePartInfo & info_, - const DiskPtr & disk = {}, - const std::optional & relative_path = {}); + const DiskPtr & disk, + const std::optional & relative_path, + Type part_type_); IMergeTreeDataPart( MergeTreeData & storage_, const String & name_, - const DiskPtr & disk = {}, - const std::optional & relative_path = {}); + const DiskPtr & disk, + const std::optional & relative_path, + Type part_type_); void assertOnDisk() const; @@ -157,11 +159,11 @@ public: String name; MergeTreePartInfo info; - MergeTreeIndexGranularityInfo index_granularity_info; DiskPtr disk; mutable String relative_path; + MergeTreeIndexGranularityInfo index_granularity_info; size_t rows_count = 0; @@ -313,7 +315,7 @@ public: static UInt64 calculateTotalSizeOnDisk(const String & from); protected: - + Type part_type; void removeIfNeeded(); virtual void checkConsistency(bool require_part_metadata) const; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h deleted file mode 100644 index 6812357b196..00000000000 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart_fwd.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once -#include - -namespace DB -{ - // class IMergeTreeDataPart; - // class IMergeTreeReader; - // class IMergeTreeWriter; - - // using MergeTreeMutableDataPartPtr = std::shared_ptr; - // using MergeTreeDataPartPtr = std::shared_ptr; - // using MergeTreeReaderPtr = std::unique_ptr; - // using MergeTreeWriterPtr = std::unique_ptr; - - enum class MergeTreeDataPartType - { - WIDE, - COMPACT, - IN_MEMORY, - UNKNOWN, - }; -} diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 37b5d07c5b6..3bb07315bdf 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1599,13 +1599,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const NamesAndTypesList & columns_list, size_t bytes_uncompressed, size_t rows_count, - const String & relative_path, - const MergeTreeIndexGranularityInfo * index_granularity_info) const + const String & relative_path) const { auto type = choosePartType(bytes_uncompressed, rows_count); auto part = createPart(name, type, part_info, disk, relative_path); - part->index_granularity_info = index_granularity_info ? *index_granularity_info - : MergeTreeIndexGranularityInfo{*this, type, columns_list.size()}; part->setColumns(columns_list); /// Don't save rows_count count here as it can be changed later return part; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index aef43bb5a2e..5caf5e5dcfa 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -188,8 +188,7 @@ public: const MergeTreePartInfo & part_info,const DiskPtr & disk, const NamesAndTypesList & columns, size_t bytes_on_disk, size_t rows_num, - const String & relative_path, - const MergeTreeIndexGranularityInfo * index_granularity_info = nullptr) const; + const String & relative_path) const; MutableDataPartPtr createPart(const String & name, MergeTreeDataPartType type, const MergeTreePartInfo & part_info, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 3226f3a14a6..325cc6af1a6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -992,8 +992,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor std::move(new_columns), source_part->bytes_on_disk, source_part->rows_count, - "tmp_mut_" + future_part.name, - &source_part->index_granularity_info); /// Granularity info can't be changed during mutation. + "tmp_mut_" + future_part.name); new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 754433a92f7..b29290e8bb9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -57,7 +57,7 @@ MergeTreeDataPartCompact::MergeTreeDataPartCompact( const String & name_, const DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, disk_, relative_path_, Type::COMPACT) { } @@ -67,7 +67,7 @@ MergeTreeDataPartCompact::MergeTreeDataPartCompact( const MergeTreePartInfo & info_, const DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_, Type::COMPACT) { } @@ -160,7 +160,6 @@ String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const void MergeTreeDataPartCompact::loadIndexGranularity() { - index_granularity_info = MergeTreeIndexGranularityInfo{storage, getType(), columns.size()}; String full_path = getFullPath(); if (columns.empty()) @@ -185,7 +184,7 @@ void MergeTreeDataPartCompact::loadIndexGranularity() index_granularity.appendMark(granularity); } - if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) + if (index_granularity.getMarksCount() * index_granularity_info.getMarkSizeInBytes(columns.size()) != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); index_granularity.setInitialized(); @@ -253,13 +252,6 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( return rename_map; } -void MergeTreeDataPartCompact::setColumns(const NamesAndTypesList & new_columns) -{ - if (new_columns.size() != columns.size()) - index_granularity_info = MergeTreeIndexGranularityInfo{storage, Type::COMPACT, new_columns.size()}; - IMergeTreeDataPart::setColumns(new_columns); -} - void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { UNUSED(require_part_metadata); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index d03ab472513..8191ed8d33e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -73,8 +73,6 @@ public: /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; - Type getType() const override { return Type::COMPACT; } - ColumnSize getColumnSize(const String & name, const IDataType & type0) const override; ColumnSize getTotalColumnsSize() const override; @@ -89,8 +87,6 @@ public: AlterAnalysisResult & analysis_result, const NamesAndTypesList & old_columns) const override; - void setColumns(const NamesAndTypesList & new_columns) override; - ~MergeTreeDataPartCompact() override; private: diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h new file mode 100644 index 00000000000..0f531351e0d --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -0,0 +1,12 @@ +#pragma once + +namespace DB +{ + enum class MergeTreeDataPartType + { + WIDE, + COMPACT, + IN_MEMORY, + UNKNOWN, + }; +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index d8c705a874c..f8010b7b774 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -54,7 +54,7 @@ MergeTreeDataPartWide::MergeTreeDataPartWide( const String & name_, const DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, disk_, relative_path_, Type::WIDE) { } @@ -64,7 +64,7 @@ MergeTreeDataPartWide::MergeTreeDataPartWide( const MergeTreePartInfo & info_, const DiskPtr & disk_, const std::optional & relative_path_) - : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_) + : IMergeTreeDataPart(storage_, name_, info_, disk_, relative_path_, Type::WIDE) { } @@ -189,7 +189,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() if (!index_granularity_info.is_adaptive) { - size_t marks_count = marks_file_size / index_granularity_info.mark_size_in_bytes; + size_t marks_count = marks_file_size / index_granularity_info.getMarkSizeInBytes(); index_granularity.resizeWithFixedGranularity(marks_count, index_granularity_info.fixed_index_granularity); /// all the same } else @@ -203,7 +203,7 @@ void MergeTreeDataPartWide::loadIndexGranularity() index_granularity.appendMark(granularity); } - if (index_granularity.getMarksCount() * index_granularity_info.mark_size_in_bytes != marks_file_size) + if (index_granularity.getMarksCount() * index_granularity_info.getMarkSizeInBytes() != marks_file_size) throw Exception("Cannot read all marks from file " + marks_file_path, ErrorCodes::CANNOT_READ_ALL_DATA); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index ae747382dba..479e6d77e67 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -74,8 +74,6 @@ public: /// If no checksums are present returns the name of the first physically existing column. String getColumnNameWithMinumumCompressedSize() const override; - Type getType() const override { return Type::WIDE; } - ColumnSize getTotalColumnsSize() const override; ColumnSize getColumnSize(const String & column_name, const IDataType & type) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index d56c69b9173..e63b06b3994 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -30,8 +30,8 @@ std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS( return {}; } -MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo( - const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num) +MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo(const MergeTreeData & storage, MergeTreeDataPartType type_) + : type(type_) { const auto storage_settings = storage.getSettings(); fixed_index_granularity = storage_settings->index_granularity; @@ -39,14 +39,12 @@ MergeTreeIndexGranularityInfo::MergeTreeIndexGranularityInfo( /// Granularity is fixed if (!storage.canUseAdaptiveGranularity()) { - if (part_type != MergeTreeDataPartType::WIDE) + if (type != MergeTreeDataPartType::WIDE) throw Exception("Only Wide parts can be used with non-adaptive granularity.", ErrorCodes::NOT_IMPLEMENTED); setNonAdaptive(); } else - setAdaptive(storage_settings->index_granularity_bytes, part_type, columns_num); - - initialized = true; + setAdaptive(storage_settings->index_granularity_bytes); } void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const std::string & path_to_part) @@ -56,47 +54,38 @@ void MergeTreeIndexGranularityInfo::changeGranularityIfRequired(const std::strin setNonAdaptive(); } -void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_, MergeTreeDataPartType part_type, size_t columns_num) +void MergeTreeIndexGranularityInfo::setAdaptive(size_t index_granularity_bytes_) { is_adaptive = true; - mark_size_in_bytes = getAdaptiveMrkSize(part_type, columns_num); - skip_index_mark_size_in_bytes = sizeof(MarkInCompressedFile) + sizeof(UInt64); - marks_file_extension = getAdaptiveMrkExtension(part_type); + marks_file_extension = getAdaptiveMrkExtension(type); index_granularity_bytes = index_granularity_bytes_; } void MergeTreeIndexGranularityInfo::setNonAdaptive() { is_adaptive = false; - mark_size_in_bytes = skip_index_mark_size_in_bytes = getNonAdaptiveMrkSize(); marks_file_extension = getNonAdaptiveMrkExtension(); index_granularity_bytes = 0; } +size_t MergeTreeIndexGranularityInfo::getMarkSizeInBytes(size_t columns_num) const +{ + if (type == MergeTreeDataPartType::WIDE) + return is_adaptive ? getAdaptiveMrkSizeWide() : getNonAdaptiveMrkSizeWide(); + else if (type == MergeTreeDataPartType::COMPACT) + return sizeof(UInt64) * (columns_num * 2 + 1); + else + throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); +} + std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) { - switch (part_type) - { - case MergeTreeDataPartType::WIDE: - return ".mrk2"; - case MergeTreeDataPartType::COMPACT: - return ".mrk3"; - default: - throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); - } -} - -size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num) -{ - switch (part_type) - { - case MergeTreeDataPartType::WIDE: - return sizeof(UInt64) * 3; - case MergeTreeDataPartType::COMPACT: - return sizeof(UInt64) * (columns_num * 2 + 1); - default: - throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); - } + if (part_type == MergeTreeDataPartType::WIDE) + return ".mrk2"; + else if (part_type == MergeTreeDataPartType::COMPACT) + return ".mrk3"; + else + throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index d72ef5f7917..c7004b6222f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -2,7 +2,7 @@ #include #include -#include +#include #include namespace DB @@ -22,11 +22,6 @@ public: /// Marks file extension '.mrk' or '.mrk2' String marks_file_extension; - /// Size of one mark in file two or three size_t numbers - UInt32 mark_size_in_bytes = 0; - - UInt8 skip_index_mark_size_in_bytes = 0; - /// Is stride in rows between marks non fixed? bool is_adaptive = false; @@ -36,14 +31,9 @@ public: /// Approximate bytes size of one granule size_t index_granularity_bytes = 0; - bool initialized = false; - MergeTreeIndexGranularityInfo() {} - MergeTreeIndexGranularityInfo( - const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num); - - void initialize(const MergeTreeData & storage, MergeTreeDataPartType part_type, size_t columns_num); + MergeTreeIndexGranularityInfo(const MergeTreeData & storage, MergeTreeDataPartType type_); void changeGranularityIfRequired(const std::string & path_to_part); @@ -52,18 +42,19 @@ public: return path_prefix + marks_file_extension; } + size_t getMarkSizeInBytes(size_t columns_num = 1) const; + static std::optional getMrkExtensionFromFS(const std::string & path_to_table); private: - void setAdaptive(size_t index_granularity_bytes_, MergeTreeDataPartType part_type, size_t columns_num); + MergeTreeDataPartType type; + void setAdaptive(size_t index_granularity_bytes_); void setNonAdaptive(); - void setCompactAdaptive(size_t index_granularity_bytes_, size_t columns_num); }; constexpr inline auto getNonAdaptiveMrkExtension() { return ".mrk"; } -constexpr inline auto getNonAdaptiveMrkSize() { return sizeof(UInt64) * 2; } - +constexpr inline auto getNonAdaptiveMrkSizeWide() { return sizeof(UInt64) * 2; } +constexpr inline auto getAdaptiveMrkSizeWide() { return sizeof(UInt64) * 3; } std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type); -size_t getAdaptiveMrkSize(MergeTreeDataPartType part_type, size_t columns_num); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp index 032f91c56df..bf53b669571 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexReader.cpp @@ -12,7 +12,6 @@ MergeTreeIndexReader::MergeTreeIndexReader( MergeTreeReaderSettings{}, nullptr, nullptr, part_->getFileSizeOrZero(index->getFileName() + ".idx"), &part_->index_granularity_info, - MergeTreeReaderStream::ReadingMode::INDEX, ReadBufferFromFileBase::ProfileCallback{}, CLOCK_MONOTONIC_COARSE) { stream.seekToStart(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index f6009a68200..973bef49d70 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -200,7 +200,7 @@ void MergeTreeReaderCompact::initMarksLoader() { size_t file_size = Poco::File(mrk_path).getSize(); size_t marks_count = data_part->getMarksCount(); - size_t mark_size_in_bytes = data_part->index_granularity_info.mark_size_in_bytes; + size_t mark_size_in_bytes = data_part->index_granularity_info.getMarkSizeInBytes(columns_num); size_t expected_file_size = mark_size_in_bytes * marks_count; if (expected_file_size != file_size) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 74c2f4f6754..b70c3c8b573 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -22,12 +22,10 @@ MergeTreeReaderStream::MergeTreeReaderStream( MarkCache * mark_cache_, UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, - ReadingMode mode_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type) : path_prefix(path_prefix_), data_file_extension(data_file_extension_), marks_count(marks_count_) , mark_cache(mark_cache_), save_marks_in_cache(settings.save_marks_in_cache) , index_granularity_info(index_granularity_info_) - , mode(mode_) { /// Compute the size of the buffer. size_t max_mark_range_bytes = 0; @@ -119,9 +117,7 @@ void MergeTreeReaderStream::initMarksLoader() auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); size_t file_size = Poco::File(mrk_path).getSize(); - size_t mark_size = mode == ReadingMode::INDEX - ? index_granularity_info->skip_index_mark_size_in_bytes - : index_granularity_info->mark_size_in_bytes; + size_t mark_size = index_granularity_info->getMarkSizeInBytes(); size_t expected_file_size = mark_size * marks_count; if (expected_file_size != file_size) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index 6d610e9ed67..8356fed8382 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -16,19 +16,12 @@ namespace DB class MergeTreeReaderStream { public: - enum class ReadingMode - { - COLUMN, - INDEX, - }; - MergeTreeReaderStream( const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const MergeTreeReaderSettings & settings_, MarkCache * mark_cache, UncompressedCache * uncompressed_cache, size_t file_size, const MergeTreeIndexGranularityInfo * index_granularity_info_, - ReadingMode mode_, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); void seekToMark(size_t index); @@ -49,7 +42,6 @@ private: bool save_marks_in_cache; const MergeTreeIndexGranularityInfo * index_granularity_info; - ReadingMode mode; std::unique_ptr cached_buffer; std::unique_ptr non_cached_buffer; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 10af08bfd08..7d7c8990c37 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -171,7 +171,6 @@ void MergeTreeReaderWide::addStreams(const String & name, const IDataType & type all_mark_ranges, settings, mark_cache, uncompressed_cache, data_part->getFileSizeOrZero(stream_name + DATA_FILE_EXTENSION), &data_part->index_granularity_info, - MergeTreeReaderStream::ReadingMode::COLUMN, profile_callback, clock_type)); }; From 2abf4bbc957ec2359a343db7ed7bb8b35aa86852 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 15 Jan 2020 16:00:08 +0300 Subject: [PATCH 0104/2007] Not working state --- .../Interpreters/InterpreterSelectQuery.cpp | 1 + .../src/Interpreters/MutationsInterpreter.cpp | 24 +- .../Interpreters/evaluateMissingDefaults.cpp | 93 +++++- .../Interpreters/evaluateMissingDefaults.h | 3 + dbms/src/Storages/AlterCommands.cpp | 36 ++- dbms/src/Storages/IStorage.cpp | 2 + .../MergeTreeBaseSelectProcessor.cpp | 2 + .../MergeTree/MergeTreeBlockReadUtils.cpp | 8 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 18 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 103 +++++-- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 3 + .../MergeTree/MergeTreeRangeReader.cpp | 3 + .../Storages/MergeTree/MergeTreeReadPool.cpp | 1 + .../Storages/MergeTree/MergeTreeReader.cpp | 97 +++++- dbms/src/Storages/MergeTree/MergeTreeReader.h | 5 + .../MergeTree/MergeTreeSelectProcessor.cpp | 40 +-- .../MergeTreeSequentialBlockInputStream.cpp | 6 +- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 18 +- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 7 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 24 +- .../MergeTree/ReplicatedMergeTreeQueue.h | 3 +- .../ReplicatedMergeTreeRestartingThread.cpp | 4 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 1 + dbms/src/Storages/MutationCommands.cpp | 24 +- dbms/src/Storages/MutationCommands.h | 4 +- .../Storages/StorageReplicatedMergeTree.cpp | 289 ++++-------------- .../src/Storages/StorageReplicatedMergeTree.h | 3 - 27 files changed, 461 insertions(+), 361 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index b147c5f4887..16163721c63 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -307,6 +307,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( } } + //std::cerr << "Source header:" << source_header.dumpStructure() << std::endl; if (storage) table_lock = storage->lockStructureForShare(false, context->getInitialQueryId()); diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 4a585cf424f..5ae7a8996d4 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -19,6 +19,7 @@ #include #include #include +#include namespace DB @@ -172,7 +173,9 @@ MutationsInterpreter::MutationsInterpreter( , context(context_) , can_execute(can_execute_) { + std::cerr << "STORAGE IS NULLPTR:" << (storage == nullptr) << std::endl; mutation_ast = prepare(!can_execute); + std::cerr << "Mutations ast:" << queryToString(mutation_ast) << std::endl; SelectQueryOptions limits = SelectQueryOptions().analyze(!can_execute).ignoreLimits(); select_interpreter = std::make_unique(mutation_ast, context, storage, limits); } @@ -259,15 +262,22 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) if (commands.empty()) throw Exception("Empty mutation commands list", ErrorCodes::LOGICAL_ERROR); + std::cerr << "PREPARING\n"; + const ColumnsDescription & columns_desc = storage->getColumns(); const IndicesDescription & indices_desc = storage->getIndices(); + std::cerr << "COLUMNS RECEIVED:" << columns_desc.toString() << std::endl; NamesAndTypesList all_columns = columns_desc.getAllPhysical(); + std::cerr << "COMMANDS SIZE:" << commands.size() << std::endl; NameSet updated_columns; for (const MutationCommand & command : commands) { for (const auto & kv : command.column_to_update_expression) + { + std::cerr << "COLUMN:" << kv.first << std::endl; updated_columns.insert(kv.first); + } } /// We need to know which columns affect which MATERIALIZED columns and data skipping indices @@ -311,6 +321,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) /// First, break a sequence of commands into stages. for (const auto & command : commands) { + std::cerr << "Processing command:" << command.ast << std::endl; if (command.type == MutationCommand::DELETE) { if (stages.empty() || !stages.back().column_to_updated.empty()) @@ -381,9 +392,16 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) const auto required_columns = syntax_result->requiredSourceColumns(); affected_indices_columns.insert(std::cbegin(required_columns), std::cend(required_columns)); } - else if (command.type == MutationCommand::CAST) + else if (command.type == MutationCommand::READ) { - stages.back().column_to_updated.emplace(command.column_name, makeASTFunction("CAST", command.column_name, command.type_name)); + if (stages.empty() || !stages.back().column_to_updated.empty()) + stages.emplace_back(context); + if (stages.size() == 1) /// First stage only supports filtering and can't update columns. + stages.emplace_back(context); + + /// TODO(alesap) + if (command.data_type) + stages.back().column_to_updated.emplace(command.column_name, std::make_shared(command.column_name)); } else throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); @@ -427,6 +445,8 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & prepared_stages, bool dry_run) { NamesAndTypesList all_columns = storage->getColumns().getAllPhysical(); + std::cerr << "Prepare interpreter storage columns:" << all_columns.toString() << std::endl; + /// Next, for each stage calculate columns changed by this and previous stages. for (size_t i = 0; i < prepared_stages.size(); ++i) diff --git a/dbms/src/Interpreters/evaluateMissingDefaults.cpp b/dbms/src/Interpreters/evaluateMissingDefaults.cpp index bef41488793..c5cb36e3979 100644 --- a/dbms/src/Interpreters/evaluateMissingDefaults.cpp +++ b/dbms/src/Interpreters/evaluateMissingDefaults.cpp @@ -1,12 +1,17 @@ #include "evaluateMissingDefaults.h" +#include #include +#include #include #include #include #include #include #include +#include +#include +#include #include #include @@ -14,7 +19,8 @@ namespace DB { -static ASTPtr requiredExpressions(Block & block, const NamesAndTypesList & required_columns, const ColumnDefaults & column_defaults) +namespace { +ASTPtr defaultRequiredExpressions(Block & block, const NamesAndTypesList & required_columns, const ColumnDefaults & column_defaults) { ASTPtr default_expr_list = std::make_shared(); @@ -27,8 +33,7 @@ static ASTPtr requiredExpressions(Block & block, const NamesAndTypesList & requi /// expressions must be cloned to prevent modification by the ExpressionAnalyzer if (it != column_defaults.end()) - default_expr_list->children.emplace_back( - setAlias(it->second.expression->clone(), it->first)); + default_expr_list->children.emplace_back(setAlias(it->second.expression->clone(), it->first)); } if (default_expr_list->children.empty()) @@ -36,22 +41,49 @@ static ASTPtr requiredExpressions(Block & block, const NamesAndTypesList & requi return default_expr_list; } -void evaluateMissingDefaults(Block & block, - const NamesAndTypesList & required_columns, - const ColumnDefaults & column_defaults, - const Context & context, bool save_unneeded_columns) +ASTPtr convertRequiredExpressions(Block & block, const NamesAndTypesList & required_columns) { - if (column_defaults.empty()) - return; + ASTPtr conversion_expr_list = std::make_shared(); + for (const auto & required_column : required_columns) + { + if (!block.has(required_column.name)) + continue; + //throw Exception("Required conversion of column " + required_column.name + " which is absent in block. It's a bug", ErrorCodes::LOGICAL_ERROR); - ASTPtr default_expr_list = requiredExpressions(block, required_columns, column_defaults); - if (!default_expr_list) + auto column_in_block = block.getByName(required_column.name); + //std::cerr << "Looking at:" << required_column.name << std::endl; + //std::cerr << "In block type:" << column_in_block.type->getName() << std::endl; + //std::cerr << "Required type:" << required_column.type->getName() << std::endl; + if (column_in_block.type->equals(*required_column.type)) + { + //std::cerr << "TYPES ARE SAME\n"; + continue; + } + //std::cerr << "TYPES ARE DIFFERENT\n"; + + auto cast_func = makeASTFunction( + "CAST", std::make_shared(required_column.name), std::make_shared(required_column.type->getName())); + + conversion_expr_list->children.emplace_back(setAlias(cast_func, required_column.name)); + + } + return conversion_expr_list; +} + +void executeExpressionsOnBlock( + Block & block, + ASTPtr expr_list, + bool save_unneeded_columns, + const NamesAndTypesList & required_columns, + const Context & context) +{ + if (!expr_list) return; if (!save_unneeded_columns) { - auto syntax_result = SyntaxAnalyzer(context).analyze(default_expr_list, block.getNamesAndTypesList()); - ExpressionAnalyzer{default_expr_list, syntax_result, context}.getActions(true)->execute(block); + auto syntax_result = SyntaxAnalyzer(context).analyze(expr_list, block.getNamesAndTypesList()); + ExpressionAnalyzer{expr_list, syntax_result, context}.getActions(true)->execute(block); return; } @@ -59,8 +91,8 @@ void evaluateMissingDefaults(Block & block, * we are going to operate on a copy instead of the original block */ Block copy_block{block}; - auto syntax_result = SyntaxAnalyzer(context).analyze(default_expr_list, block.getNamesAndTypesList()); - auto expression_analyzer = ExpressionAnalyzer{default_expr_list, syntax_result, context}; + auto syntax_result = SyntaxAnalyzer(context).analyze(expr_list, block.getNamesAndTypesList()); + auto expression_analyzer = ExpressionAnalyzer{expr_list, syntax_result, context}; auto required_source_columns = syntax_result->requiredSourceColumns(); auto rows_was = copy_block.rows(); @@ -82,7 +114,9 @@ void evaluateMissingDefaults(Block & block, copy_block.insert({DataTypeUInt8().createColumnConst(rows_was, 0u), std::make_shared(), "__dummy"}); } + //std::cerr << "Block before expression:" << copy_block.dumpStructure() << std::endl; expression_analyzer.getActions(true)->execute(copy_block); + //std::cerr << "Block after expression:" << copy_block.dumpStructure() << std::endl; /// move evaluated columns to the original block, materializing them at the same time size_t pos = 0; @@ -93,9 +127,36 @@ void evaluateMissingDefaults(Block & block, auto evaluated_col = copy_block.getByName(col->name); evaluated_col.column = evaluated_col.column->convertToFullColumnIfConst(); - block.insert(pos, std::move(evaluated_col)); + if (block.has(col->name)) + block.getByName(col->name) = std::move(evaluated_col); + else + block.insert(pos, std::move(evaluated_col)); } } } } + +void performRequiredConversions(Block & block, const NamesAndTypesList & required_columns, const Context & context) +{ + ASTPtr conversion_expr_list = convertRequiredExpressions(block, required_columns); + //std::cerr << queryToString(conversion_expr_list) << std::endl; + //std::cerr << "Block:" << block.dumpStructure() << std::endl; + if (conversion_expr_list->children.empty()) + return; + executeExpressionsOnBlock(block, conversion_expr_list, true, required_columns, context); +} + +void evaluateMissingDefaults(Block & block, + const NamesAndTypesList & required_columns, + const ColumnDefaults & column_defaults, + const Context & context, bool save_unneeded_columns) +{ + if (column_defaults.empty()) + return; + + ASTPtr default_expr_list = defaultRequiredExpressions(block, required_columns, column_defaults); + executeExpressionsOnBlock(block, default_expr_list, save_unneeded_columns, required_columns, context); +} + +} diff --git a/dbms/src/Interpreters/evaluateMissingDefaults.h b/dbms/src/Interpreters/evaluateMissingDefaults.h index 320fb35c9cb..51db620c86f 100644 --- a/dbms/src/Interpreters/evaluateMissingDefaults.h +++ b/dbms/src/Interpreters/evaluateMissingDefaults.h @@ -17,4 +17,7 @@ void evaluateMissingDefaults(Block & block, const std::unordered_map & column_defaults, const Context & context, bool save_unneeded_columns = true); +void performRequiredConversions(Block & block, + const NamesAndTypesList & required_columns, + const Context & context); } diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 58e21caedd6..b663f2a2322 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -49,7 +49,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ if (command_ast->type == ASTAlterCommand::ADD_COLUMN) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = AlterCommand::ADD_COLUMN; const auto & ast_col_decl = command_ast->col_decl->as(); @@ -90,7 +90,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = AlterCommand::DROP_COLUMN; command.column_name = getIdentifierName(command_ast->column); command.if_exists = command_ast->if_exists; @@ -99,7 +99,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_COLUMN) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = AlterCommand::MODIFY_COLUMN; const auto & ast_col_decl = command_ast->col_decl->as(); @@ -135,7 +135,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::COMMENT_COLUMN) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = COMMENT_COLUMN; command.column_name = getIdentifierName(command_ast->column); const auto & ast_comment = command_ast->comment->as(); @@ -146,7 +146,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_ORDER_BY) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = AlterCommand::MODIFY_ORDER_BY; command.order_by = command_ast->order_by; return command; @@ -154,7 +154,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::ADD_INDEX) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.index_decl = command_ast->index_decl; command.type = AlterCommand::ADD_INDEX; @@ -172,7 +172,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::ADD_CONSTRAINT) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.constraint_decl = command_ast->constraint_decl; command.type = AlterCommand::ADD_CONSTRAINT; @@ -190,7 +190,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ throw Exception("\"ALTER TABLE table CLEAR COLUMN column\" queries are not supported yet. Use \"CLEAR COLUMN column IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.if_exists = command_ast->if_exists; command.type = AlterCommand::DROP_CONSTRAINT; command.constraint_name = command_ast->constraint->as().name; @@ -203,7 +203,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ throw Exception("\"ALTER TABLE table CLEAR INDEX index\" queries are not supported yet. Use \"CLEAR INDEX index IN PARTITION\".", ErrorCodes::NOT_IMPLEMENTED); AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = AlterCommand::DROP_INDEX; command.index_name = command_ast->index->as().name; command.if_exists = command_ast->if_exists; @@ -213,7 +213,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_TTL) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = AlterCommand::MODIFY_TTL; command.ttl = command_ast->ttl; return command; @@ -221,7 +221,7 @@ std::optional AlterCommand::parse(const ASTAlterCommand * command_ else if (command_ast->type == ASTAlterCommand::MODIFY_SETTING) { AlterCommand command; - command.ast = command_ast; + command.ast = command_ast->clone(); command.type = AlterCommand::MODIFY_SETTING; command.settings_changes = command_ast->settings_changes->as().changes; return command; @@ -499,12 +499,18 @@ bool isMetadataOnlyConversion(const IDataType * from, const IDataType * to) bool AlterCommand::isRequireMutationStage(const StorageInMemoryMetadata & metadata) const { + if (ignore) + return false; + + if (type == DROP_COLUMN) + return true; + if (type != MODIFY_COLUMN || data_type == nullptr) return false; for (const auto & column : metadata.columns.getAllPhysical()) { - if (column.name == column_name && !isMetadataOnlyConversion(column.type, data_type)) + if (column.name == column_name && !isMetadataOnlyConversion(column.type.get(), data_type.get())) return true; } return false; @@ -534,11 +540,11 @@ std::optional AlterCommand::tryConvertToMutationCommand(const S MutationCommand result; - result.type = MutationCommand::Type::CAST; + result.type = MutationCommand::Type::READ; result.column_name = column_name; result.data_type = data_type; result.predicate = nullptr; - result.ast = ast; + result.ast = ast->clone(); return result; } @@ -835,7 +841,7 @@ bool AlterCommands::isCommentAlter() const } -MutationCommands getMutationCommands(const StorageInMemoryMetadata & metadata) const +MutationCommands AlterCommands::getMutationCommands(const StorageInMemoryMetadata & metadata) const { MutationCommands result; for (const auto & alter_cmd : *this) diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index e48e9896597..13f86b9750d 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -101,6 +101,7 @@ Block IStorage::getSampleBlockForColumns(const Names & column_names) const Block res; NamesAndTypesList all_columns = getColumns().getAll(); + //std::cerr << "ALL LLLL COLUMNS:" << all_columns.toString() << std::endl; std::unordered_map columns_map; for (const auto & elem : all_columns) columns_map.emplace(elem.name, elem.type); @@ -120,6 +121,7 @@ Block IStorage::getSampleBlockForColumns(const Names & column_names) const } } + //std::cerr << "RES:" << res.dumpStructure() << std::endl; return res; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index a519e2a4b71..963296dd30f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -44,6 +44,8 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( save_marks_in_cache(save_marks_in_cache_), virt_column_names(virt_column_names_) { + //std::cerr << "HEADER IN SELECT PROCESSOR:" << getPort().getHeader().dumpStructure() << std::endl; + //std::cerr << "STACK:" << StackTrace().toString() << std::endl; header_without_virtual_columns = getPort().getHeader(); for (auto it = virt_column_names.rbegin(); it != virt_column_names.rend(); ++it) diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 920697f3c32..05e137ddb55 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -241,10 +241,10 @@ MergeTreeReadTaskColumns getReadTaskColumns(const MergeTreeData & storage, const { /// Under owned_data_part->columns_lock we check that all requested columns are of the same type as in the table. /// This may be not true in case of ALTER MODIFY. - if (!pre_column_names.empty()) - storage.check(data_part->columns, pre_column_names); - if (!column_names.empty()) - storage.check(data_part->columns, column_names); + //if (!pre_column_names.empty()) + // storage.check(data_part->columns, pre_column_names); + //if (!column_names.empty()) + // storage.check(data_part->columns, column_names); const NamesAndTypesList & physical_columns = storage.getColumns().getAllPhysical(); result.pre_columns = physical_columns.addTypes(pre_column_names); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 3b7c90ff9fb..7a0af8fab70 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1536,15 +1536,15 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S } } - if (commands.isModifyingData()) - { - /// Check that type conversions are possible. - ExpressionActionsPtr unused_expression; - NameToNameMap unused_map; - bool unused_bool; - createConvertExpression(nullptr, getColumns().getAllPhysical(), metadata.columns.getAllPhysical(), - getIndices().indices, metadata.indices.indices, unused_expression, unused_map, unused_bool); - } + //if (commands.isModifyingData()) + //{ + // /// Check that type conversions are possible. + // ExpressionActionsPtr unused_expression; + // NameToNameMap unused_map; + // bool unused_bool; + // createConvertExpression(nullptr, getColumns().getAllPhysical(), metadata.columns.getAllPhysical(), + // getIndices().indices, metadata.indices.indices, unused_expression, unused_map, unused_bool); + //} } void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 3295c863ef9..6e8756b1c81 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -962,6 +962,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared( data, space_reservation->getDisk(), future_part.name, future_part.part_info); + new_data_part->relative_path = "tmp_mut_" + future_part.name; new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; @@ -988,6 +989,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor const auto data_settings = data.getSettings(); Block in_header = in->getHeader(); + std::cerr << "Mutations header:" << in_header.dumpStructure() << std::endl; UInt64 watch_prev_elapsed = 0; MergeStageProgress stage_progress(1.0); @@ -1069,6 +1071,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor /// Don't change granularity type while mutating subset of columns auto mrk_extension = source_part->index_granularity_info.is_adaptive ? getAdaptiveMrkExtension() : getNonAdaptiveMrkExtension(); + + /// Skip updated files for (const auto & entry : updated_header) { IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) @@ -1087,6 +1091,33 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor files_to_skip.insert(index->getFileName() + mrk_extension); } + /// TODO(alesap) better + for (const auto & part_column : source_part->columns) + { + bool found = false; + for (const auto & all_column : all_columns) + { + if (part_column.name == all_column.name) + { + found = true; + break; + } + } + if (!found) + { + std::cerr << "REMOVING COLUMN:" << part_column.name << std::endl; + IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) + { + String stream_name = IDataType::getFileNameForStream(part_column.name, substream_path); + files_to_skip.insert(stream_name + ".bin"); + files_to_skip.insert(stream_name + mrk_extension); + }; + + IDataType::SubstreamPath stream_path; + part_column.type->enumerateStreams(callback, stream_path); + } + } + Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator dir_it(source_part->getFullPath()); dir_it != dir_end; ++dir_it) { @@ -1101,37 +1132,46 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor merge_entry->columns_written = all_columns.size() - updated_header.columns(); - IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; - MergedColumnOnlyOutputStream out( - data, - updated_header, - new_part_tmp_path, - /* sync = */ false, - compression_codec, - /* skip_offsets = */ false, - std::vector(indices_to_recalc.begin(), indices_to_recalc.end()), - unused_written_offsets, - source_part->index_granularity, - &source_part->index_granularity_info - ); - - in->readPrefix(); - out.writePrefix(); - - Block block; - while (check_not_cancelled() && (block = in->read())) + new_data_part->checksums = source_part->checksums; + if (updated_header.columns() != 0) { - out.write(block); + IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; + MergedColumnOnlyOutputStream out( + data, + updated_header, + new_part_tmp_path, + /* sync = */ false, + compression_codec, + /* skip_offsets = */ false, + std::vector(indices_to_recalc.begin(), indices_to_recalc.end()), + unused_written_offsets, + source_part->index_granularity, + &source_part->index_granularity_info + ); - merge_entry->rows_written += block.rows(); - merge_entry->bytes_written_uncompressed += block.bytes(); + in->readPrefix(); + out.writePrefix(); + + Block block; + while (check_not_cancelled() && (block = in->read())) + { + out.write(block); + + merge_entry->rows_written += block.rows(); + merge_entry->bytes_written_uncompressed += block.bytes(); + } + + in->readSuffix(); + + auto changed_checksums = out.writeSuffixAndGetChecksums(); + + new_data_part->checksums.add(std::move(changed_checksums)); } - in->readSuffix(); - auto changed_checksums = out.writeSuffixAndGetChecksums(); + for (const String & file_to_skip : files_to_skip) + if (new_data_part->checksums.files.count(file_to_skip)) + new_data_part->checksums.files.erase(file_to_skip); - new_data_part->checksums = source_part->checksums; - new_data_part->checksums.add(std::move(changed_checksums)); { /// Write file with checksums. WriteBufferFromFile out_checksums(new_part_tmp_path + "checksums.txt", 4096); @@ -1144,8 +1184,17 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor NameSet source_columns_name_set(source_column_names.begin(), source_column_names.end()); for (auto it = new_data_part->columns.begin(); it != new_data_part->columns.end();) { - if (source_columns_name_set.count(it->name) || updated_header.has(it->name)) + if (updated_header.has(it->name)) + { + auto updated_type = updated_header.getByName(it->name).type; + if (updated_type != it->type) + it->type = updated_type; ++it; + } + else if (source_columns_name_set.count(it->name)) + { + ++it; + } else it = new_data_part->columns.erase(it); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 09c4fe835d6..043e389458a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -644,6 +644,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( } else { + std::cerr << "Spreading marks among streams\n"; res = spreadMarkRangesAmongStreams( std::move(parts_with_ranges), num_streams, @@ -763,6 +764,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( num_streams, sum_marks, min_marks_for_concurrent_read, parts, data, query_info.prewhere_info, true, column_names, MergeTreeReadPool::BackoffSettings(settings), settings.preferred_block_size_bytes, false); + std::cerr << "POOL HEADER:" << pool->getHeader().dumpStructure() << std::endl; /// Let's estimate total number of rows for progress bar. LOG_TRACE(log, "Reading approx. " << total_rows << " rows with " << num_streams << " streams"); @@ -790,6 +792,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( { RangesInDataPart & part = parts[part_index]; + std::cerr << "Creating sequential stream from part:" << part_index << std::endl; auto source = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, part.ranges, use_uncompressed_cache, diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index a09bd548b64..b78f2bc49eb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -618,6 +618,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar } merge_tree_reader->evaluateMissingDefaults(block, columns); + merge_tree_reader->performRequiredConversions(columns); } read_result.columns.reserve(read_result.columns.size() + columns.size()); @@ -637,6 +638,8 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar if (should_evaluate_missing_defaults) merge_tree_reader->evaluateMissingDefaults({}, read_result.columns); + + merge_tree_reader->performRequiredConversions(read_result.columns); } else read_result.columns.clear(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index d308667a67b..e22b10b8319 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -155,6 +155,7 @@ MarkRanges MergeTreeReadPool::getRestMarks(const MergeTreeDataPart & part, const Block MergeTreeReadPool::getHeader() const { + //////////////////std::cerr << "COLUMN NAMES IN POOL:" << column_names.front() << std::endl; return data.getSampleBlockForColumns(column_names); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 72c31a9dcb1..f3dcf3adafc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -56,10 +56,32 @@ MergeTreeReader::MergeTreeReader( , mmap_threshold(mmap_threshold_) , max_read_buffer_size(max_read_buffer_size_) { + //std::cerr << "Merge tree reader created for part:" << data_part->name << std::endl; try { + for (const NameAndTypePair & column_from_part : data_part->columns) + { + columns_from_part[column_from_part.name] = column_from_part.type; + } + for (const NameAndTypePair & column : columns) - addStreams(column.name, *column.type, profile_callback_, clock_type_); + { + //std::cerr << "Column name to read:" << column.name << std::endl; + if (columns_from_part.count(column.name)) + { + //std::cerr << "With type:" << columns_from_part[column.name]->getName() << std::endl; + //std::cerr << "Original type:" << column.type->getName() << std::endl; + + addStreams(column.name, *columns_from_part[column.name], profile_callback_, clock_type_); + } + else + { + //std::cerr << "Original type:" << column.type->getName() << std::endl; + addStreams(column.name, *column.type, profile_callback_, clock_type_); + } + + } + //std::cerr << "COLUMNS IN CONSTRUCTOR:" << columns.toString() << std::endl; } catch (...) { @@ -95,12 +117,17 @@ size_t MergeTreeReader::readRows(size_t from_mark, bool continue_reading, size_t auto name_and_type = columns.begin(); for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) { - auto & [name, type] = *name_and_type; + String & name = name_and_type->name; + DataTypePtr type; + if (columns_from_part.count(name)) + type = columns_from_part[name]; + else + type = name_and_type->type; /// The column is already present in the block so we will append the values to the end. bool append = res_columns[pos] != nullptr; if (!append) - res_columns[pos] = name_and_type->type->createColumn(); + res_columns[pos] = type->createColumn(); /// To keep offsets shared. TODO Very dangerous. Get rid of this. MutableColumnPtr column = res_columns[pos]->assumeMutable(); @@ -214,6 +241,7 @@ void MergeTreeReader::readData( size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) { + //std::cerr << "READ DATA:" << name << " with type:" << type.getName() << std::endl; auto get_stream_getter = [&](bool stream_for_prefix) -> IDataType::InputStreamGetter { return [&, stream_for_prefix](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * @@ -370,7 +398,7 @@ void MergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns size_t num_columns = columns.size(); if (res_columns.size() != num_columns) - throw Exception("invalid number of columns passed to MergeTreeReader::fillMissingColumns. " + throw Exception("invalid number of columns passed to MergeTreeReader::evaluateMissingDefaults. " "Expected " + toString(num_columns) + ", " "got " + toString(res_columns.size()), ErrorCodes::LOGICAL_ERROR); @@ -400,4 +428,65 @@ void MergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns } } +void MergeTreeReader::performRequiredConversions(Columns & res_columns) +{ + try + { + size_t num_columns = columns.size(); + + if (res_columns.size() != num_columns) + { + throw Exception( + "invalid number of columns passed to MergeTreeReader::performRequiredConversions. " + "Expected " + + toString(num_columns) + + ", " + "got " + + toString(res_columns.size()), + ErrorCodes::LOGICAL_ERROR); + } + + Block copy_block; + auto name_and_type = columns.begin(); + //std::cerr << "DATAPART NAMES AND TYPES:" << data_part->columns.toString() << std::endl; + //std::cerr << "REQUIRED COLUMNS NAMES AND TYPES:" << columns.toString() << std::endl; + //std::cerr << "RES COLUMNS SIZE:" << res_columns.size() << std::endl; + //std::cerr << "RES COLUMNS STRUCTURE:\n"; + //for (const auto & column : res_columns) + //{ + // std::cerr << column->dumpStructure() << std::endl; + //} + + for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) + { + //std::cerr << "POS:" << pos << std::endl; + if (res_columns[pos] == nullptr) + continue; + + //std::cerr << "POS NAME:" << name_and_type->name << std::endl; + //std::cerr << "POS TYPE:" << name_and_type->type->getName() << std::endl; + if (columns_from_part.count(name_and_type->name)) + copy_block.insert({res_columns[pos], columns_from_part[name_and_type->name], name_and_type->name}); + else + copy_block.insert({res_columns[pos], name_and_type->type, name_and_type->name}); + } + + //std::cerr << "Copy block: " << copy_block.dumpStructure() << std::endl; + DB::performRequiredConversions(copy_block, columns, storage.global_context); + std::cerr << "Result copy block: " << copy_block.dumpStructure() << std::endl; + + /// Move columns from block. + name_and_type = columns.begin(); + for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) + { + res_columns[pos] = std::move(copy_block.getByName(name_and_type->name).column); + } + } + catch(Exception & e) + { + /// Better diagnostics. + e.addMessage("(while reading from part " + path + ")"); + throw; + } +} } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.h b/dbms/src/Storages/MergeTree/MergeTreeReader.h index b0642c06108..d0562fe3300 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.h @@ -45,6 +45,9 @@ public: /// Evaluate defaulted columns if necessary. void evaluateMissingDefaults(Block additional_columns, Columns & res_columns); + /// Perform conversions TODO(alesap) + void performRequiredConversions(Columns & res_columns); + const NamesAndTypesList & getColumns() const { return columns; } size_t numColumnsInResult() const { return columns.size(); } @@ -74,6 +77,8 @@ private: /// Columns that are read. NamesAndTypesList columns; + std::unordered_map columns_from_part; + UncompressedCache * uncompressed_cache; MarkCache * mark_cache; /// If save_marks_in_cache is false, then, if marks are not in cache, we will load them but won't save in the cache, to avoid evicting other data. diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index dac42859eef..8c483c3fa68 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -11,25 +11,25 @@ namespace ErrorCodes extern const int MEMORY_LIMIT_EXCEEDED; } -static Block replaceTypes(Block && header, const MergeTreeData::DataPartPtr & data_part) -{ - /// Types may be different during ALTER (when this stream is used to perform an ALTER). - /// NOTE: We may use similar code to implement non blocking ALTERs. - for (const auto & name_type : data_part->columns) - { - if (header.has(name_type.name)) - { - auto & elem = header.getByName(name_type.name); - if (!elem.type->equals(*name_type.type)) - { - elem.type = name_type.type; - elem.column = elem.type->createColumn(); - } - } - } - - return std::move(header); -} +//static Block replaceTypes(Block && header, const MergeTreeData::DataPartPtr & data_part) +//{ +// /// Types may be different during ALTER (when this stream is used to perform an ALTER). +// /// NOTE: We may use similar code to implement non blocking ALTERs. +// for (const auto & name_type : data_part->columns) +// { +// if (header.has(name_type.name)) +// { +// auto & elem = header.getByName(name_type.name); +// if (!elem.type->equals(*name_type.type)) +// { +// elem.type = name_type.type; +// elem.column = elem.type->createColumn(); +// } +// } +// } +// +// return std::move(header); +//} MergeTreeSelectProcessor::MergeTreeSelectProcessor( const MergeTreeData & storage_, @@ -51,7 +51,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( bool quiet) : MergeTreeBaseSelectProcessor{ - replaceTypes(storage_.getSampleBlockForColumns(required_columns_), owned_data_part_), + storage_.getSampleBlockForColumns(required_columns_), storage_, prewhere_info_, max_block_size_rows_, preferred_block_size_bytes_, preferred_max_column_in_block_size_bytes_, min_bytes_to_use_direct_io_, min_bytes_to_use_mmap_io_, max_read_buffer_size_, use_uncompressed_cache_, save_marks_in_cache_, virt_column_names_}, diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 372c29a3ac3..08beda32d74 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -35,18 +35,20 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( addTotalRowsApprox(data_part->rows_count); header = storage.getSampleBlockForColumns(columns_to_read); - fixHeader(header); + //fixHeader(header); /// Add columns because we don't want to read empty blocks injectRequiredColumns(storage, data_part, columns_to_read); NamesAndTypesList columns_for_reader; if (take_column_types_from_storage) { + std::cerr << "Taking columns from storage\n"; const NamesAndTypesList & physical_columns = storage.getColumns().getAllPhysical(); columns_for_reader = physical_columns.addTypes(columns_to_read); } else { + std::cerr << "Taking columns from data part\n"; /// take columns from data_part columns_for_reader = data_part->columns.addTypes(columns_to_read); } @@ -107,6 +109,8 @@ try if (should_evaluate_missing_defaults) reader->evaluateMissingDefaults({}, columns); + reader->performRequiredConversions(columns); + res = header.cloneEmpty(); /// Reorder columns and fill result block. diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 0599a35905d..7c6cfa42873 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -67,10 +67,10 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const case FINISH_ALTER: /// Just make local /metadata and /columns consistent with global out << "alter\n"; - for (const String & s : source_parts) - out << s << '\n'; - out << "finish"; + out << required_mutation_znode << "\n"; + out << "finish\n"; break; + default: throw Exception("Unknown log entry type: " + DB::toString(type), ErrorCodes::LOGICAL_ERROR); } @@ -161,18 +161,14 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) else if (type_str == "alter") { type = FINISH_ALTER; - while (!in.eof()) - { - String s; - in >> s >> "\n"; - if (s == "finish") - break; - source_parts.push_back(s); - } + in >> required_mutation_znode >> "\nfinish\n"; } + std::cerr << "Read backn\n"; in >> "\n"; + std::cerr << "Readed\n"; + /// Optional field. if (!in.eof()) in >> "quorum: " >> quorum >> "\n"; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index 7d63c20ce9b..f7744d5a110 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -89,6 +89,9 @@ struct ReplicatedMergeTreeLogEntryData /// For DROP_RANGE, true means that the parts need not be deleted, but moved to the `detached` directory. bool detach = false; + /// For ALTER TODO(alesap) + String required_mutation_znode; + /// REPLACE PARTITION FROM command struct ReplaceRangeEntry { @@ -111,6 +114,10 @@ struct ReplicatedMergeTreeLogEntryData /// selection of merges. These parts are added to queue.virtual_parts. Strings getVirtualPartNames() const { + /// Doesn't produce any part + if (type == FINISH_ALTER) + return {}; + /// DROP_RANGE does not add a real part, but we must disable merges in that range if (type == DROP_RANGE) return {new_part_name}; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 41e1d408b52..9af93996c0c 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -517,6 +517,8 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C } catch (...) { + std::cerr << "DIE HERE:\n"; + tryLogCurrentException(log); /// If it fails, the data in RAM is incorrect. In order to avoid possible further corruption of data in ZK, we will kill ourselves. /// This is possible only if there is an unknown logical error. std::terminate(); @@ -532,7 +534,8 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C } -static Names getPartNamesToMutate( +namespace { +Names getPartNamesToMutate( const ReplicatedMergeTreeMutationEntry & mutation, const ActiveDataPartSet & parts) { Names result; @@ -556,8 +559,9 @@ static Names getPartNamesToMutate( return result; } +} -Names getPartNamesToMutate(ReplicatedMergeTreeMutationEntry & entry) const +Names ReplicatedMergeTreeQueue::getCurrentPartNamesToMutate(ReplicatedMergeTreeMutationEntry & entry) const { return getPartNamesToMutate(entry, current_parts); } @@ -1008,16 +1012,12 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (entry.type == LogEntry::FINISH_ALTER) { - for (const auto & name : entry.source_parts) - { - if (future_parts.count(name)) - { - String reason = "Not altering storage because part " + name - + " is not ready yet (log entry for that part is being processed)."; - LOG_TRACE(log, reason); - out_postpone_reason = reason; - return false; - } + std::cerr << "Entry finish alter\n"; + if (mutations_by_znode.count(entry.required_mutation_znode) && !mutations_by_znode.at(entry.required_mutation_znode).is_done) { + String reason = "Not altering storage because mutation " + entry.required_mutation_znode + " is not ready yet (mutation is beeing processed)."; + LOG_TRACE(log, reason); + out_postpone_reason = reason; + return false; } } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 6ed2f1889ed..cad23df6f46 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -117,6 +117,7 @@ private: String latest_fail_reason; }; + /// Mapping from znode path to Mutations Status std::map mutations_by_znode; std::unordered_map> mutations_by_partition; /// Znode ID of the latest mutation that is done. @@ -339,7 +340,7 @@ public: /// Adds a subscriber SubscriberHandler addSubscriber(SubscriberCallBack && callback); - Names getPartNamesToMutate(ReplicatedMergeTreeMutationEntry & entry) const; + Names getCurrentPartNamesToMutate(ReplicatedMergeTreeMutationEntry & entry) const; struct Status { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index fce4479c16f..5789bd1f9a7 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -199,7 +199,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() storage.mutations_updating_task->activateAndSchedule(); storage.mutations_finalizing_task->activateAndSchedule(); storage.cleanup_thread.start(); - storage.alter_thread.start(); + //storage.alter_thread.start(); storage.part_check_thread.start(); return true; @@ -346,7 +346,7 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown() storage.mutations_finalizing_task->deactivate(); storage.cleanup_thread.stop(); - storage.alter_thread.stop(); + //storage.alter_thread.stop(); storage.part_check_thread.stop(); LOG_TRACE(log, "Threads finished"); diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 6865cc956fa..2ad4abf607b 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -39,6 +39,7 @@ public: for (auto & pipe : pipes) streams.emplace_back(std::make_shared(std::move(pipe))); + //std::cerr << "Streams header:" << streams.back()->getHeader().dumpStructure() << std::endl; return streams; } diff --git a/dbms/src/Storages/MutationCommands.cpp b/dbms/src/Storages/MutationCommands.cpp index f8bc781f166..2d261a8c0b7 100644 --- a/dbms/src/Storages/MutationCommands.cpp +++ b/dbms/src/Storages/MutationCommands.cpp @@ -2,12 +2,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include namespace DB @@ -19,7 +21,7 @@ namespace ErrorCodes extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN; } -std::optional MutationCommand::parse(ASTAlterCommand * command) +std::optional MutationCommand::parse(ASTAlterCommand * command, bool parse_modify) { if (command->type == ASTAlterCommand::DELETE) { @@ -55,6 +57,24 @@ std::optional MutationCommand::parse(ASTAlterCommand * command) res.index_name = command->index->as().name; return res; } + else if (parse_modify && command->type == ASTAlterCommand::MODIFY_COLUMN) + { + MutationCommand res; + res.ast = command->ptr(); + res.type = MutationCommand::Type::READ; + const auto & ast_col_decl = command->col_decl->as(); + res.column_name = ast_col_decl.name; + res.data_type = DataTypeFactory::instance().get(ast_col_decl.type); + return res; + } + else if (parse_modify && command->type == ASTAlterCommand::DROP_COLUMN) + { + MutationCommand res; + res.ast = command->ptr(); + res.type = MutationCommand::Type::READ; + res.column_name = getIdentifierName(command->column); + return res; + } else return {}; } @@ -85,7 +105,7 @@ void MutationCommands::readText(ReadBuffer & in) p_alter_commands, commands_str.data(), commands_str.data() + commands_str.length(), "mutation commands list", 0); for (ASTAlterCommand * command_ast : commands_ast->as().commands) { - auto command = MutationCommand::parse(command_ast); + auto command = MutationCommand::parse(command_ast, true); if (!command) throw Exception("Unknown mutation command type: " + DB::toString(command_ast->type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); push_back(std::move(*command)); diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index ce84c08e631..d808fbca4d5 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -27,7 +27,7 @@ struct MutationCommand DELETE, UPDATE, MATERIALIZE_INDEX, - CAST /// for ALTER MODIFY column + READ }; Type type = EMPTY; @@ -46,7 +46,7 @@ struct MutationCommand String column_name; DataTypePtr data_type; - static std::optional parse(ASTAlterCommand * command); + static std::optional parse(ASTAlterCommand * command, bool parse_modify=false); }; /// Multiple mutation commands, possible from different ALTER queries diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 3998189b81a..15318aecefb 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -208,7 +208,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( zookeeper_path(global_context.getMacros()->expand(zookeeper_path_, database_name_, table_name_)), replica_name(global_context.getMacros()->expand(replica_name_, database_name_, table_name_)), reader(*this), writer(*this), merger_mutator(*this, global_context.getBackgroundPool().getNumberOfThreads()), - queue(*this), fetcher(*this), cleanup_thread(*this), alter_thread(*this), + queue(*this), fetcher(*this), cleanup_thread(*this), part_check_thread(*this), restarting_thread(*this) { if (!zookeeper_path.empty() && zookeeper_path.back() == '/') @@ -1156,34 +1156,37 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } -bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree::LogEntry & entry) +bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree::LogEntry & /*entry*/) { + std::cerr << "Trying to finish alter\n"; auto zookeeper = getZooKeeper(); String columns_path = zookeeper_path + "/columns"; - auto columns_znode = zookeeper->get(columns_path); - if (!columns_znode.exists) + String columns_str; + Coordination::Stat columns_znode_stat; + if (!zookeeper->tryGet(columns_path, columns_str, &columns_znode_stat)) throw Exception(columns_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - int32_t columns_version = columns_znode.stat.version; + int32_t columns_version_zk = columns_znode_stat.version; String metadata_path = zookeeper_path + "/metadata"; - auto metadata_znode = zookeeper->get(metadata_path); - if (!metadata_znode.exists) - throw Exception(metadata_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - int32_t metadata_version = metadata_znode.stat.version; + String metadata_str; + Coordination::Stat metadata_znode_stat; - const bool changed_columns_version = (columns_version != storage.columns_version); - const bool changed_metadata_version = (metadata_version != storage.metadata_version); + if (!zookeeper->tryGet(metadata_path, metadata_str, &metadata_znode_stat)) + throw Exception(metadata_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); + int32_t metadata_version_zk = metadata_znode_stat.version; + + const bool changed_columns_version = (columns_version_zk != this->columns_version); + const bool changed_metadata_version = (metadata_version_zk != this->metadata_version); + + std::cerr << "Versions changed: columns" << changed_columns_version << " metadata:" << changed_metadata_version << std::endl; if (!(changed_columns_version || changed_metadata_version)) - return; + return true; - const String & columns_str = columns_znode.contents; auto columns_in_zk = ColumnsDescription::parse(columns_str); - - const String & metadata_str = metadata_znode.contents; auto metadata_in_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); - auto metadata_diff = ReplicatedMergeTreeTableMetadata(storage).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); + auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); MergeTreeData::DataParts parts; @@ -1194,7 +1197,7 @@ bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); - if (columns_in_zk == storage.getColumns() && metadata_diff.empty()) + if (columns_in_zk == getColumns() && metadata_diff.empty()) { LOG_INFO( log, @@ -1210,8 +1213,8 @@ bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree LOG_INFO(log, "Applied changes to the metadata of the table."); } - columns_version = columns_version; - metadata_version = metadata_version; + this->columns_version = columns_version_zk; + this->metadata_version = metadata_version_zk; recalculateColumnSizes(); /// Update metadata ZK nodes for a specific replica. @@ -1220,6 +1223,7 @@ bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree if (changed_metadata_version) zookeeper->set(replica_path + "/metadata", metadata_str); } + return true; } bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedMergeTree::LogEntry & entry) @@ -3291,13 +3295,32 @@ void StorageReplicatedMergeTree::alter( } - auto ast_to_str = [](ASTPtr query) -> String + struct ChangedNode { + ChangedNode(const String & table_path_, String name_, String new_value_) + : table_path(table_path_), name(std::move(name_)), shared_path(table_path + "/" + name), new_value(std::move(new_value_)) + { + } + + const String & table_path; + String name; + + String shared_path; + + String new_value; + int32_t new_version = -1; /// Initialization is to suppress (useless) false positive warning found by cppcheck. + }; + + auto ast_to_str = [](ASTPtr query) -> String { if (!query) return ""; return queryToString(query); }; + std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; + + /// /columns and /metadata nodes + std::vector changed_nodes; { /// Just to read current structure. Alter will be done in separate thread. auto table_lock = lockStructureForShare(false, query_context.getCurrentQueryId()); @@ -3357,215 +3380,21 @@ void StorageReplicatedMergeTree::alter( entry.type = LogEntry::FINISH_ALTER; entry.source_replica = replica_name; - if (maybe_mutation_commands) - { - ReplicatedMergeTreeMutationEntry entry = mutateImpl(*maybe_mutation_commands, context); - entry.source_parts = queue.getPartNamesToMutate(entry); - } + std::cerr << " Columns before mutation:" << getColumns().getAllPhysical().toString() << std::endl; - entry.new_part_name = new_part_name; + entry.new_part_name = ""; entry.create_time = time(nullptr); - zookeeper->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); + String path_created = getZooKeeper()->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); + entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); waitForAllReplicasToProcessLogEntry(entry); - ///// Wait until all replicas will apply ALTER. - - //for (const auto & node : changed_nodes) - //{ - // Coordination::Stat stat; - // /// Subscribe to change of shared ZK metadata nodes, to finish waiting if someone will do another ALTER. - // if (!getZooKeeper()->exists(node.shared_path, &stat, alter_query_event)) - // throw Exception(node.shared_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - - // if (stat.version != node.new_version) - // { - // LOG_WARNING(log, node.shared_path + " changed before this ALTER finished; " + - // "overlapping ALTER-s are fine but use caution with nontransitive changes"); - // return; - // } - //} - - //Strings replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); - - //std::set inactive_replicas; - //std::set timed_out_replicas; - - //time_t replication_alter_columns_timeout = query_context.getSettingsRef().replication_alter_columns_timeout; - - ///// This code is quite similar with waitMutationToFinishOnReplicas - ///// but contains more complicated details (versions manipulations, multiple nodes, etc.). - ///// It will be removed soon in favor of alter-modify implementation on top of mutations. - ///// TODO (alesap) - //for (const String & replica : replicas) - //{ - // LOG_DEBUG(log, "Waiting for " << replica << " to apply changes"); - - // while (!partial_shutdown_called) - // { - // auto zookeeper = getZooKeeper(); - - // /// Replica could be inactive. - // if (!zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) - // { - // LOG_WARNING(log, "Replica " << replica << " is not active during ALTER query." - // " ALTER will be done asynchronously when replica becomes active."); - - // inactive_replicas.emplace(replica); - // break; - // } - - // struct ReplicaNode - // { - // explicit ReplicaNode(String path_) : path(std::move(path_)) {} - - // String path; - // String value; - // int32_t version = -1; - // }; - - // std::vector replica_nodes; - // for (const auto & node : changed_nodes) - // replica_nodes.emplace_back(node.getReplicaPath(replica)); - - // bool replica_was_removed = false; - // for (auto & node : replica_nodes) - // { - // Coordination::Stat stat; - - // /// Replica could has been removed. - // if (!zookeeper->tryGet(node.path, node.value, &stat)) - // { - // LOG_WARNING(log, replica << " was removed"); - // replica_was_removed = true; - // break; - // } - - // node.version = stat.version; - // } - - // if (replica_was_removed) - // break; - - // bool alter_was_applied = true; - // for (size_t i = 0; i < replica_nodes.size(); ++i) - // { - // if (replica_nodes[i].value != changed_nodes[i].new_value) - // { - // alter_was_applied = false; - // break; - // } - // } - - // /// The ALTER has been successfully applied. - // if (alter_was_applied) - // break; - - // for (const auto & node : changed_nodes) - // { - // Coordination::Stat stat; - // if (!zookeeper->exists(node.shared_path, &stat)) - // throw Exception(node.shared_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - - // if (stat.version != node.new_version) - // { - // LOG_WARNING(log, node.shared_path + " changed before this ALTER finished; " - // "overlapping ALTER-s are fine but use caution with nontransitive changes"); - // return; - // } - // } - - // bool replica_nodes_changed_concurrently = false; - // for (const auto & replica_node : replica_nodes) - // { - // Coordination::Stat stat; - // if (!zookeeper->exists(replica_node.path, &stat, alter_query_event)) - // { - // LOG_WARNING(log, replica << " was removed"); - // replica_was_removed = true; - // break; - // } - - // if (stat.version != replica_node.version) - // { - // replica_nodes_changed_concurrently = true; - // break; - // } - // } - - // if (replica_was_removed) - // break; - - // if (replica_nodes_changed_concurrently) - // continue; - - // /// alter_query_event subscribed with zookeeper watch callback to /repliacs/{replica}/metadata - // /// and /replicas/{replica}/columns nodes for current relica + shared nodes /columns and /metadata, - // /// which is common for all replicas. If changes happen with this nodes (delete, set and create) - // /// than event will be notified and wait will be interrupted. - // /// - // /// ReplicatedMergeTreeAlterThread responsible for local /replicas/{replica}/metadata and - // /// /replicas/{replica}/columns changes. Shared /columns and /metadata nodes can be changed by *newer* - // /// concurrent alter from other replica. First of all it will update shared nodes and we will have no - // /// ability to identify, that our *current* alter finshed. So we cannot do anything better than just - // /// return from *current* alter with success result. - // if (!replication_alter_columns_timeout) - // { - // alter_query_event->wait(); - // /// Everything is fine. - // } - // else if (alter_query_event->tryWait(replication_alter_columns_timeout * 1000)) - // { - // /// Everything is fine. - // } - // else - // { - // LOG_WARNING(log, "Timeout when waiting for replica " << replica << " to apply ALTER." - // " ALTER will be done asynchronously."); - - // timed_out_replicas.emplace(replica); - // break; - // } - // } - - // if (partial_shutdown_called) - // throw Exception("Alter is not finished because table shutdown was called. Alter will be done after table restart.", - // ErrorCodes::UNFINISHED); - - // if (!inactive_replicas.empty() || !timed_out_replicas.empty()) - // { - // std::stringstream exception_message; - // exception_message << "Alter is not finished because"; - - // if (!inactive_replicas.empty()) - // { - // exception_message << " some replicas are inactive right now"; - - // for (auto it = inactive_replicas.begin(); it != inactive_replicas.end(); ++it) - // exception_message << (it == inactive_replicas.begin() ? ": " : ", ") << *it; - // } - - // if (!timed_out_replicas.empty() && !inactive_replicas.empty()) - // exception_message << " and"; - - // if (!timed_out_replicas.empty()) - // { - // exception_message << " timeout when waiting for some replicas"; - - // for (auto it = timed_out_replicas.begin(); it != timed_out_replicas.end(); ++it) - // exception_message << (it == timed_out_replicas.begin() ? ": " : ", ") << *it; - - // exception_message << " (replication_alter_columns_timeout = " << replication_alter_columns_timeout << ")"; - // } - - // exception_message << ". Alter will be done asynchronously."; - - // throw Exception(exception_message.str(), ErrorCodes::UNFINISHED); - // } - //} - - //LOG_DEBUG(log, "ALTER finished"); + if (!maybe_mutation_commands.empty()) + { + std::cerr << "We have mutation commands:" << maybe_mutation_commands.size() << std::endl; + ReplicatedMergeTreeMutationEntry mutation_entry = mutateImpl(maybe_mutation_commands, query_context); + } } void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) @@ -4519,10 +4348,10 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const Context & query_context) { - mutateImpl(commands, context); + mutateImpl(commands, query_context); } -StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & query_context) +ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & query_context) { /// Overview of the mutation algorithm. /// @@ -4627,16 +4456,16 @@ StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const } /// we have to wait - if (query_context.getSettingsRef().mutations_sync != 0) - { + //if (query_context.getSettingsRef().mutations_sync != 0) + //{ Strings replicas; if (query_context.getSettingsRef().mutations_sync == 2) /// wait for all replicas replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); - else if (query_context.getSettingsRef().mutations_sync == 1) /// just wait for ourself - replicas.push_back(replica_path); + //else if (query_context.getSettingsRef().mutations_sync == 1) /// just wait for ourself + // replicas.push_back(replica_path); waitMutationToFinishOnReplicas(replicas, entry.znode_name); - } + //} return entry; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 0ccdf046892..6ad7b08b34e 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -288,9 +288,6 @@ private: /// A thread that removes old parts, log entries, and blocks. ReplicatedMergeTreeCleanupThread cleanup_thread; - /// A thread monitoring changes to the column list in ZooKeeper and updating the parts in accordance with these changes. - ReplicatedMergeTreeAlterThread alter_thread; - /// A thread that checks the data of the parts, as well as the queue of the parts to be checked. ReplicatedMergeTreePartCheckThread part_check_thread; From 92648955c4b2d69319a849d03f81c79037d094ff Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 15 Jan 2020 16:47:00 +0300 Subject: [PATCH 0105/2007] Almost working drop --- .../src/Interpreters/MutationsInterpreter.cpp | 1 - .../MergeTree/MergeTreeDataMergerMutator.cpp | 45 ++++++++++++------- .../Storages/MergeTree/MergeTreeReader.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 3 +- 4 files changed, 32 insertions(+), 19 deletions(-) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 5ae7a8996d4..5cfcd3f0504 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -399,7 +399,6 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) if (stages.size() == 1) /// First stage only supports filtering and can't update columns. stages.emplace_back(context); - /// TODO(alesap) if (command.data_type) stages.back().column_to_updated.emplace(command.column_name, std::make_shared(command.column_name)); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 6e8756b1c81..50387bb4c32 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -980,20 +980,24 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor static_cast(source_part->bytes_on_disk) / data.getTotalActiveSizeInBytes()); Poco::File(new_part_tmp_path).createDirectories(); + BlockInputStreamPtr in = nullptr; + Block updated_header; + + if(!std::all_of(commands_for_part.begin(), commands_for_part.end(), [](const auto & cmd) { return cmd.type == MutationCommand::Type::READ && cmd.data_type == nullptr;})) + { + MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading, true); + in = mutations_interpreter.execute(table_lock_holder); + updated_header = mutations_interpreter.getUpdatedHeader(); + } - MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading, true); - auto in = mutations_interpreter.execute(table_lock_holder); - const auto & updated_header = mutations_interpreter.getUpdatedHeader(); NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); const auto data_settings = data.getSettings(); - Block in_header = in->getHeader(); - std::cerr << "Mutations header:" << in_header.dumpStructure() << std::endl; - UInt64 watch_prev_elapsed = 0; MergeStageProgress stage_progress(1.0); - in->setProgressCallback(MergeProgressCallback(merge_entry, watch_prev_elapsed, stage_progress)); + if (in) + in->setProgressCallback(MergeProgressCallback(merge_entry, watch_prev_elapsed, stage_progress)); if (updated_header.columns() == all_columns.size()) { @@ -1033,7 +1037,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor /// Checks if columns used in skipping indexes modified. std::set indices_to_recalc; ASTPtr indices_recalc_expr_list = std::make_shared(); - for (const auto & col : in_header.getNames()) + for (const auto & col : updated_header.getNames()) { for (size_t i = 0; i < data.skip_indices.size(); ++i) { @@ -1052,8 +1056,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (!indices_to_recalc.empty()) { - auto indices_recalc_syntax = SyntaxAnalyzer(context, {}).analyze( - indices_recalc_expr_list, in_header.getNamesAndTypesList()); + auto indices_recalc_syntax + = SyntaxAnalyzer(context, {}).analyze(indices_recalc_expr_list, updated_header.getNamesAndTypesList()); auto indices_recalc_expr = ExpressionAnalyzer( indices_recalc_expr_list, indices_recalc_syntax, context).getActions(false); @@ -1091,6 +1095,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor files_to_skip.insert(index->getFileName() + mrk_extension); } + std::unordered_set removed_columns; /// TODO(alesap) better for (const auto & part_column : source_part->columns) { @@ -1106,11 +1111,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (!found) { std::cerr << "REMOVING COLUMN:" << part_column.name << std::endl; + IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) { String stream_name = IDataType::getFileNameForStream(part_column.name, substream_path); - files_to_skip.insert(stream_name + ".bin"); - files_to_skip.insert(stream_name + mrk_extension); + removed_columns.insert(stream_name + ".bin"); + removed_columns.insert(stream_name + mrk_extension); }; IDataType::SubstreamPath stream_path; @@ -1133,8 +1139,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor merge_entry->columns_written = all_columns.size() - updated_header.columns(); new_data_part->checksums = source_part->checksums; - if (updated_header.columns() != 0) + if (in) { + std::cerr << "Updated header:" << updated_header.dumpStructure() << std::endl; IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( data, @@ -1157,6 +1164,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor { out.write(block); + std::cerr << "Block readed:" << block.dumpStructure() << std::endl; + std::cerr << "Block rows:" << block.rows() << std::endl; merge_entry->rows_written += block.rows(); merge_entry->bytes_written_uncompressed += block.bytes(); } @@ -1167,10 +1176,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor new_data_part->checksums.add(std::move(changed_checksums)); } + else + { + std::cerr << "Updated header empty\n"; + } - for (const String & file_to_skip : files_to_skip) - if (new_data_part->checksums.files.count(file_to_skip)) - new_data_part->checksums.files.erase(file_to_skip); + for (const String & removed_file : removed_columns) + if (new_data_part->checksums.files.count(removed_file)) + new_data_part->checksums.files.erase(removed_file); { /// Write file with checksums. diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index f3dcf3adafc..45e7a6e8642 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -473,7 +473,7 @@ void MergeTreeReader::performRequiredConversions(Columns & res_columns) //std::cerr << "Copy block: " << copy_block.dumpStructure() << std::endl; DB::performRequiredConversions(copy_block, columns, storage.global_context); - std::cerr << "Result copy block: " << copy_block.dumpStructure() << std::endl; + //std::cerr << "Result copy block: " << copy_block.dumpStructure() << std::endl; /// Move columns from block. name_and_type = columns.begin(); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 15318aecefb..fbc0e9458ab 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3394,6 +3394,7 @@ void StorageReplicatedMergeTree::alter( { std::cerr << "We have mutation commands:" << maybe_mutation_commands.size() << std::endl; ReplicatedMergeTreeMutationEntry mutation_entry = mutateImpl(maybe_mutation_commands, query_context); + std::cerr << "Mutation finished\n"; } } @@ -4465,7 +4466,7 @@ ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const Mu // replicas.push_back(replica_path); waitMutationToFinishOnReplicas(replicas, entry.znode_name); - //} + //} return entry; } From fb7c65c9aa75c414cb1e9dfcab69c5981255b35f Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 15 Jan 2020 16:51:23 +0300 Subject: [PATCH 0106/2007] Simple test --- .../01062_alter_on_mutataion.reference | 7 +++ .../0_stateless/01062_alter_on_mutataion.sql | 51 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference create mode 100644 dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference new file mode 100644 index 00000000000..ed4fa235634 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference @@ -0,0 +1,7 @@ +14850 +14850 +59700 +59700 +59700 +0 +0 diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql new file mode 100644 index 00000000000..e59849b2d2e --- /dev/null +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql @@ -0,0 +1,51 @@ +DROP TABLE IF EXISTS test_alter_on_mutation; + +CREATE TABLE test_alter_on_mutation +( + date Date, + key UInt64, + value String +) +ENGINE ReplicatedMergeTree('/clickhouse/tables/test_alter_on_mutation', '1') +ORDER BY key PARTITION BY date; + +INSERT INTO test_alter_on_mutation select toDate('2020-01-05'), number, toString(number) from system.numbers limit 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-06'), number, toString(number) from system.numbers limit 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-07'), number, toString(number) from system.numbers limit 100; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation MODIFY COLUMN value UInt64; + +SELECT sum(value) from test_alter_on_mutation; + +INSERT INTO test_alter_on_mutation select toDate('2020-01-05'), number, toString(number) from system.numbers limit 100, 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-06'), number, toString(number) from system.numbers limit 100, 100; +INSERT INTO test_alter_on_mutation select toDate('2020-01-07'), number, toString(number) from system.numbers limit 100, 100; + +OPTIMIZE TABLE test_alter_on_mutation FINAL; + +SELECT sum(value) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation MODIFY COLUMN value String; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +OPTIMIZE TABLE test_alter_on_mutation FINAL; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation ADD COLUMN value1 Float64; + +SELECT sum(value1) from test_alter_on_mutation; + +ALTER TABLE test_alter_on_mutation DROP COLUMN value; + +SELECT sum(value) from test_alter_on_mutation; -- {serverError 47} + +-- this is bug +ALTER TABLE test_alter_on_mutation ADD COLUMN value String DEFAULT '0'; + +SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; + +--DROP TABLE IF EXISTS test_alter_on_mutation; From 2a8873a9b76a1555c0bd68be7bb047edf1df708b Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 15 Jan 2020 19:18:11 +0300 Subject: [PATCH 0107/2007] Better --- .../MergeTree/MergeTreeBaseSelectProcessor.cpp | 8 +++++++- dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp | 1 + dbms/src/Storages/MergeTree/MergeTreeReader.cpp | 3 +++ dbms/src/Storages/StorageReplicatedMergeTree.cpp | 6 +++--- .../0_stateless/01062_alter_on_mutataion.reference | 3 ++- .../queries/0_stateless/01062_alter_on_mutataion.sql | 11 ++++++++--- 6 files changed, 24 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 963296dd30f..183623babf4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -44,7 +44,7 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( save_marks_in_cache(save_marks_in_cache_), virt_column_names(virt_column_names_) { - //std::cerr << "HEADER IN SELECT PROCESSOR:" << getPort().getHeader().dumpStructure() << std::endl; + std::cerr << "HEADER IN SELECT PROCESSOR:" << getPort().getHeader().dumpStructure() << std::endl; //std::cerr << "STACK:" << StackTrace().toString() << std::endl; header_without_virtual_columns = getPort().getHeader(); @@ -103,6 +103,7 @@ void MergeTreeBaseSelectProcessor::initializeRangeReaders(MergeTreeReadTask & cu Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() { + std::cerr << "Reading from part impl\n"; if (task->size_predictor) task->size_predictor->startBlock(); @@ -151,6 +152,11 @@ Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() UInt64 rows_to_read = std::max(UInt64(1), std::min(current_max_block_size_rows, recommended_rows)); auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); + std::cerr << "Read result:\n"; + for (const auto & column : read_result.columns) + { + std::cerr << "Column:" << column->dumpStructure() << std::endl; + } /// All rows were filtered. Repeat. if (read_result.num_rows == 0) diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index b78f2bc49eb..6faca2c2c3f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -594,6 +594,7 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar merge_tree_reader->fillMissingColumns(columns, should_evaluate_missing_defaults, num_rows); } + std::cerr << "SHOULD eVALUATE:" << should_evaluate_missing_defaults << std::endl; if (!columns.empty() && should_evaluate_missing_defaults) { auto block = prev_reader->sample_block.cloneWithColumns(read_result.columns); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 45e7a6e8642..5b115d2ee7a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -393,6 +393,7 @@ void MergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_ev void MergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns & res_columns) { + std::cerr << "EVALUATING\n"; try { size_t num_columns = columns.size(); @@ -413,7 +414,9 @@ void MergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns additional_columns.insert({res_columns[pos], name_and_type->type, name_and_type->name}); } + std::cerr << "additional columns before:" << additional_columns.dumpStructure() << std::endl; DB::evaluateMissingDefaults(additional_columns, columns, storage.getColumns().getDefaults(), storage.global_context); + std::cerr << "additional columns after:" << additional_columns.dumpStructure() << std::endl; /// Move columns from block. name_and_type = columns.begin(); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index fbc0e9458ab..befa567ae4f 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -4352,7 +4352,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const mutateImpl(commands, query_context); } -ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & query_context) +ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & /*query_context*/) { /// Overview of the mutation algorithm. /// @@ -4460,8 +4460,8 @@ ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const Mu //if (query_context.getSettingsRef().mutations_sync != 0) //{ Strings replicas; - if (query_context.getSettingsRef().mutations_sync == 2) /// wait for all replicas - replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); + //if (query_context.getSettingsRef().mutations_sync == ) /// wait for all replicas + replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); //else if (query_context.getSettingsRef().mutations_sync == 1) /// just wait for ourself // replicas.push_back(replica_path); diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference index ed4fa235634..dd5108d9c30 100644 --- a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference @@ -4,4 +4,5 @@ 59700 59700 0 -0 +6000 +6000 diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql index e59849b2d2e..7e494a87579 100644 --- a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql @@ -43,9 +43,14 @@ ALTER TABLE test_alter_on_mutation DROP COLUMN value; SELECT sum(value) from test_alter_on_mutation; -- {serverError 47} --- this is bug -ALTER TABLE test_alter_on_mutation ADD COLUMN value String DEFAULT '0'; +ALTER TABLE test_alter_on_mutation ADD COLUMN value String DEFAULT '10'; SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; ---DROP TABLE IF EXISTS test_alter_on_mutation; +--OPTIMIZE table test_alter_on_mutation FINAL; + +ALTER TABLE test_alter_on_mutation MODIFY COLUMN value UInt64; + +SELECT sum(value) from test_alter_on_mutation; + +DROP TABLE IF EXISTS test_alter_on_mutation; From b3bd306a5dfd98d00bfef646ae136354ea6cd9d1 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 15 Jan 2020 19:39:29 +0300 Subject: [PATCH 0108/2007] improve performance of compact parts --- .../MergeTreeDataPartWriterCompact.cpp | 7 ++-- .../MergeTree/MergeTreeReaderCompact.cpp | 40 ++++++++++++------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index d157d19baa4..b4d9ec9dd92 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -98,12 +98,13 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) if (rows_to_write) data_written = true; - /// There could already be enough data to compress into the new block. - if (stream->compressed.offset() >= settings.min_compress_block_size) - stream->compressed.next(); for (const auto & column : columns_list) { + /// There could already be enough data to compress into the new block. + if (stream->compressed.offset() >= settings.min_compress_block_size) + stream->compressed.next(); + size_t old_uncompressed_size = stream->compressed.count(); writeIntBinary(stream->plain_hashing.count(), stream->marks); writeIntBinary(stream->compressed.offset(), stream->marks); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 973bef49d70..4b3e87f113c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -82,6 +82,19 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t read_rows = 0; size_t num_columns = columns.size(); + MutableColumns mutable_columns(num_columns); + auto column_it = columns.begin(); + for (size_t i = 0; i < num_columns; ++i, ++column_it) + { + if (!column_positions[i]) + continue; + + bool append = res_columns[i] != nullptr; + if (!append) + res_columns[i] = column_it->type->createColumn(); + mutable_columns[i] = res_columns[i]->assumeMutable(); + } + while (read_rows < max_rows_to_read) { size_t rows_to_read = data_part->index_granularity.getMarkRows(from_mark); @@ -89,17 +102,11 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, auto name_and_type = columns.begin(); for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) { - auto & [name, type] = *name_and_type; - - if (!column_positions[pos]) + if (!res_columns[pos]) continue; - bool append = res_columns[pos] != nullptr; - if (!append) - res_columns[pos] = name_and_type->type->createColumn(); - - /// To keep offsets shared. TODO Very dangerous. Get rid of this. - MutableColumnPtr column = res_columns[pos]->assumeMutable(); + const auto & [name, type] = *name_and_type; + auto & column = mutable_columns[pos]; try { @@ -108,6 +115,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, readData(*column, *type, from_mark, *column_positions[pos], rows_to_read, read_only_offsets[pos]); size_t read_rows_in_column = column->size() - column_size_before_reading; + if (read_rows_in_column < rows_to_read) throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) + ". Rows expected: " + toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA); @@ -124,17 +132,21 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, e.addMessage("(while reading column " + name + ")"); throw; } - - if (column->size()) - res_columns[pos] = std::move(column); - else - res_columns[pos] = nullptr; } ++from_mark; read_rows += rows_to_read; } + for (size_t i = 0; i < num_columns; ++i) + { + auto & column = mutable_columns[i]; + if (column && column->size()) + res_columns[i] = std::move(column); + else + res_columns[i] = nullptr; + } + next_mark = from_mark; return read_rows; From 3ff8f424edb51565eab781dff4293b2a4c40a642 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 15 Jan 2020 21:24:10 +0300 Subject: [PATCH 0109/2007] remove almost useless columns sizes from compact parts --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 60 +++++++++++------- .../Storages/MergeTree/IMergeTreeDataPart.h | 7 +-- .../MergeTree/IMergeTreeDataPartWriter.h | 4 -- .../MergeTree/MergeTreeDataPartCompact.cpp | 61 +++---------------- .../MergeTree/MergeTreeDataPartCompact.h | 6 -- .../MergeTree/MergeTreeDataPartWide.cpp | 28 --------- .../MergeTree/MergeTreeDataPartWide.h | 4 -- .../MergeTreeDataPartWriterCompact.cpp | 9 --- .../MergeTree/MergedBlockOutputStream.cpp | 19 +----- 9 files changed, 50 insertions(+), 148 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index affcb04b79f..35cb761b7cc 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -357,6 +357,32 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const return checksum->second.file_size; } +String IMergeTreeDataPart::getColumnNameWithMinumumCompressedSize() const +{ + const auto & storage_columns = storage.getColumns().getAllPhysical(); + const std::string * minimum_size_column = nullptr; + UInt64 minimum_size = std::numeric_limits::max(); + + for (const auto & column : storage_columns) + { + if (!hasColumnFiles(column.name, *column.type)) + continue; + + const auto size = getColumnSize(column.name, *column.type).data_compressed; + if (size < minimum_size) + { + minimum_size = size; + minimum_size_column = &column.name; + } + } + + if (!minimum_size_column) + throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); + + return *minimum_size_column; +} + + String IMergeTreeDataPart::getFullPath() const { assertOnDisk(); @@ -380,7 +406,6 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks loadChecksums(require_columns_checksums); loadIndexGranularity(); loadIndex(); /// Must be called after loadIndexGranularity as it uses the value of `index_granularity` - loadColumnSizes(); loadRowsCount(); /// Must be called after loadIndex() as it uses the value of `index_granularity`. loadPartitionAndMinMaxIndex(); loadTTLInfos(); @@ -490,13 +515,13 @@ void IMergeTreeDataPart::loadChecksums(bool require) void IMergeTreeDataPart::loadRowsCount() { + String path = getFullPath() + "count.txt"; if (index_granularity.empty()) { rows_count = 0; } else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - String path = getFullPath() + "count.txt"; if (!Poco::File(path).exists()) throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); @@ -506,6 +531,14 @@ void IMergeTreeDataPart::loadRowsCount() } else { + if (Poco::File(path).exists()) + { + ReadBufferFromFile file = openForReading(path); + readIntText(rows_count, file); + assertEOF(file); + return; + } + for (const NameAndTypePair & column : columns) { ColumnPtr column_col = column.type->createColumn(); @@ -575,7 +608,8 @@ void IMergeTreeDataPart::loadColumns(bool require) Poco::File poco_file_path{path}; if (!poco_file_path.exists()) { - if (require || isCompactPart(shared_from_this())) + /// We can get list of columns only from columns.txt in compact parts. + if (require || part_type == Type::COMPACT) throw Exception("No columns.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); /// If there is no file with a list of columns, write it down. @@ -604,26 +638,6 @@ void IMergeTreeDataPart::loadColumns(bool require) column_name_to_position.emplace(column.name, pos++); } -void IMergeTreeDataPart::loadColumnSizes() -{ - size_t columns_num = columns.size(); - - if (columns_num == 0) - throw Exception("No columns in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); - - auto column_sizes_path = getFullPath() + "columns_sizes.txt"; - auto columns_sizes_file = Poco::File(column_sizes_path); - if (!columns_sizes_file.exists()) - return; - - ReadBufferFromFile buffer(column_sizes_path, columns_sizes_file.getSize()); - auto it = columns.begin(); - for (size_t i = 0; i < columns_num; ++i, ++it) - readPODBinary(columns_sizes[it->name], buffer); - assertEOF(buffer); -} - - UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const String & from) { Poco::File cur(from); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index db00652e520..f5ca6153a5c 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -83,7 +83,7 @@ public: /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). /// If no checksums are present returns the name of the first physically existing column. - virtual String getColumnNameWithMinumumCompressedSize() const { return columns.front().name; } + String getColumnNameWithMinumumCompressedSize() const; virtual String getFileNameForColumn(const NameAndTypePair & column) const = 0; @@ -295,8 +295,6 @@ public: */ mutable std::shared_mutex columns_lock; - ColumnSizeByName columns_sizes; - /// For data in RAM ('index') UInt64 getIndexSizeInBytes() const; UInt64 getIndexSizeInAllocatedBytes() const; @@ -320,6 +318,7 @@ protected: Type part_type; void removeIfNeeded(); virtual void checkConsistency(bool require_part_metadata) const; + void checkConsistencyBase(bool require_part_metadata) const; private: /// In compact parts order of columns is necessary @@ -346,8 +345,6 @@ private: void loadPartitionAndMinMaxIndex(); - void loadColumnSizes(); - String getRelativePathForDetachedPart(const String & prefix) const; }; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 1408370417c..20d6aa16131 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -92,8 +92,6 @@ public: return Columns(std::make_move_iterator(index_columns.begin()), std::make_move_iterator(index_columns.end())); } - const MergeTreeData::ColumnSizeByName & getColumnsSizes() const { return columns_sizes; } - void setWrittenOffsetColumns(WrittenOffsetColumns * written_offset_columns_) { written_offset_columns = written_offset_columns_; @@ -158,8 +156,6 @@ protected: bool primary_index_initialized = false; bool skip_indices_initialized = false; - MergeTreeData::ColumnSizeByName columns_sizes; - /// To correctly write Nested elements column-by-column. WrittenOffsetColumns * written_offset_columns = nullptr; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index b29290e8bb9..e43cdc8669f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -30,10 +30,6 @@ namespace DB { -// namespace -// { -// } - namespace ErrorCodes { extern const int FILE_DOESNT_EXIST; @@ -47,11 +43,6 @@ namespace ErrorCodes } -// static ReadBufferFromFile openForReading(const String & path) -// { -// return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); -// } - MergeTreeDataPartCompact::MergeTreeDataPartCompact( MergeTreeData & storage_, const String & name_, @@ -107,55 +98,21 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( default_codec, writer_settings, computed_index_granularity); } -ColumnSize MergeTreeDataPartCompact::getColumnSize(const String & column_name, const IDataType & /* type */) const -{ - auto column_size = columns_sizes.find(column_name); - if (column_size == columns_sizes.end()) - return {}; - return column_size->second; -} - ColumnSize MergeTreeDataPartCompact::getTotalColumnsSize() const { - ColumnSize totals; - size_t marks_size = 0; - for (const auto & column : columns) + ColumnSize total_size; + auto bin_checksum = checksums.files.find(DATA_FILE_NAME_WITH_EXTENSION); + if (bin_checksum != checksums.files.end()) { - auto column_size = getColumnSize(column.name, *column.type); - totals.add(column_size); - if (!marks_size && column_size.marks) - marks_size = column_size.marks; - } - /// Marks are shared between all columns - totals.marks = marks_size; - return totals; -} - -/** Returns the name of a column with minimum compressed size (as returned by getColumnSize()). - * If no checksums are present returns the name of the first physically existing column. - */ -String MergeTreeDataPartCompact::getColumnNameWithMinumumCompressedSize() const -{ - const auto & storage_columns = storage.getColumns().getAllPhysical(); - const std::string * minimum_size_column = nullptr; - UInt64 minimum_size = std::numeric_limits::max(); - for (const auto & column : storage_columns) - { - if (!getColumnPosition(column.name)) - continue; - - auto size = getColumnSize(column.name, *column.type).data_compressed; - if (size < minimum_size) - { - minimum_size = size; - minimum_size_column = &column.name; - } + total_size.data_compressed += bin_checksum->second.file_size; + total_size.data_compressed += bin_checksum->second.uncompressed_size; } - if (!minimum_size_column) - throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); + auto mrk_checksum = checksums.files.find(DATA_FILE_NAME + index_granularity_info.marks_file_extension); + if (mrk_checksum != checksums.files.end()) + total_size.marks += mrk_checksum->second.file_size; - return *minimum_size_column; + return total_size; } void MergeTreeDataPartCompact::loadIndexGranularity() diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 8191ed8d33e..0ce131bdb80 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -69,12 +69,6 @@ public: bool isStoredOnDisk() const override { return true; } - /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). - /// If no checksums are present returns the name of the first physically existing column. - String getColumnNameWithMinumumCompressedSize() const override; - - ColumnSize getColumnSize(const String & name, const IDataType & type0) const override; - ColumnSize getTotalColumnsSize() const override; void checkConsistency(bool /* require_part_metadata */) const override {} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index f8010b7b774..5fd7dfab0c9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -126,34 +126,6 @@ ColumnSize MergeTreeDataPartWide::getColumnSizeImpl( return size; } -/** Returns the name of a column with minimum compressed size (as returned by getColumnSize()). - * If no checksums are present returns the name of the first physically existing column. - */ -String MergeTreeDataPartWide::getColumnNameWithMinumumCompressedSize() const -{ - const auto & storage_columns = storage.getColumns().getAllPhysical(); - const std::string * minimum_size_column = nullptr; - UInt64 minimum_size = std::numeric_limits::max(); - - for (const auto & column : storage_columns) - { - if (!hasColumnFiles(column.name, *column.type)) - continue; - - const auto size = getColumnSizeImpl(column.name, *column.type, nullptr).data_compressed; - if (size < minimum_size) - { - minimum_size = size; - minimum_size_column = &column.name; - } - } - - if (!minimum_size_column) - throw Exception("Could not find a column of minimum size in MergeTree, part " + getFullPath(), ErrorCodes::LOGICAL_ERROR); - - return *minimum_size_column; -} - ColumnSize MergeTreeDataPartWide::getTotalColumnsSize() const { ColumnSize totals; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 479e6d77e67..5d9815915da 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -70,10 +70,6 @@ public: String getFileNameForColumn(const NameAndTypePair & column) const override; - /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). - /// If no checksums are present returns the name of the first physically existing column. - String getColumnNameWithMinumumCompressedSize() const override; - ColumnSize getTotalColumnsSize() const override; ColumnSize getColumnSize(const String & column_name, const IDataType & type) const override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index b4d9ec9dd92..e818c4f452c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -105,15 +105,10 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) if (stream->compressed.offset() >= settings.min_compress_block_size) stream->compressed.next(); - size_t old_uncompressed_size = stream->compressed.count(); writeIntBinary(stream->plain_hashing.count(), stream->marks); writeIntBinary(stream->compressed.offset(), stream->marks); writeColumnSingleGranule(block.getByName(column.name), current_row, rows_to_write); - - /// We can't calculate compressed size by single column in compact format. - size_t uncompressed_size = stream->compressed.count(); - columns_sizes[column.name].add(ColumnSize{0, 0, uncompressed_size - old_uncompressed_size}); } ++from_mark; @@ -163,10 +158,6 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: writeIntBinary(0ULL, stream->marks); } - size_t marks_size = stream->marks.count(); - for (auto it = columns_sizes.begin(); it != columns_sizes.end(); ++it) - it->second.marks = marks_size; - stream->finalize(); if (sync) stream->sync(); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index f7b20bee00d..baaac34ee79 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -113,7 +113,9 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( else if (rows_count) throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); + } + { WriteBufferFromFile count_out(part_path + "count.txt", 4096); HashingWriteBuffer count_out_hashing(count_out); writeIntText(rows_count, count_out_hashing); @@ -132,22 +134,6 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( checksums.files["ttl.txt"].file_hash = out_hashing.getHash(); } - const auto & columns_sizes = writer->getColumnsSizes(); - if (!columns_sizes.empty()) - { - WriteBufferFromFile out(part_path + "columns_sizes.txt", 4096); - HashingWriteBuffer out_hashing(out); - for (const auto & column : columns_list) - { - auto it = columns_sizes.find(column.name); - if (it == columns_sizes.end()) - throw Exception("Not found size for column " + column.name, ErrorCodes::LOGICAL_ERROR); - writePODBinary(it->second, out_hashing); - checksums.files["columns_sizes.txt"].file_size = out_hashing.count(); - checksums.files["columns_sizes.txt"].file_hash = out_hashing.getHash(); - } - } - { /// Write a file with a description of columns. WriteBufferFromFile out(part_path + "columns.txt", 4096); @@ -166,7 +152,6 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->checksums = checksums; new_part->bytes_on_disk = checksums.getTotalSizeOnDisk(); new_part->index_granularity = writer->getIndexGranularity(); - new_part->columns_sizes = columns_sizes; } void MergedBlockOutputStream::init() From 7a549b2734c98cb0d36fed91f06003a974eae080 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 15 Jan 2020 22:16:56 +0300 Subject: [PATCH 0110/2007] implement 'checkConsistency' method in compact parts --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 56 +++++++ .../Storages/MergeTree/IMergeTreeDataPart.h | 2 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 144 ++++++------------ .../MergeTree/MergeTreeDataPartWide.cpp | 49 +----- 4 files changed, 102 insertions(+), 149 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 35cb761b7cc..87fcebe295d 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -875,6 +875,62 @@ void IMergeTreeDataPart::makeCloneOnDiskDetached(const ReservationPtr & reservat cloning_directory.copyTo(path_to_clone); } +void IMergeTreeDataPart::checkConsistencyBase() const +{ + String path = getFullPath(); + + if (!checksums.empty()) + { + if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) + throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + if (!checksums.files.count("count.txt")) + throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (storage.partition_key_expr && !checksums.files.count("partition.dat")) + throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (!isEmpty()) + { + for (const String & col_name : storage.minmax_idx_columns) + { + if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) + throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); + } + } + } + + checksums.checkSizes(path); + } + else + { + auto check_file_not_empty = [&path](const String & file_path) + { + Poco::File file(file_path); + if (!file.exists() || file.getSize() == 0) + throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + return file.getSize(); + }; + + /// Check that the primary key index is not empty. + if (!storage.primary_key_columns.empty()) + check_file_not_empty(path + "primary.idx"); + + if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + { + check_file_not_empty(path + "count.txt"); + + if (storage.partition_key_expr) + check_file_not_empty(path + "partition.dat"); + + for (const String & col_name : storage.minmax_idx_columns) + check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); + } + } +} + bool isCompactPart(const MergeTreeDataPartPtr & data_part) { return (data_part && data_part->getType() == MergeTreeDataPartType::COMPACT); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index f5ca6153a5c..2e6d9c934ed 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -318,7 +318,7 @@ protected: Type part_type; void removeIfNeeded(); virtual void checkConsistency(bool require_part_metadata) const; - void checkConsistencyBase(bool require_part_metadata) const; + void checkConsistencyBase() const; private: /// In compact parts order of columns is necessary diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index e43cdc8669f..fba9653bc6c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -211,109 +211,51 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) { - UNUSED(require_part_metadata); - /// FIXME implement for compact parts + checkConsistencyBase(); + String path = getFullPath(); + String mrk_file_name = DATA_FILE_NAME + index_granularity_info.marks_file_extension; + if (!checksums.empty()) + { + /// count.txt should be present even in non custom-partitioned parts + if (!checksums.files.count("count.txt")) + throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); + + if (require_part_metadata) + { + if (!checksums.files.count(mrk_file_name)) + throw Exception("No marks file checksum for column in part " + path, ErrorCodes::NO_FILE_IN_DATA_PART); + if (!checksums.files.count(DATA_FILE_NAME_WITH_EXTENSION)) + throw Exception("No data file checksum for in part " + path, ErrorCodes::NO_FILE_IN_DATA_PART); + } + } + else + { + { + /// count.txt should be present even in non custom-partitioned parts + Poco::File file(path + "count.txt"); + if (!file.exists() || file.getSize() == 0) + throw Exception("Part " + path + " is broken: " + file.path() + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + } + + /// Check that marks are nonempty and have the consistent size with columns number. + Poco::File file(path + mrk_file_name); - // String path = getFullPath(); - - // if (!checksums.empty()) - // { - // if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) - // throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); - - // if (require_part_metadata) - // { - // for (const NameAndTypePair & name_type : columns) - // { - // IDataType::SubstreamPath stream_path; - // name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - // { - // String file_name = IDataType::getFileNameForStream(name_type.name, substream_path); - // String mrk_file_name = file_name + index_granularity_info.marks_file_extension; - // String bin_file_name = file_name + ".bin"; - // if (!checksums.files.count(mrk_file_name)) - // throw Exception("No " + mrk_file_name + " file checksum for column " + name_type.name + " in part " + path, - // ErrorCodes::NO_FILE_IN_DATA_PART); - // if (!checksums.files.count(bin_file_name)) - // throw Exception("No " + bin_file_name + " file checksum for column " + name_type.name + " in part " + path, - // ErrorCodes::NO_FILE_IN_DATA_PART); - // }, stream_path); - // } - // } - - // if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - // { - // if (!checksums.files.count("count.txt")) - // throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); - - // if (storage.partition_key_expr && !checksums.files.count("partition.dat")) - // throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); - - // if (!isEmpty()) - // { - // for (const String & col_name : storage.minmax_idx_columns) - // { - // if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) - // throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); - // } - // } - // } - - // checksums.checkSizes(path); - // } - // else - // { - // auto check_file_not_empty = [&path](const String & file_path) - // { - // Poco::File file(file_path); - // if (!file.exists() || file.getSize() == 0) - // throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - // return file.getSize(); - // }; - - // /// Check that the primary key index is not empty. - // if (!storage.primary_key_columns.empty()) - // check_file_not_empty(path + "primary.idx"); - - // if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - // { - // check_file_not_empty(path + "count.txt"); - - // if (storage.partition_key_expr) - // check_file_not_empty(path + "partition.dat"); - - // for (const String & col_name : storage.minmax_idx_columns) - // check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); - // } - - // /// Check that all marks are nonempty and have the same size. - - // std::optional marks_size; - // for (const NameAndTypePair & name_type : columns) - // { - // name_type.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path) - // { - // Poco::File file(IDataType::getFileNameForStream(name_type.name, substream_path) + index_granularity_info.marks_file_extension); - - // /// Missing file is Ok for case when new column was added. - // if (file.exists()) - // { - // UInt64 file_size = file.getSize(); - - // if (!file_size) - // throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", - // ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - - // if (!marks_size) - // marks_size = file_size; - // else if (file_size != *marks_size) - // throw Exception("Part " + path + " is broken: marks have different sizes.", - // ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - // } - // }); - // } - // } + if (file.exists()) + { + UInt64 file_size = file.getSize(); + if (!file_size) + throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + + UInt64 expected_file_size = index_granularity_info.getMarkSizeInBytes(columns.size()) * index_granularity.getMarksCount(); + if (expected_file_size != file_size) + throw Exception( + "Part " + path + " is broken: bad size of marks file '" + file.path() + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), + ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + + } + } } MergeTreeDataPartCompact::~MergeTreeDataPartCompact() diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 5fd7dfab0c9..b091dda704e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -205,13 +205,11 @@ void MergeTreeDataPartWide::accumulateColumnSizes(ColumnToSize & column_to_size) void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const { + checkConsistencyBase(); String path = getFullPath(); if (!checksums.empty()) { - if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx")) - throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART); - if (require_part_metadata) { for (const NameAndTypePair & name_type : columns) @@ -231,54 +229,11 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const }, stream_path); } } - - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - if (!checksums.files.count("count.txt")) - throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); - - if (storage.partition_key_expr && !checksums.files.count("partition.dat")) - throw Exception("No checksum for partition.dat", ErrorCodes::NO_FILE_IN_DATA_PART); - - if (!isEmpty()) - { - for (const String & col_name : storage.minmax_idx_columns) - { - if (!checksums.files.count("minmax_" + escapeForFileName(col_name) + ".idx")) - throw Exception("No minmax idx file checksum for column " + col_name, ErrorCodes::NO_FILE_IN_DATA_PART); - } - } - } - - checksums.checkSizes(path); + } else { - auto check_file_not_empty = [&path](const String & file_path) - { - Poco::File file(file_path); - if (!file.exists() || file.getSize() == 0) - throw Exception("Part " + path + " is broken: " + file_path + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - return file.getSize(); - }; - - /// Check that the primary key index is not empty. - if (!storage.primary_key_columns.empty()) - check_file_not_empty(path + "primary.idx"); - - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) - { - check_file_not_empty(path + "count.txt"); - - if (storage.partition_key_expr) - check_file_not_empty(path + "partition.dat"); - - for (const String & col_name : storage.minmax_idx_columns) - check_file_not_empty(path + "minmax_" + escapeForFileName(col_name) + ".idx"); - } - /// Check that all marks are nonempty and have the same size. - std::optional marks_size; for (const NameAndTypePair & name_type : columns) { From d39f1a0fce03bbc4c6af58d4e0315b00a1e36618 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 16 Jan 2020 17:18:09 +0300 Subject: [PATCH 0111/2007] Better nested handling --- .../MergeTree/MergeTreeDataMergerMutator.cpp | 23 ++++++++++++++++--- .../Storages/MergeTree/MergeTreeDataPart.cpp | 6 +++++ .../MergeTreeSequentialBlockInputStream.cpp | 4 ++++ .../01062_alter_on_mutataion.reference | 2 ++ .../0_stateless/01062_alter_on_mutataion.sql | 18 ++++++++++++++- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 50387bb4c32..092f67fcf0f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1095,6 +1095,18 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor files_to_skip.insert(index->getFileName() + mrk_extension); } + /// Collect counts for shared streams of different columns. As an example, Nested columns have shared stream with array sizes. + std::map stream_counts; + for (const NameAndTypePair & column : source_part->columns) + { + column.type->enumerateStreams( + [&](const IDataType::SubstreamPath & substream_path) { + ++stream_counts[IDataType::getFileNameForStream(column.name, substream_path)]; + }, + {}); + } + + std::unordered_set removed_columns; /// TODO(alesap) better for (const auto & part_column : source_part->columns) @@ -1115,8 +1127,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) { String stream_name = IDataType::getFileNameForStream(part_column.name, substream_path); - removed_columns.insert(stream_name + ".bin"); - removed_columns.insert(stream_name + mrk_extension); + /// Delete files if they are no longer shared with another column. + if (--stream_counts[stream_name] == 0) + { + removed_columns.insert(stream_name + ".bin"); + removed_columns.insert(stream_name + mrk_extension); + } }; IDataType::SubstreamPath stream_path; @@ -1127,10 +1143,11 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator dir_it(source_part->getFullPath()); dir_it != dir_end; ++dir_it) { - if (files_to_skip.count(dir_it.name())) + if (files_to_skip.count(dir_it.name()) || removed_columns.count(dir_it.name())) continue; Poco::Path destination(new_part_tmp_path); + std::cerr << "HARDLINKING FROM:" << dir_it.path().toString() << " TO " << destination.toString() << std::endl; destination.append(dir_it.name()); createHardLink(dir_it.path().toString(), destination.toString()); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 8ec47777dee..69fcd0f6116 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -464,6 +464,12 @@ void MergeTreeDataPart::remove() const LOG_ERROR(storage.log, "Cannot quickly remove directory " << to << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); + std::vector files; + to_dir.list(files); + for (const auto & f : files) + { + std::cerr << "NOT REMOVED FILE:" << f << std::endl; + } to_dir.remove(true); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 08beda32d74..ce58da673e9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -88,6 +88,7 @@ Block MergeTreeSequentialBlockInputStream::getHeader() const Block MergeTreeSequentialBlockInputStream::readImpl() try { + //std::cerr << "READING\n"; Block res; if (!isCancelled() && current_row < data_part->rows_count) { @@ -105,9 +106,12 @@ try bool should_evaluate_missing_defaults = false; reader->fillMissingColumns(columns, should_evaluate_missing_defaults, rows_readed); + //std::cerr << "Should evaluate missing defaults:" << should_evaluate_missing_defaults << std::endl; if (should_evaluate_missing_defaults) + { reader->evaluateMissingDefaults({}, columns); + } reader->performRequiredConversions(columns); diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference index dd5108d9c30..7049ddce9a4 100644 --- a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.reference @@ -6,3 +6,5 @@ 0 6000 6000 +2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] ['2000-01-01','2000-01-01','2000-01-03'] 100500 +2015-01-01 6 38 2014-07-15 13:26:50 [10,20,30] ['asd','qwe','qwe'] 100500 diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql index 7e494a87579..656c3b502de 100644 --- a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql @@ -47,10 +47,26 @@ ALTER TABLE test_alter_on_mutation ADD COLUMN value String DEFAULT '10'; SELECT sum(cast(value as UInt64)) from test_alter_on_mutation; ---OPTIMIZE table test_alter_on_mutation FINAL; +-- TODO(alesap) +OPTIMIZE table test_alter_on_mutation FINAL; ALTER TABLE test_alter_on_mutation MODIFY COLUMN value UInt64; SELECT sum(value) from test_alter_on_mutation; + DROP TABLE IF EXISTS test_alter_on_mutation; + +DROP TABLE IF EXISTS nested_alter; + +CREATE TABLE nested_alter (`d` Date, `k` UInt64, `i32` Int32, `dt` DateTime, `n.ui8` Array(UInt8), `n.s` Array(String), `n.d` Array(Date), `s` String DEFAULT '0') ENGINE = ReplicatedMergeTree('/clickhouse/tables/nested_alter', 'r2', d, k, 8192); + +INSERT INTO nested_alter VALUES ('2015-01-01', 6,38,'2014-07-15 13:26:50',[10,20,30],['asd','qwe','qwe'],['2000-01-01','2000-01-01','2000-01-03'],'100500'); + +SELECT * FROM nested_alter; + +ALTER TABLE nested_alter DROP COLUMN `n.d`; + +SELECT * FROM nested_alter; + +--DROP TABLE nested_alter; From 2797873921135515f58fc8b00f34fb599160bdd9 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 16 Jan 2020 19:15:01 +0300 Subject: [PATCH 0112/2007] code cleanup --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 10 ------ .../Storages/MergeTree/IMergeTreeDataPart.h | 35 ++++++++----------- .../src/Storages/MergeTree/IMergeTreeReader.h | 1 - .../MergeTree/MergeTreeBlockReadUtils.cpp | 9 +++-- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 13 ++++--- dbms/src/Storages/MergeTree/MergeTreeData.h | 3 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 6 ++-- .../MergeTree/MergeTreeDataPartCompact.cpp | 9 +++-- .../MergeTree/MergeTreeDataPartCompact.h | 6 ++-- .../MergeTree/MergeTreeDataPartWide.cpp | 2 +- .../MergeTree/MergeTreeDataPartWide.h | 3 +- .../MergeTreeDataPartWriterCompact.cpp | 1 - .../MergeTree/MergeTreeDataPartWriterWide.cpp | 1 - .../MergeTree/MergeTreeDataPartWriterWide.h | 3 +- .../Storages/MergeTree/MergeTreeIOSettings.h | 2 ++ .../MergeTree/MergeTreeReaderCompact.cpp | 8 ++--- .../MergeTree/MergeTreeReaderCompact.h | 10 ++---- .../Storages/MergeTree/MergeTreeReaderWide.h | 8 +---- .../MergeTreeReverseSelectProcessor.cpp | 2 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 2 +- .../MergeTreeSequentialBlockInputStream.cpp | 4 +-- .../Storages/MergeTree/MergeTreeSettings.h | 5 ++- .../MergedColumnOnlyOutputStream.cpp | 2 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 2 +- .../ReplicatedMergeTreePartCheckThread.cpp | 2 +- dbms/src/Storages/MergeTree/checkDataPart.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 10 +++--- .../System/StorageSystemPartsColumns.cpp | 2 +- 28 files changed, 62 insertions(+), 101 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 87fcebe295d..71614043a31 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -787,16 +787,6 @@ void IMergeTreeDataPart::remove() const } } -void IMergeTreeDataPart::accumulateColumnSizes(ColumnToSize & /* column_to_size */) const -{ - throw Exception("Method 'accumulateColumnSizes' is not supported for data part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMENTED); -} - -void IMergeTreeDataPart::checkConsistency(bool /* require_part_metadata */) const -{ - throw Exception("Method 'checkConsistency' is not supported for data part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMENTED); -} - String IMergeTreeDataPart::typeToString(Type type) { switch (type) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 2e6d9c934ed..31d404fee16 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -81,31 +81,16 @@ public: virtual ColumnSize getTotalColumnsSize() const { return {}; } - /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). - /// If no checksums are present returns the name of the first physically existing column. - String getColumnNameWithMinumumCompressedSize() const; - virtual String getFileNameForColumn(const NameAndTypePair & column) const = 0; - void setColumns(const NamesAndTypesList & columns_); - virtual NameToNameMap createRenameMapForAlter( AlterAnalysisResult & /* analysis_result */, const NamesAndTypesList & /* old_columns */) const { return {}; } virtual ~IMergeTreeDataPart(); - // virtual Checksums check( - // bool require_checksums, - // const DataTypes & primary_key_data_types, /// Check the primary key. If it is not necessary, pass an empty array. - // const MergeTreeIndices & indices = {}, /// Check skip indices - // std::function is_cancelled = []{ return false; }) - // { - // return {}; - // } - using ColumnToSize = std::map; - virtual void accumulateColumnSizes(ColumnToSize & column_to_size) const; + virtual void accumulateColumnSizes(ColumnToSize & /* column_to_size */) const {} using Type = MergeTreeDataPartType; Type getType() const { return part_type; } @@ -128,6 +113,10 @@ public: const std::optional & relative_path, Type part_type_); + void setColumns(const NamesAndTypesList & new_columns); + + const NamesAndTypesList & getColumns() const { return columns; } + void assertOnDisk() const; void remove() const; @@ -142,9 +131,12 @@ public: /// This is useful when you want to change e.g. block numbers or the mutation version of the part. String getNewName(const MergeTreePartInfo & new_part_info) const; - // Block sample_block; std::optional getColumnPosition(const String & column_name) const; + /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). + /// If no checksums are present returns the name of the first physically existing column. + String getColumnNameWithMinumumCompressedSize() const; + bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); } /// If the partition key includes date column (a common case), these functions will return min and max values for this column. @@ -284,9 +276,6 @@ public: Checksums checksums; - /// Columns description. - NamesAndTypesList columns; - /// Columns with values, that all have been zeroed by expired ttl NameSet expired_columns; @@ -315,9 +304,13 @@ public: static UInt64 calculateTotalSizeOnDisk(const String & from); protected: + /// Columns description. + NamesAndTypesList columns; Type part_type; + void removeIfNeeded(); - virtual void checkConsistency(bool require_part_metadata) const; + + virtual void checkConsistency(bool require_part_metadata) const = 0; void checkConsistencyBase() const; private: diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index 1a750940b5e..ed2fd3e4d2c 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -68,7 +68,6 @@ protected: UncompressedCache * uncompressed_cache; MarkCache * mark_cache; - /// If save_marks_in_cache is false, then, if marks are not in cache, we will load them but won't save in the cache, to avoid evicting other data. MergeTreeReaderSettings settings; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 4d6c9d25ef6..54742364277 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -9,7 +9,6 @@ namespace DB { -/// FIXME: implement for compact parts NameSet injectRequiredColumns(const MergeTreeData & storage, const MergeTreeData::DataPartPtr & part, Names & columns) { NameSet required_columns{std::begin(columns), std::end(columns)}; @@ -244,9 +243,9 @@ MergeTreeReadTaskColumns getReadTaskColumns(const MergeTreeData & storage, const /// Under owned_data_part->columns_lock we check that all requested columns are of the same type as in the table. /// This may be not true in case of ALTER MODIFY. if (!pre_column_names.empty()) - storage.check(data_part->columns, pre_column_names); + storage.check(data_part->getColumns(), pre_column_names); if (!column_names.empty()) - storage.check(data_part->columns, column_names); + storage.check(data_part->getColumns(), column_names); const NamesAndTypesList & physical_columns = storage.getColumns().getAllPhysical(); result.pre_columns = physical_columns.addTypes(pre_column_names); @@ -254,8 +253,8 @@ MergeTreeReadTaskColumns getReadTaskColumns(const MergeTreeData & storage, const } else { - result.pre_columns = data_part->columns.addTypes(pre_column_names); - result.columns = data_part->columns.addTypes(column_names); + result.pre_columns = data_part->getColumns().addTypes(pre_column_names); + result.columns = data_part->getColumns().addTypes(column_names); } result.should_reorder = should_reorder; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 4fe20d25b81..b701b7ad3d6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1707,10 +1707,10 @@ void MergeTreeData::alterDataPart( const auto settings = getSettings(); const auto & part = transaction->getDataPart(); - auto res = analyzeAlterConversions(part->columns, new_columns, getIndices().indices, new_indices); + auto res = analyzeAlterConversions(part->getColumns(), new_columns, getIndices().indices, new_indices); NamesAndTypesList additional_columns; - transaction->rename_map = part->createRenameMapForAlter(res, part->columns); + transaction->rename_map = part->createRenameMapForAlter(res, part->getColumns()); if (!transaction->rename_map.empty()) { @@ -1853,7 +1853,7 @@ void MergeTreeData::alterDataPart( /// Write the new column list to the temporary file. { - transaction->new_columns = new_columns.filter(part->columns.getNames()); + transaction->new_columns = new_columns.filter(part->getColumns().getNames()); WriteBufferFromFile columns_file(part->getFullPath() + "columns.txt.tmp", 4096); transaction->new_columns.writeText(columns_file); transaction->rename_map["columns.txt.tmp"] = "columns.txt"; @@ -1888,7 +1888,7 @@ void MergeTreeData::removeEmptyColumnsFromPart(MergeTreeData::MutableDataPartPtr return; NamesAndTypesList new_columns; - for (const auto & [name, type] : data_part->columns) + for (const auto & [name, type] : data_part->getColumns()) if (!empty_columns.count(name)) new_columns.emplace_back(name, type); @@ -2762,7 +2762,6 @@ void MergeTreeData::loadPartAndFixMetadata(MutableDataPartPtr part) String full_part_path = part->getFullPath(); /// Earlier the list of columns was written incorrectly. Delete it and re-create. - /// FIXME looks not right if (isWidePart(part)) if (Poco::File(full_part_path + "columns.txt").exists()) Poco::File(full_part_path + "columns.txt").remove(); @@ -2799,7 +2798,7 @@ void MergeTreeData::addPartContributionToColumnSizes(const DataPartPtr & part) { std::shared_lock lock(part->columns_lock); - for (const auto & column : part->columns) + for (const auto & column : part->getColumns()) { ColumnSize & total_column_size = column_sizes[column.name]; ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); @@ -2811,7 +2810,7 @@ void MergeTreeData::removePartContributionToColumnSizes(const DataPartPtr & part { std::shared_lock lock(part->columns_lock); - for (const auto & column : part->columns) + for (const auto & column : part->getColumns()) { ColumnSize & total_column_size = column_sizes[column.name]; ColumnSize part_column_size = part->getColumnSize(column.name, *column.type); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d781bf170e2..d38eba15386 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -183,7 +183,6 @@ public: MergeTreeDataPartType choosePartType(size_t bytes_on_disk, size_t rows_count) const; /// After this methods setColumns must be called - /// FIXME make this inside this function MutableDataPartPtr createPart(const String & name, const MergeTreePartInfo & part_info,const DiskPtr & disk, const NamesAndTypesList & columns, @@ -194,7 +193,7 @@ public: MergeTreeDataPartType type, const MergeTreePartInfo & part_info, const DiskPtr & disk, const String & relative_path) const; - /// After this methods loadColumnsChecksumsIndexes must be called + /// After this methods 'loadColumnsChecksumsIndexes' must be called /// FIXME make this inside this function MutableDataPartPtr createPart(const String & name, const DiskPtr & disk, const String & relative_path) const; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 9332ece1810..d514c007c7b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -965,7 +965,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor }); if (isCompactPart(source_part)) - commands_for_part.additional_columns = source_part->columns.getNames(); + commands_for_part.additional_columns = source_part->getColumns().getNames(); if (!isStorageTouchedByMutations(storage_from_source_part, commands_for_part, context_for_reading)) { @@ -981,7 +981,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); - const auto & source_column_names = source_part->columns.getNames(); + const auto & source_column_names = source_part->getColumns().getNames(); const auto & updated_column_names = updated_header.getNames(); NameSet new_columns_set(source_column_names.begin(), source_column_names.end()); @@ -1170,7 +1170,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor { /// Write a file with a description of columns. WriteBufferFromFile out_columns(new_part_tmp_path + "columns.txt", 4096); - new_data_part->columns.writeText(out_columns); + new_data_part->getColumns().writeText(out_columns); } new_data_part->rows_count = source_part->rows_count; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index fba9653bc6c..d705f24fc80 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -209,7 +209,7 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( return rename_map; } -void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) +void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) const { checkConsistencyBase(); String path = getFullPath(); @@ -220,7 +220,7 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) /// count.txt should be present even in non custom-partitioned parts if (!checksums.files.count("count.txt")) throw Exception("No checksum for count.txt", ErrorCodes::NO_FILE_IN_DATA_PART); - + if (require_part_metadata) { if (!checksums.files.count(mrk_file_name)) @@ -237,7 +237,7 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) if (!file.exists() || file.getSize() == 0) throw Exception("Part " + path + " is broken: " + file.path() + " is empty", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); } - + /// Check that marks are nonempty and have the consistent size with columns number. Poco::File file(path + mrk_file_name); @@ -247,13 +247,12 @@ void MergeTreeDataPartCompact::checkConsistency(bool require_part_metadata) if (!file_size) throw Exception("Part " + path + " is broken: " + file.path() + " is empty.", ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - + UInt64 expected_file_size = index_granularity_info.getMarkSizeInBytes(columns.size()) * index_granularity.getMarksCount(); if (expected_file_size != file_size) throw Exception( "Part " + path + " is broken: bad size of marks file '" + file.path() + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); - } } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 0ce131bdb80..834bbb6df6b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -71,8 +71,6 @@ public: ColumnSize getTotalColumnsSize() const override; - void checkConsistency(bool /* require_part_metadata */) const override {} - bool hasColumnFiles(const String & column_name, const IDataType & type) const override; String getFileNameForColumn(const NameAndTypePair & /* column */) const override { return DATA_FILE_NAME; } @@ -84,10 +82,10 @@ public: ~MergeTreeDataPartCompact() override; private: + void checkConsistency(bool require_part_metadata) const override; + /// Loads marks index granularity into memory void loadIndexGranularity() override; - - void checkConsistency(bool require_part_metadata); }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index b091dda704e..579552cdf3d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -229,7 +229,7 @@ void MergeTreeDataPartWide::checkConsistency(bool require_part_metadata) const }, stream_path); } } - + } else { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 5d9815915da..be36b390bb1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -82,10 +82,9 @@ public: bool hasColumnFiles(const String & column, const IDataType & type) const override; -protected: +private: void checkConsistency(bool require_part_metadata) const override; -private: /// Loads marks index granularity into memory void loadIndexGranularity() override; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index e818c4f452c..daad489be58 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -42,7 +42,6 @@ void MergeTreeDataPartWriterCompact::write( /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, /// but not in case of vertical merge) - /// FIXME maybe it's wrong at this stage. if (compute_granularity) fillIndexGranularity(block); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 92c21e3166f..a5984c25b1f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -294,7 +294,6 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch { if (!serialization_states.empty()) { - /// FIXME maybe we need skip_offsets=false in some cases serialize_settings.getter = createStreamGetter(it->name, written_offset_columns ? *written_offset_columns : offset_columns); it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[it->name]); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 10260fb7a90..065e8b5e48e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -6,7 +6,6 @@ namespace DB class MergeTreeDataPartWriterWide : public IMergeTreeDataPartWriter { public: - using ColumnToSize = std::map; MergeTreeDataPartWriterWide( @@ -26,6 +25,7 @@ public: IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns); +private: /// Write data of one column. /// Return how many marks were written and /// how many rows were written for last mark @@ -35,7 +35,6 @@ public: const IColumn & column, WrittenOffsetColumns & offset_columns); -private: /// Write single granule of one column (rows between 2 marks) size_t writeSingleGranule( const String & name, diff --git a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h index 5b30fb2ce62..f79a948dd33 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -10,6 +10,8 @@ struct MergeTreeReaderSettings size_t min_bytes_to_use_direct_io = 0; size_t min_bytes_to_use_mmap_io = 0; size_t max_read_buffer_size = DBMS_DEFAULT_BUFFER_SIZE; + /// If save_marks_in_cache is false, then, if marks are not in cache, + /// we will load them but won't save in the cache, to avoid evicting other data. bool save_marks_in_cache = false; }; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 4b3e87f113c..b11c02a5d44 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -74,8 +74,6 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, size_t max_rows_to_read, Columns & res_columns) { - /// FIXME compute correct granularity - if (continue_reading) from_mark = next_mark; @@ -155,7 +153,7 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, MergeTreeReaderCompact::ColumnPosition MergeTreeReaderCompact::findColumnForOffsets(const String & column_name) { String table_name = Nested::extractTableName(column_name); - for (const auto & part_column : data_part->columns) + for (const auto & part_column : data_part->getColumns()) { if (typeid_cast(part_column.type.get())) { @@ -206,7 +204,7 @@ void MergeTreeReaderCompact::initMarksLoader() if (marks_loader.initialized()) return; - size_t columns_num = data_part->columns.size(); + size_t columns_num = data_part->getColumns().size(); auto load = [this, columns_num](const String & mrk_path) -> MarkCache::MappedPtr { @@ -273,7 +271,7 @@ bool MergeTreeReaderCompact::isContinuousReading(size_t mark, size_t column_posi return false; const auto & [last_mark, last_column] = *last_read_granule; return (mark == last_mark && column_position == last_column + 1) - || (mark == last_mark + 1 && column_position == 0 && last_column == data_part->columns.size() - 1); + || (mark == last_mark + 1 && column_position == 0 && last_column == data_part->getColumns().size() - 1); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 2d78eacb16b..0dc0434fad4 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -9,9 +9,7 @@ namespace DB { -/// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. -/// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. -/// Avoids loading the marks file if it is not needed (e.g. when reading the whole part).ca +/// Reader for compact parts class MergeTreeReaderCompact : public IMergeTreeReader { public: @@ -39,7 +37,9 @@ private: MergeTreeMarksLoader marks_loader; using ColumnPosition = std::optional; + /// Positions of columns in part structe. std::vector column_positions; + /// Should we read full column or only it's offsets std::vector read_only_offsets; size_t next_mark = 0; @@ -52,10 +52,6 @@ private: size_t from_mark, size_t column_position, size_t rows_to_read, bool only_offsets = false); ColumnPosition findColumnForOffsets(const String & column_name); - - /// Columns that are read. - - friend class MergeTreeRangeReader::DelayedStream; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h index c9ce5dddf23..4ff62d89300 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -8,9 +8,7 @@ namespace DB { -/// Reads the data between pairs of marks in the same part. When reading consecutive ranges, avoids unnecessary seeks. -/// When ranges are almost consecutive, seeks are fast because they are performed inside the buffer. -/// Avoids loading the marks file if it is not needed (e.g. when reading the whole part). +/// Reader for Wide parts. class MergeTreeReaderWide : public IMergeTreeReader { public: @@ -35,8 +33,6 @@ private: FileStreams streams; - /// Columns that are read. - void addStreams(const String & name, const IDataType & type, const ReadBufferFromFileBase::ProfileCallback & profile_callback, clockid_t clock_type); @@ -44,8 +40,6 @@ private: const String & name, const IDataType & type, IColumn & column, size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool read_offsets = true); - - friend class MergeTreeRangeReader::DelayedStream; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 419dc5fcec9..750001e1384 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -15,7 +15,7 @@ static Block replaceTypes(Block && header, const MergeTreeData::DataPartPtr & da { /// Types may be different during ALTER (when this stream is used to perform an ALTER). /// NOTE: We may use similar code to implement non blocking ALTERs. - for (const auto & name_type : data_part->columns) + for (const auto & name_type : data_part->getColumns()) { if (header.has(name_type.name)) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index 9cdde2a36a1..5321c74f6d1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -15,7 +15,7 @@ static Block replaceTypes(Block && header, const MergeTreeData::DataPartPtr & da { /// Types may be different during ALTER (when this stream is used to perform an ALTER). /// NOTE: We may use similar code to implement non blocking ALTERs. - for (const auto & name_type : data_part->columns) + for (const auto & name_type : data_part->getColumns()) { if (header.has(name_type.name)) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index 9caf605262a..f79af378951 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -48,7 +48,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( else { /// take columns from data_part - columns_for_reader = data_part->columns.addTypes(columns_to_read); + columns_for_reader = data_part->getColumns().addTypes(columns_to_read); } MergeTreeReaderSettings reader_settings = @@ -68,7 +68,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( void MergeTreeSequentialBlockInputStream::fixHeader(Block & header_block) const { /// Types may be different during ALTER (when this stream is used to perform an ALTER). - for (const auto & name_type : data_part->columns) + for (const auto & name_type : data_part->getColumns()) { if (header_block.has(name_type.name)) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 49e6e7e6823..7ce2deaca60 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -25,13 +25,12 @@ class ASTStorage; struct MergeTreeSettings : public SettingsCollection { -/// FIXME description for settings #define LIST_OF_MERGE_TREE_SETTINGS(M) \ M(SettingUInt64, index_granularity, 8192, "How many rows correspond to one primary key value.", 0) \ \ /** Data storing format settigns. */ \ - M(SettingUInt64, min_bytes_for_wide_part, 0, "", 0) \ - M(SettingUInt64, min_rows_for_wide_part, 10000000000, "", 0) \ + M(SettingUInt64, min_bytes_for_wide_part, 0, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \ + M(SettingUInt64, min_rows_for_wide_part, 10000000000, "Minimal number of rows to create part in wide format instead of compact", 0) \ \ /** Merge settings. */ \ M(SettingUInt64, merge_max_block_size, DEFAULT_MERGE_BLOCK_SIZE, "How many rows in blocks should be formed for merge operations.", 0) \ diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 5b3da800d99..57361487e98 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -27,7 +27,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( void MergedColumnOnlyOutputStream::write(const Block & block) { - std::set skip_indexes_column_names_set; + std::unordered_set skip_indexes_column_names_set; for (const auto & index : writer->getSkipIndices()) std::copy(index->columns.cbegin(), index->columns.cend(), std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end())); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 3bb7e04fe6e..b25b4d3d804 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -202,7 +202,7 @@ void ReplicatedMergeTreeBlockOutputStream::writeExistingPart(MergeTreeData::Muta void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zookeeper, MergeTreeData::MutableDataPartPtr & part, const String & block_id) { - storage.check(part->columns); + storage.check(part->getColumns()); assertSessionIsNotExpired(zookeeper); /// Obtain incremental block number and lock it. The lock holds our intention to add the block to the filesystem. diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 3d9c36c18cb..17b716d14c2 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -206,7 +206,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na auto table_lock = storage.lockStructureForShare(false, RWLockImpl::NO_QUERY); auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( - part->columns, part->checksums); + part->getColumns(), part->checksums); String part_path = storage.replica_path + "/parts/" + part_name; String part_znode; diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index 0317e904a09..012e0aecc5d 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -138,7 +138,7 @@ IMergeTreeDataPart::Checksums checkDataPart( { return checkDataPart( data_part->getFullPath(), - data_part->columns, + data_part->getColumns(), data_part->getType(), require_checksums, is_cancelled); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 5b24054f440..feb525402d6 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -767,11 +767,11 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (part_name.empty()) part_name = part->name; - check(part->columns); + check(part->getColumns()); int expected_columns_version = columns_version; auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( - part->columns, part->checksums); + part->getColumns(), part->checksums); Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); std::shuffle(replicas.begin(), replicas.end(), thread_local_rng); @@ -853,7 +853,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: ops.emplace_back(zkutil::makeCreateRequest( part_path, "", zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( - part_path + "/columns", part->columns.toString(), zkutil::CreateMode::Persistent)); + part_path + "/columns", part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( part_path + "/checksums", getChecksumsForZooKeeper(part->checksums), zkutil::CreateMode::Persistent)); } @@ -5323,7 +5323,7 @@ void StorageReplicatedMergeTree::getCommitPartOps( { ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name, - ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->columns, part->checksums).toString(), + ReplicatedMergeTreePartHeader::fromColumnsAndChecksums(part->getColumns(), part->checksums).toString(), zkutil::CreateMode::Persistent)); } else @@ -5334,7 +5334,7 @@ void StorageReplicatedMergeTree::getCommitPartOps( zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name + "/columns", - part->columns.toString(), + part->getColumns().toString(), zkutil::CreateMode::Persistent)); ops.emplace_back(zkutil::makeCreateRequest( replica_path + "/parts/" + part->name + "/checksums", diff --git a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp index 25565810227..e1f0e5f8045 100644 --- a/dbms/src/Storages/System/StorageSystemPartsColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsColumns.cpp @@ -101,7 +101,7 @@ void StorageSystemPartsColumns::processNextStorage(MutableColumns & columns_, co using State =IMergeTreeDataPart::State; - for (const auto & column : part->columns) + for (const auto & column : part->getColumns()) { size_t j = 0; From b914c4262df43a9bcc901324f4271f9c756a6135 Mon Sep 17 00:00:00 2001 From: liyang Date: Fri, 17 Jan 2020 21:52:02 +0800 Subject: [PATCH 0113/2007] add array function auc --- dbms/src/Functions/array/arrayScalarProduct.h | 157 ++++++++++++++++++ dbms/src/Functions/array/auc.cpp | 97 +++++++++++ .../array/registerFunctionsArray.cpp | 3 +- .../0_stateless/01064_array_auc.reference | 1 + .../queries/0_stateless/01064_array_auc.sql | 1 + .../functions/array_functions.md | 6 + 6 files changed, 264 insertions(+), 1 deletion(-) create mode 100644 dbms/src/Functions/array/arrayScalarProduct.h create mode 100644 dbms/src/Functions/array/auc.cpp create mode 100644 dbms/tests/queries/0_stateless/01064_array_auc.reference create mode 100644 dbms/tests/queries/0_stateless/01064_array_auc.sql diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h new file mode 100644 index 00000000000..9d25f551483 --- /dev/null +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -0,0 +1,157 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +template +class FunctionArrayScalarProduct : public IFunction +{ +public: + static constexpr auto name = Name::name; + static FunctionPtr create(const Context & context) { return std::make_shared(context); } + FunctionArrayScalarProduct(const Context & context_): context(context_) {} + +private: + using ResultColumnType = ColumnVector; + + template + bool executeNumber(Block & block, const ColumnNumbers & arguments, size_t result) + { + return executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result); + } + + + template + bool executeNumberNumber(Block & block, const ColumnNumbers & arguments, size_t result) + { + ColumnPtr col1 = block.getByPosition(arguments[0]).column->convertToFullColumnIfConst(); + ColumnPtr col2 = block.getByPosition(arguments[1]).column->convertToFullColumnIfConst(); + if (! col1 || ! col2) + return false; + + const ColumnArray* col_array1 = checkAndGetColumn(col1.get()); + const ColumnArray* col_array2 = checkAndGetColumn(col2.get()); + if (! col_array1 || ! col_array2) + return false; + + const ColumnVector * col_nested1 = checkAndGetColumn>(col_array1->getData()); + const ColumnVector * col_nested2 = checkAndGetColumn>(col_array2->getData()); + if (! col_nested1 || ! col_nested2) + return false; + + auto col_res = ResultColumnType::create(); + vector(col_nested1->getData(), col_array1->getOffsets(), + col_nested2->getData(), col_array2->getOffsets(), col_res->getData()); + block.getByPosition(result).column = std::move(col_res); + return true; + } + + template + static void vector(const PaddedPODArray & data1, const ColumnArray::Offsets & offsets1, + const PaddedPODArray & data2, const ColumnArray::Offsets & offsets2, + PaddedPODArray & result) + { + size_t size = offsets1.size(); + result.resize(size); + + ColumnArray::Offset current_offset1 = 0; + ColumnArray::Offset current_offset2 = 0; + for (size_t i = 0; i < size; ++i) { + size_t array1_size = offsets1[i] - current_offset1; + size_t array2_size = offsets2[i] - current_offset2; + result[i] = Method::apply(data1, current_offset1, array1_size, data2, current_offset2, array2_size); + + current_offset1 = offsets1[i]; + current_offset2 = offsets2[i]; + } + } + +public: + /// Get function name. + String getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 2; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + // Basic type check + std::vector nested_types(2, nullptr); + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const DataTypeArray * array_type = checkAndGetDataType(arguments[i].get()); + if (!array_type) + throw Exception("All argument for function " + getName() + " must be an array.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + auto & nested_type = array_type->getNestedType(); + WhichDataType which(nested_type); + bool is_number = which.isNativeInt() || which.isNativeUInt() || which.isFloat(); + if (! is_number) + { + throw Exception(getName() + " cannot process values of type " + nested_type->getName(), + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + nested_types[i] = nested_type; + } + + // Detail type check in Method, then return ReturnType + return Method::getReturnType(nested_types[0], nested_types[1]); + } + + void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) override + { + if (!(executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result))) + throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() + + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; + + } + +private: + const Context & context; +}; + +} \ No newline at end of file diff --git a/dbms/src/Functions/array/auc.cpp b/dbms/src/Functions/array/auc.cpp new file mode 100644 index 00000000000..2ac7d85a501 --- /dev/null +++ b/dbms/src/Functions/array/auc.cpp @@ -0,0 +1,97 @@ +#include +#include +#include +#include "arrayScalarProduct.h" + +namespace DB +{ + +struct NameAUC { static constexpr auto name = "auc"; }; + +class AUCImpl +{ +public: + using ResultType = Float64; + + struct ScoreLabel + { + ResultType score; + UInt8 label; + }; + + static DataTypePtr getReturnType(const DataTypePtr & /* nested_type1 */, const DataTypePtr & nested_type2) + { + WhichDataType which2(nested_type2); + if (! which2.isUInt8()) { + throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } + return std::make_shared>(); + } + + template + static ResultType apply(const PaddedPODArray & scores, ColumnArray::Offset score_offset, size_t score_len, + const PaddedPODArray & labels, ColumnArray::Offset label_offset, size_t label_len) + { + if (score_len != label_len) + throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), + ErrorCodes::LOGICAL_ERROR}; + if (score_len == 0) + return {}; + + // Order pairs of score and lable by score ascending + size_t num_pos = 0; + size_t num_neg = 0; + std::vector pairs(score_len); + for (size_t i = 0; i < score_len; ++i) + { + pairs[i].score = scores[i + score_offset]; + pairs[i].label = (labels[i + label_offset] ? 1 : 0); + if (pairs[i].label) + ++num_pos; + else + ++num_neg; + } + std::sort(pairs.begin(), pairs.end(), + [](const auto & lhs, const auto & rhs) {return lhs.score < rhs.score; }); + + // Calculate AUC + size_t curr_cnt = 0; + size_t curr_pos_cnt = 0; + size_t curr_sum = 0; + ResultType last_score = -1; + ResultType rank_sum = 0; + for (size_t i = 0; i < pairs.size(); ++i) + { + if (pairs[i].score == last_score) + { + curr_sum += i + 1; + ++curr_cnt; + if (pairs[i].label) + ++curr_pos_cnt; + } + else + { + if (i > 0) + rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; + curr_sum = i + 1; + curr_cnt = 1; + curr_pos_cnt = pairs[i].label ? 1 : 0; + } + last_score = pairs[i].score; + } + rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; + return (rank_sum - num_pos*(num_pos+1)/2)/(num_pos * num_neg); + } +}; + +/// auc(array_score, array_label) - Calculate AUC with array of score and label +using FunctionAUC = FunctionArrayScalarProduct; + +void registerFunctionAUC(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} \ No newline at end of file diff --git a/dbms/src/Functions/array/registerFunctionsArray.cpp b/dbms/src/Functions/array/registerFunctionsArray.cpp index ababc7603e3..96f19f547c0 100644 --- a/dbms/src/Functions/array/registerFunctionsArray.cpp +++ b/dbms/src/Functions/array/registerFunctionsArray.cpp @@ -33,7 +33,7 @@ void registerFunctionArrayDistinct(FunctionFactory & factory); void registerFunctionArrayFlatten(FunctionFactory & factory); void registerFunctionArrayWithConstant(FunctionFactory & factory); void registerFunctionArrayZip(FunctionFactory & factory); - +void registerFunctionAUC(FunctionFactory &); void registerFunctionsArray(FunctionFactory & factory) { @@ -67,6 +67,7 @@ void registerFunctionsArray(FunctionFactory & factory) registerFunctionArrayFlatten(factory); registerFunctionArrayWithConstant(factory); registerFunctionArrayZip(factory); + registerFunctionAUC(factory); } } diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.reference b/dbms/tests/queries/0_stateless/01064_array_auc.reference new file mode 100644 index 00000000000..d6e1fa0f619 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_array_auc.reference @@ -0,0 +1 @@ +0.75 \ No newline at end of file diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql new file mode 100644 index 00000000000..33b54e7db45 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -0,0 +1 @@ +select select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) \ No newline at end of file diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 6d5d9ccd6bf..64f631a84e4 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -896,5 +896,11 @@ Result: │ [('a','d'),('b','e'),('c','f')] │ └────────────────────────────────────────────┘ ``` +## auc(arr_scores, arr_labels) + +Returns AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc); + +`arr_scores` represents scores prediction model gives, while `arr_labels` represents labels of samples, usually 1 for positive sample and 0 for negtive sample. + [Original article](https://clickhouse.yandex/docs/en/query_language/functions/array_functions/) From 58b9e73a7ae48929bdfb81d5d4ed8c42721ef72b Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 17 Jan 2020 16:54:22 +0300 Subject: [PATCH 0114/2007] Better working on files names --- dbms/src/Core/NamesAndTypes.cpp | 9 ++ dbms/src/Core/NamesAndTypes.h | 2 + .../src/Interpreters/MutationsInterpreter.cpp | 5 +- dbms/src/Storages/AlterCommands.cpp | 24 +++++- .../MergeTree/MergeTreeDataMergerMutator.cpp | 84 +++++++++++++------ .../MergeTree/MergeTreeDataMergerMutator.h | 5 ++ dbms/src/Storages/MutationCommands.cpp | 18 ++-- dbms/src/Storages/MutationCommands.h | 9 +- .../Storages/StorageReplicatedMergeTree.cpp | 70 ++++++++++++---- .../src/Storages/StorageReplicatedMergeTree.h | 4 +- .../0_stateless/01062_alter_on_mutataion.sql | 2 +- 11 files changed, 170 insertions(+), 62 deletions(-) diff --git a/dbms/src/Core/NamesAndTypes.cpp b/dbms/src/Core/NamesAndTypes.cpp index d76f457fd1a..b45e7d771a9 100644 --- a/dbms/src/Core/NamesAndTypes.cpp +++ b/dbms/src/Core/NamesAndTypes.cpp @@ -165,4 +165,13 @@ bool NamesAndTypesList::contains(const String & name) const return false; } +std::optional NamesAndTypesList::tryGetByName(const std::string & name) const +{ + for (const NameAndTypePair & column : *this) + { + if (column.name == name) + return column; + } + return {}; +} } diff --git a/dbms/src/Core/NamesAndTypes.h b/dbms/src/Core/NamesAndTypes.h index 1b7d2d1f175..55f3d989dbc 100644 --- a/dbms/src/Core/NamesAndTypes.h +++ b/dbms/src/Core/NamesAndTypes.h @@ -74,6 +74,8 @@ public: NamesAndTypesList addTypes(const Names & names) const; bool contains(const String & name) const; + + std::optional tryGetByName(const std::string & name) const; }; } diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 5cfcd3f0504..ad680c71a0f 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -392,15 +392,14 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) const auto required_columns = syntax_result->requiredSourceColumns(); affected_indices_columns.insert(std::cbegin(required_columns), std::cend(required_columns)); } - else if (command.type == MutationCommand::READ) + else if (command.type == MutationCommand::READ_COLUMN) { if (stages.empty() || !stages.back().column_to_updated.empty()) stages.emplace_back(context); if (stages.size() == 1) /// First stage only supports filtering and can't update columns. stages.emplace_back(context); - if (command.data_type) - stages.back().column_to_updated.emplace(command.column_name, std::make_shared(command.column_name)); + stages.back().column_to_updated.emplace(command.column_name, std::make_shared(command.column_name)); } else throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index b663f2a2322..62e8398a98a 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -540,10 +540,26 @@ std::optional AlterCommand::tryConvertToMutationCommand(const S MutationCommand result; - result.type = MutationCommand::Type::READ; - result.column_name = column_name; - result.data_type = data_type; - result.predicate = nullptr; + if (type == MODIFY_COLUMN) + { + result.type = MutationCommand::Type::READ_COLUMN; + result.column_name = column_name; + result.data_type = data_type; + result.predicate = nullptr; + } + else if (type == DROP_COLUMN) + { + result.type = MutationCommand::Type::DROP_COLUMN; + result.column_name = column_name; + result.predicate = nullptr; + } + else if (type == DROP_INDEX) + { + result.type = MutationCommand::Type::DROP_INDEX; + result.column_name = column_name; + result.predicate = nullptr; + } + result.ast = ast->clone(); return result; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 092f67fcf0f..36eeb634556 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -982,14 +982,19 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor Poco::File(new_part_tmp_path).createDirectories(); BlockInputStreamPtr in = nullptr; Block updated_header; + std::optional interpreter; - if(!std::all_of(commands_for_part.begin(), commands_for_part.end(), [](const auto & cmd) { return cmd.type == MutationCommand::Type::READ && cmd.data_type == nullptr;})) - { - MutationsInterpreter mutations_interpreter(storage_from_source_part, commands_for_part, context_for_reading, true); - in = mutations_interpreter.execute(table_lock_holder); - updated_header = mutations_interpreter.getUpdatedHeader(); - } + std::vector for_interpreter; + std::vector for_file_renames; + splitMutationCommands(source_part, commands_for_part, for_interpreter, for_file_renames); + + if (!for_interpreter.empty()) + { + interpreter.emplace(storage_from_source_part, for_interpreter, context_for_reading, true); + in = interpreter->execute(table_lock_holder); + updated_header = interpreter->getUpdatedHeader(); + } NamesAndTypesList all_columns = data.getColumns().getAllPhysical(); const auto data_settings = data.getSettings(); @@ -1107,43 +1112,38 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor } - std::unordered_set removed_columns; - /// TODO(alesap) better - for (const auto & part_column : source_part->columns) + std::unordered_set remove_files; + /// Remove old indices + for (const auto & command : for_file_renames) { - bool found = false; - for (const auto & all_column : all_columns) + if (command.type == MutationCommand::Type::DROP_INDEX) { - if (part_column.name == all_column.name) - { - found = true; - break; - } + remove_files.emplace("skp_idx_" + command.column_name + ".idx"); + remove_files.emplace("skp_idx_" + command.column_name + mrk_extension); } - if (!found) + else if (command.type == MutationCommand::Type::DROP_COLUMN) { - std::cerr << "REMOVING COLUMN:" << part_column.name << std::endl; - - IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) - { - String stream_name = IDataType::getFileNameForStream(part_column.name, substream_path); + IDataType::StreamCallback callback = [&](const IDataType::SubstreamPath & substream_path) { + String stream_name = IDataType::getFileNameForStream(command.column_name, substream_path); /// Delete files if they are no longer shared with another column. if (--stream_counts[stream_name] == 0) { - removed_columns.insert(stream_name + ".bin"); - removed_columns.insert(stream_name + mrk_extension); + remove_files.emplace(stream_name + ".bin"); + remove_files.emplace(stream_name + mrk_extension); } }; IDataType::SubstreamPath stream_path; - part_column.type->enumerateStreams(callback, stream_path); + auto column = source_part->columns.tryGetByName(command.column_name); + if (column) + column->type->enumerateStreams(callback, stream_path); } } Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator dir_it(source_part->getFullPath()); dir_it != dir_end; ++dir_it) { - if (files_to_skip.count(dir_it.name()) || removed_columns.count(dir_it.name())) + if (files_to_skip.count(dir_it.name()) || remove_files.count(dir_it.name())) continue; Poco::Path destination(new_part_tmp_path); @@ -1198,7 +1198,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor std::cerr << "Updated header empty\n"; } - for (const String & removed_file : removed_columns) + for (const String & removed_file : remove_files) if (new_data_part->checksums.files.count(removed_file)) new_data_part->checksums.files.erase(removed_file); @@ -1340,4 +1340,34 @@ size_t MergeTreeDataMergerMutator::estimateNeededDiskSpace(const MergeTreeData:: return static_cast(res * DISK_USAGE_COEFFICIENT_TO_RESERVE); } +void MergeTreeDataMergerMutator::splitMutationCommands( + MergeTreeData::DataPartPtr part, + const std::vector & commands, + std::vector & for_interpreter, + std::vector & for_file_renames) +{ + for (const auto & command : commands) + { + if (command.type == MutationCommand::Type::DELETE + || command.type == MutationCommand::Type::UPDATE + || command.type == MutationCommand::Type::MATERIALIZE_INDEX) + { + for_interpreter.push_back(command); + } + else if (command.type == MutationCommand::Type::READ_COLUMN) + { + /// If we don't have this column in source part, than we don't + /// need to materialize it + if (part->columns.contains(command.column_name)) + for_interpreter.push_back(command); + else + for_file_renames.push_back(command); + } + else + { + for_file_renames.push_back(command); + } + } +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index 54c164566a8..08276e051a1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -114,6 +114,11 @@ public: const MergeTreeData::DataPartsVector & parts, MergeTreeData::Transaction * out_transaction = nullptr); + void splitMutationCommands( + MergeTreeData::DataPartPtr part, + const std::vector & commands, + std::vector & for_interpreter, + std::vector & for_file_renames); /// The approximate amount of disk space needed for merge or mutation. With a surplus. static size_t estimateNeededDiskSpace(const MergeTreeData::DataPartsVector & source_parts); diff --git a/dbms/src/Storages/MutationCommands.cpp b/dbms/src/Storages/MutationCommands.cpp index 2d261a8c0b7..cc535293c66 100644 --- a/dbms/src/Storages/MutationCommands.cpp +++ b/dbms/src/Storages/MutationCommands.cpp @@ -21,7 +21,7 @@ namespace ErrorCodes extern const int MULTIPLE_ASSIGNMENTS_TO_COLUMN; } -std::optional MutationCommand::parse(ASTAlterCommand * command, bool parse_modify) +std::optional MutationCommand::parse(ASTAlterCommand * command, bool from_zookeeper) { if (command->type == ASTAlterCommand::DELETE) { @@ -57,24 +57,32 @@ std::optional MutationCommand::parse(ASTAlterCommand * command, res.index_name = command->index->as().name; return res; } - else if (parse_modify && command->type == ASTAlterCommand::MODIFY_COLUMN) + else if (from_zookeeper && command->type == ASTAlterCommand::MODIFY_COLUMN) { MutationCommand res; res.ast = command->ptr(); - res.type = MutationCommand::Type::READ; + res.type = MutationCommand::Type::READ_COLUMN; const auto & ast_col_decl = command->col_decl->as(); res.column_name = ast_col_decl.name; res.data_type = DataTypeFactory::instance().get(ast_col_decl.type); return res; } - else if (parse_modify && command->type == ASTAlterCommand::DROP_COLUMN) + else if (from_zookeeper && command->type == ASTAlterCommand::DROP_COLUMN) { MutationCommand res; res.ast = command->ptr(); - res.type = MutationCommand::Type::READ; + res.type = MutationCommand::Type::DROP_COLUMN; res.column_name = getIdentifierName(command->column); return res; } + else if (from_zookeeper && command->type == ASTAlterCommand::DROP_INDEX) + { + MutationCommand res; + res.ast = command->ptr(); + res.type = MutationCommand::Type::DROP_INDEX; + res.column_name = command->index->as().name; + return res; + } else return {}; } diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index d808fbca4d5..265c5ae1a6f 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -27,7 +27,9 @@ struct MutationCommand DELETE, UPDATE, MATERIALIZE_INDEX, - READ + READ_COLUMN, + DROP_COLUMN, + DROP_INDEX, }; Type type = EMPTY; @@ -42,11 +44,12 @@ struct MutationCommand String index_name; ASTPtr partition; - /// For cast + /// For reads, drops and etc. String column_name; DataTypePtr data_type; - static std::optional parse(ASTAlterCommand * command, bool parse_modify=false); + /// If from_zookeeper, than consider more Alter commands as mutation commands + static std::optional parse(ASTAlterCommand * command, bool from_zookeeper=false); }; /// Multiple mutation commands, possible from different ALTER queries diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index befa567ae4f..806ce71a206 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -979,7 +979,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) } else if (entry.type == LogEntry::FINISH_ALTER) { - tryFinishAlter(entry); + executeMetadataAlter(entry); } else { @@ -1156,7 +1156,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } -bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree::LogEntry & /*entry*/) +bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & /*entry*/) { std::cerr << "Trying to finish alter\n"; auto zookeeper = getZooKeeper(); @@ -1179,15 +1179,21 @@ bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree const bool changed_columns_version = (columns_version_zk != this->columns_version); const bool changed_metadata_version = (metadata_version_zk != this->metadata_version); - std::cerr << "Versions changed: columns" << changed_columns_version << " metadata:" << changed_metadata_version << std::endl; + std::cerr << "Versions changed: columns:" << changed_columns_version << " metadata:" << changed_metadata_version << std::endl; if (!(changed_columns_version || changed_metadata_version)) + { + std::cerr << "Nothing changed\n"; return true; + } + std::cerr << "Receiving metadata from zookeeper\n"; auto columns_in_zk = ColumnsDescription::parse(columns_str); auto metadata_in_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); + std::cerr << "Metadata received\n"; + MergeTreeData::DataParts parts; /// If metadata nodes have changed, we will update table structure locally. @@ -1213,16 +1219,29 @@ bool StorageReplicatedMergeTree::tryFinishAlter(const StorageReplicatedMergeTree LOG_INFO(log, "Applied changes to the metadata of the table."); } - this->columns_version = columns_version_zk; - this->metadata_version = metadata_version_zk; + std::cerr << "Columns version before:" << columns_version << std::endl; + std::cerr << "Columns version after:" << columns_version_zk << std::endl; + columns_version = columns_version_zk; + metadata_version = metadata_version_zk; + std::cerr << "Recalculating columns sizes\n"; recalculateColumnSizes(); /// Update metadata ZK nodes for a specific replica. if (changed_columns_version) + { zookeeper->set(replica_path + "/columns", columns_str); + } + else + { + } if (changed_metadata_version) + { zookeeper->set(replica_path + "/metadata", metadata_str); + } + + std::cerr << "Nodes in zk updated\n"; } + std::cerr << "Done\n"; return true; } @@ -3380,7 +3399,7 @@ void StorageReplicatedMergeTree::alter( entry.type = LogEntry::FINISH_ALTER; entry.source_replica = replica_name; - std::cerr << " Columns before mutation:" << getColumns().getAllPhysical().toString() << std::endl; + //std::cerr << " Columns before mutation:" << getColumns().getAllPhysical().toString() << std::endl; entry.new_part_name = ""; entry.create_time = time(nullptr); @@ -3388,14 +3407,24 @@ void StorageReplicatedMergeTree::alter( String path_created = getZooKeeper()->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - waitForAllReplicasToProcessLogEntry(entry); + std::cerr << "Waiting for replicas\n"; + auto unwaited = waitForAllReplicasToProcessLogEntry(entry, false); + + std::cerr << "Replicas done"; if (!maybe_mutation_commands.empty()) { std::cerr << "We have mutation commands:" << maybe_mutation_commands.size() << std::endl; - ReplicatedMergeTreeMutationEntry mutation_entry = mutateImpl(maybe_mutation_commands, query_context); + Context copy_context = query_context; + copy_context.getSettingsRef().mutations_sync = 2; + ReplicatedMergeTreeMutationEntry mutation_entry = mutateImpl(maybe_mutation_commands, copy_context); std::cerr << "Mutation finished\n"; } + + if (!unwaited.empty()) + { + throw Exception("Some replicas doesn't finish alter", ErrorCodes::UNFINISHED); + } } void StorageReplicatedMergeTree::alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) @@ -3782,21 +3811,27 @@ StorageReplicatedMergeTree::allocateBlockNumber( } -void StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) +Strings StorageReplicatedMergeTree::waitForAllReplicasToProcessLogEntry(const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active) { LOG_DEBUG(log, "Waiting for all replicas to process " << entry.znode_name); auto zookeeper = getZooKeeper(); Strings replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + Strings unwaited; for (const String & replica : replicas) { if (wait_for_non_active || zookeeper->exists(zookeeper_path + "/replicas/" + replica + "/is_active")) { waitForReplicaToProcessLogEntry(replica, entry); } + else + { + unwaited.push_back(replica); + } } LOG_DEBUG(log, "Finished waiting for all replicas to process " << entry.znode_name); + return unwaited; } @@ -4352,7 +4387,7 @@ void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const mutateImpl(commands, query_context); } -ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & /*query_context*/) +ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & query_context) { /// Overview of the mutation algorithm. /// @@ -4457,16 +4492,17 @@ ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const Mu } /// we have to wait - //if (query_context.getSettingsRef().mutations_sync != 0) - //{ + if (query_context.getSettingsRef().mutations_sync != 0) + { Strings replicas; - //if (query_context.getSettingsRef().mutations_sync == ) /// wait for all replicas - replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); - //else if (query_context.getSettingsRef().mutations_sync == 1) /// just wait for ourself - // replicas.push_back(replica_path); + if (query_context.getSettingsRef().mutations_sync == 2) /// wait for all replicas + replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); + else if (query_context.getSettingsRef().mutations_sync == 1) /// just wait for ourself + replicas.push_back(replica_path); + std::cerr << "Waiting for mutation on replicas:" << replicas.size() << std::endl; waitMutationToFinishOnReplicas(replicas, entry.znode_name); - //} + } return entry; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 6ad7b08b34e..0b02477e987 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -380,7 +380,7 @@ private: /// Do the merge or recommend to make the fetch instead of the merge bool tryExecuteMerge(const LogEntry & entry); - bool tryFinishAlter(const LogEntry & entry); + bool executeMetadataAlter(const LogEntry & entry); bool tryExecutePartMutation(const LogEntry & entry); @@ -489,7 +489,7 @@ private: * Because it effectively waits for other thread that usually has to also acquire a lock to proceed and this yields deadlock. * TODO: There are wrong usages of this method that are not fixed yet. */ - void waitForAllReplicasToProcessLogEntry(const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active = true); + Strings waitForAllReplicasToProcessLogEntry(const ReplicatedMergeTreeLogEntryData & entry, bool wait_for_non_active = true); /** Wait until the specified replica executes the specified action from the log. * NOTE: See comment about locks above. diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql index 656c3b502de..32e9b9f22bc 100644 --- a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql @@ -69,4 +69,4 @@ ALTER TABLE nested_alter DROP COLUMN `n.d`; SELECT * FROM nested_alter; ---DROP TABLE nested_alter; +DROP TABLE nested_alter; From b0906abb0d61df17ef5c2c8d957d8235238be247 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Fri, 17 Jan 2020 15:24:27 +0300 Subject: [PATCH 0115/2007] code cleanup --- .../Storages/MergeTree/AlterAnalysisResult.h | 7 ++++ .../Storages/MergeTree/IMergeTreeDataPart.h | 4 ++- .../MergeTree/IMergeTreeDataPartWriter.h | 2 -- .../MergeTree/IMergedBlockOutputStream.cpp | 12 ------- .../MergeTree/IMergedBlockOutputStream.h | 5 --- .../MergeTree/MergeTreeBlockReadUtils.h | 1 - dbms/src/Storages/MergeTree/MergeTreeData.cpp | 23 ++++++------ dbms/src/Storages/MergeTree/MergeTreeData.h | 10 ++---- .../MergeTree/MergeTreeDataPartCompact.cpp | 35 +++---------------- .../MergeTree/MergeTreeDataPartCompact.h | 22 ------------ .../MergeTree/MergeTreeDataPartWide.cpp | 34 ------------------ .../MergeTree/MergeTreeDataPartWide.h | 20 ----------- .../MergeTreeDataPartWriterCompact.h | 1 - .../MergeTree/MergeTreeDataPartWriterWide.cpp | 15 +------- .../MergeTree/MergeTreeDataPartWriterWide.h | 1 + .../MergeTreeIndexGranularityInfo.cpp | 1 + .../MergeTree/MergeTreeIndexGranularityInfo.h | 5 --- .../MergeTree/MergeTreeReaderCompact.cpp | 23 +++++------- .../MergeTree/MergeTreeReaderCompact.h | 9 ++--- .../MergeTree/MergeTreeReaderWide.cpp | 3 +- .../MergeTreeSequentialBlockInputStream.cpp | 2 +- .../MergeTree/MergedBlockOutputStream.cpp | 11 ++---- .../MergedColumnOnlyOutputStream.cpp | 3 +- .../MergeTree/MergedColumnOnlyOutputStream.h | 2 -- dbms/src/Storages/MergeTree/checkDataPart.cpp | 1 + 25 files changed, 53 insertions(+), 199 deletions(-) diff --git a/dbms/src/Storages/MergeTree/AlterAnalysisResult.h b/dbms/src/Storages/MergeTree/AlterAnalysisResult.h index 40fd3dfe353..96f86623ef9 100644 --- a/dbms/src/Storages/MergeTree/AlterAnalysisResult.h +++ b/dbms/src/Storages/MergeTree/AlterAnalysisResult.h @@ -5,9 +5,16 @@ namespace DB { struct AlterAnalysisResult { + /// Expression for column type conversion. + /// If no conversions are needed, expression=nullptr. ExpressionActionsPtr expression = nullptr; + + /// Denotes if metadata must be changed even if no file should be overwritten + /// (used for transformation-free changing of Enum values list). bool force_update_metadata = false; + std::map new_types; + /// For every column that need to be converted: source column name, /// column name of calculated expression for conversion. std::vector> conversions; diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 31d404fee16..54e918f5b49 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -83,6 +83,8 @@ public: virtual String getFileNameForColumn(const NameAndTypePair & column) const = 0; + /// Returns rename map of column files for the alter converting expression onto new table files. + /// Files to be deleted are mapped to an empty string in rename map. virtual NameToNameMap createRenameMapForAlter( AlterAnalysisResult & /* analysis_result */, const NamesAndTypesList & /* old_columns */) const { return {}; } @@ -306,7 +308,7 @@ public: protected: /// Columns description. NamesAndTypesList columns; - Type part_type; + const Type part_type; void removeIfNeeded(); diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 20d6aa16131..1c751b04da1 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -1,7 +1,5 @@ #pragma once -#include -#include #include #include #include diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index e54c190c03c..f8b32d86a9e 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -6,18 +6,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} - -namespace -{ - // constexpr auto DATA_FILE_EXTENSION = ".bin"; - // constexpr auto INDEX_FILE_EXTENSION = ".idx"; -} - - IMergedBlockOutputStream::IMergedBlockOutputStream( const MergeTreeDataPartPtr & data_part) : storage(data_part->storage) diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h index 6f312c5071e..983750862e8 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.h @@ -1,10 +1,6 @@ #pragma once #include -#include -#include -#include -#include #include #include #include @@ -28,7 +24,6 @@ public: protected: using SerializationState = IDataType::SerializeBinaryBulkStatePtr; - using SerializationStates = std::vector; IDataType::OutputStreamGetter createStreamGetter(const String & name, WrittenOffsetColumns & offset_columns, bool skip_offsets); diff --git a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h index 89565f734f4..19c6adbd9c7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h +++ b/dbms/src/Storages/MergeTree/MergeTreeBlockReadUtils.h @@ -4,7 +4,6 @@ #include #include #include -// #include namespace DB diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b701b7ad3d6..bebf6b25cf5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -103,6 +103,7 @@ namespace ErrorCodes extern const int UNKNOWN_SETTING; extern const int READONLY_SETTING; extern const int ABORTED; + extern const int UNKNOWN_PART_TYPE; } @@ -2272,13 +2273,13 @@ void MergeTreeData::removePartsFromWorkingSet(const MergeTreeData::DataPartsVect for (const DataPartPtr & part : remove) { - if (part->state ==IMergeTreeDataPart::State::Committed) + if (part->state == IMergeTreeDataPart::State::Committed) removePartContributionToColumnSizes(part); - if (part->state ==IMergeTreeDataPart::State::Committed || clear_without_timeout) + if (part->state == IMergeTreeDataPart::State::Committed || clear_without_timeout) part->remove_time.store(remove_time, std::memory_order_relaxed); - if (part->state !=IMergeTreeDataPart::State::Outdated) + if (part->state != IMergeTreeDataPart::State::Outdated) modifyPartState(part,IMergeTreeDataPart::State::Outdated); } } @@ -2762,9 +2763,9 @@ void MergeTreeData::loadPartAndFixMetadata(MutableDataPartPtr part) String full_part_path = part->getFullPath(); /// Earlier the list of columns was written incorrectly. Delete it and re-create. - if (isWidePart(part)) - if (Poco::File(full_part_path + "columns.txt").exists()) - Poco::File(full_part_path + "columns.txt").remove(); + /// But in compact parts we can't get list of columns without this file. + if (isWidePart(part) && Poco::File(full_part_path + "columns.txt").exists()) + Poco::File(full_part_path + "columns.txt").remove(); part->loadColumnsChecksumsIndexes(false, true); part->modification_time = Poco::File(full_part_path).getLastModified().epochTime(); @@ -3746,7 +3747,7 @@ MergeTreeData::CurrentlyMovingPartsTagger::CurrentlyMovingPartsTagger(MergeTreeM : parts_to_move(std::move(moving_parts_)), data(data_) { for (const auto & moving_part : parts_to_move) - if (!data.currently_moving_parts.emplace(moving_part.part->name).second) + if (!data.currently_moving_parts.emplace(moving_part.part).second) throw Exception("Cannot move part '" + moving_part.part->name + "'. It's already moving.", ErrorCodes::LOGICAL_ERROR); } @@ -3756,9 +3757,9 @@ MergeTreeData::CurrentlyMovingPartsTagger::~CurrentlyMovingPartsTagger() for (const auto & moving_part : parts_to_move) { /// Something went completely wrong - if (!data.currently_moving_parts.count(moving_part.part->name)) + if (!data.currently_moving_parts.count(moving_part.part)) std::terminate(); - data.currently_moving_parts.erase(moving_part.part->name); + data.currently_moving_parts.erase(moving_part.part); } } @@ -3802,7 +3803,7 @@ MergeTreeData::CurrentlyMovingPartsTagger MergeTreeData::selectPartsForMove() *reason = "part already assigned to background operation."; return false; } - if (currently_moving_parts.count(part->name)) + if (currently_moving_parts.count(part)) { *reason = "part is already moving."; return false; @@ -3836,7 +3837,7 @@ MergeTreeData::CurrentlyMovingPartsTagger MergeTreeData::checkPartsForMove(const "Move is not possible: " + path_to_clone + part->name + " already exists", ErrorCodes::DIRECTORY_ALREADY_EXISTS); - if (currently_moving_parts.count(part->name) || partIsAssignedToBackgroundOperation(part)) + if (currently_moving_parts.count(part) || partIsAssignedToBackgroundOperation(part)) throw Exception( "Cannot move part '" + part->name + "' because it's participating in background process", ErrorCodes::PART_IS_TEMPORARILY_LOCKED); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index d38eba15386..fe74fa8d4b2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -80,7 +80,7 @@ namespace ErrorCodes /// The same files as for month-partitioned tables, plus /// count.txt - contains total number of rows in this part. /// partition.dat - contains the value of the partitioning expression. -/// minmax_[Column].idx - MinMax indexes (seeIMergeTreeDataPart::MinMaxIndex class) for the columns required by the partitioning expression. +/// minmax_[Column].idx - MinMax indexes (see IMergeTreeDataPart::MinMaxIndex class) for the columns required by the partitioning expression. /// /// Several modes are implemented. Modes determine additional actions during merge: /// - Ordinary - don't do anything special @@ -184,7 +184,7 @@ public: /// After this methods setColumns must be called MutableDataPartPtr createPart(const String & name, - const MergeTreePartInfo & part_info,const DiskPtr & disk, + const MergeTreePartInfo & part_info, const DiskPtr & disk, const NamesAndTypesList & columns, size_t bytes_on_disk, size_t rows_num, const String & relative_path) const; @@ -789,7 +789,7 @@ public: /// if we decide to move some part to another disk, than we /// assuredly will choose this disk for containing part, which will appear /// as result of merge or mutation. - NameSet currently_moving_parts; + DataParts currently_moving_parts; /// Mutex for currently_moving_parts mutable std::mutex moving_parts_mutex; @@ -797,8 +797,6 @@ public: protected: friend class IMergeTreeDataPart; - friend class MergeTreeDataPartWide; - friend class MergeTreeDataPartCompact; friend class MergeTreeDataMergerMutator; friend class ReplicatedMergeTreeAlterThread; friend struct ReplicatedMergeTreeTableMetadata; @@ -935,8 +933,6 @@ protected: void setStoragePolicy(const String & new_storage_policy_name, bool only_check = false); - /// Expression for column type conversion. - /// If no conversions are needed, out_expression=nullptr. /// out_rename_map maps column files for the out_expression onto new table files. /// out_force_update_metadata denotes if metadata must be changed even if out_rename_map is empty (used /// for transformation-free changing of Enum values list). diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index d705f24fc80..1eff2a33cf0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -1,30 +1,8 @@ #include "MergeTreeDataPartCompact.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - #include #include #include +#include namespace DB @@ -32,14 +10,8 @@ namespace DB namespace ErrorCodes { - extern const int FILE_DOESNT_EXIST; extern const int NO_FILE_IN_DATA_PART; - extern const int EXPECTED_END_OF_FILE; - extern const int CORRUPTED_DATA; - extern const int NOT_FOUND_EXPECTED_DATA_PART; extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; - extern const int BAD_TTL_FILE; - extern const int CANNOT_UNLINK; } @@ -69,12 +41,13 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( MarkCache * mark_cache, const MergeTreeReaderSettings & reader_settings, const ValueSizeMap & avg_value_size_hints, - const ReadBufferFromFileBase::ProfileCallback & /* profile_callback */) const + const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { /// FIXME maybe avoid shared_from_this return std::make_unique( shared_from_this(), columns_to_read, uncompressed_cache, - mark_cache, mark_ranges, reader_settings, avg_value_size_hints); + mark_cache, mark_ranges, reader_settings, + avg_value_size_hints, profile_callback); } IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter( diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index 834bbb6df6b..fd1d8c18327 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -1,24 +1,6 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include - -#include - -#include - namespace DB { @@ -26,7 +8,6 @@ namespace DB struct ColumnSize; class MergeTreeData; - /// Description of the data part. class MergeTreeDataPartCompact : public IMergeTreeDataPart { @@ -88,7 +69,4 @@ private: void loadIndexGranularity() override; }; - -// using MergeTreeDataPartState =IMergeTreeDataPart::State; - } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 579552cdf3d..95efffc012d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -1,29 +1,6 @@ #include "MergeTreeDataPartWide.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include -#include - #include -#include #include #include @@ -33,22 +10,11 @@ namespace DB namespace ErrorCodes { - extern const int FILE_DOESNT_EXIST; extern const int NO_FILE_IN_DATA_PART; - extern const int EXPECTED_END_OF_FILE; - extern const int CORRUPTED_DATA; - extern const int NOT_FOUND_EXPECTED_DATA_PART; extern const int BAD_SIZE_OF_FILE_IN_DATA_PART; - extern const int BAD_TTL_FILE; - extern const int CANNOT_UNLINK; } -// static ReadBufferFromFile openForReading(const String & path) -// { -// return ReadBufferFromFile(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); -// } - MergeTreeDataPartWide::MergeTreeDataPartWide( MergeTreeData & storage_, const String & name_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index be36b390bb1..7af6d9f7680 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -1,24 +1,6 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include - -#include - -#include - namespace DB { @@ -91,6 +73,4 @@ private: ColumnSize getColumnSizeImpl(const String & name, const IDataType & type, std::unordered_set * processed_substreams) const; }; -// using MergeTreeDataPartState =IMergeTreeDataPart::State; - } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 8cefde6cac9..8b39b522c17 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -1,5 +1,4 @@ #include -#include namespace DB { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index a5984c25b1f..1d4f4df11de 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -80,20 +80,6 @@ void MergeTreeDataPartWriterWide::write(const Block & block, const IColumn::Permutation * permutation, const Block & primary_key_block, const Block & skip_indexes_block) { - // if (serialization_states.empty()) - // { - // serialization_states.reserve(columns_list.size()); - // WrittenOffsetColumns tmp_offset_columns; - // IDataType::SerializeBinaryBulkSettings serialize_settings; - - // for (const auto & col : columns_list) - // { - // serialize_settings.getter = createStreamGetter(col.name, tmp_offset_columns, false); - // serialization_states.emplace_back(nullptr); - // col.type->serializeBinaryBulkStatePrefix(serialize_settings, serialization_states.back()); - // } - // } - /// Fill index granularity for this block /// if it's unknown (in case of insert data or horizontal merge, /// but not in case of vertical merge) @@ -294,6 +280,7 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch { if (!serialization_states.empty()) { + /// FIXME maybe we need skip_offsets=false in some cases serialize_settings.getter = createStreamGetter(it->name, written_offset_columns ? *written_offset_columns : offset_columns); it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[it->name]); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 065e8b5e48e..9a49a6402e1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -6,6 +6,7 @@ namespace DB class MergeTreeDataPartWriterWide : public IMergeTreeDataPartWriter { public: + using ColumnToSize = std::map; MergeTreeDataPartWriterWide( diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index e63b06b3994..967139638b1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -11,6 +11,7 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; + extern const int UNKNOWN_PART_TYPE; } std::optional MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(const std::string & path_to_part) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index c7004b6222f..2c596401228 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -8,11 +8,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int UNKNOWN_PART_TYPE; -} - class MergeTreeData; /// Meta information about index granularity diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index b11c02a5d44..e7027ec52d5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -17,7 +17,8 @@ namespace ErrorCodes MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, - const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_) + const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) : IMergeTreeReader(data_part_, columns_ , uncompressed_cache_, mark_cache_, mark_ranges_ , settings_, avg_value_size_hints_) @@ -31,8 +32,8 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr auto buffer = std::make_unique( full_data_path, uncompressed_cache, 0, settings.min_bytes_to_use_direct_io, buffer_size); - // if (profile_callback) - // buffer->setProfileCallback(profile_callback, clock_type); + if (profile_callback_) + buffer->setProfileCallback(profile_callback_, clock_type_); cached_buffer = std::move(buffer); data_buffer = cached_buffer.get(); @@ -42,8 +43,8 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr auto buffer = std::make_unique( full_data_path, 0, settings.min_bytes_to_use_direct_io, buffer_size); - // if (profile_callback) - // buffer->setProfileCallback(profile_callback, clock_type); + if (profile_callback_) + buffer->setProfileCallback(profile_callback_, clock_type_); non_cached_buffer = std::move(buffer); data_buffer = non_cached_buffer.get(); @@ -110,19 +111,13 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading, { size_t column_size_before_reading = column->size(); - readData(*column, *type, from_mark, *column_positions[pos], rows_to_read, read_only_offsets[pos]); + readData(name, *column, *type, from_mark, *column_positions[pos], rows_to_read, read_only_offsets[pos]); size_t read_rows_in_column = column->size() - column_size_before_reading; if (read_rows_in_column < rows_to_read) throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) + ". Rows expected: " + toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA); - - /// For elements of Nested, column_size_before_reading may be greater than column size - /// if offsets are not empty and were already read, but elements are empty. - /// FIXME - // if (column->size()) - // read_rows_in_mark = std::max(read_rows, column->size() - column_size_before_reading); } catch (Exception & e) { @@ -168,7 +163,7 @@ MergeTreeReaderCompact::ColumnPosition MergeTreeReaderCompact::findColumnForOffs void MergeTreeReaderCompact::readData( - IColumn & column, const IDataType & type, + const String & name, IColumn & column, const IDataType & type, size_t from_mark, size_t column_position, size_t rows_to_read, bool only_offsets) { if (!isContinuousReading(from_mark, column_position)) @@ -184,7 +179,7 @@ void MergeTreeReaderCompact::readData( IDataType::DeserializeBinaryBulkSettings deserialize_settings; deserialize_settings.getter = buffer_getter; - // deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; + deserialize_settings.avg_value_size_hint = avg_value_size_hints[name]; deserialize_settings.position_independent_encoding = true; IDataType::DeserializeBinaryBulkStatePtr state; diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 0dc0434fad4..55a3ab15bac 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -2,7 +2,6 @@ #include #include -#include #include @@ -19,7 +18,9 @@ public: MarkCache * mark_cache_, const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, - const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}); + const ValueSizeMap & avg_value_size_hints_ = ValueSizeMap{}, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_ = ReadBufferFromFileBase::ProfileCallback{}, + clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE); /// Return the number of rows has been read or zero if there is no columns to read. /// If continue_reading is true, continue reading from last state, otherwise seek to from_mark @@ -37,7 +38,7 @@ private: MergeTreeMarksLoader marks_loader; using ColumnPosition = std::optional; - /// Positions of columns in part structe. + /// Positions of columns in part structure. std::vector column_positions; /// Should we read full column or only it's offsets std::vector read_only_offsets; @@ -48,7 +49,7 @@ private: void initMarksLoader(); void seekToMark(size_t row_index, size_t column_index); - void readData(IColumn & column, const IDataType & type, + void readData(const String & name, IColumn & column, const IDataType & type, size_t from_mark, size_t column_position, size_t rows_to_read, bool only_offsets = false); ColumnPosition findColumnForOffsets(const String & column_name); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 7d7c8990c37..1db7b53a799 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -27,8 +27,7 @@ namespace ErrorCodes MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_, - clockid_t clock_type_) + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) : IMergeTreeReader(data_part_, columns_ , uncompressed_cache_, mark_cache_, mark_ranges_ , settings_, avg_value_size_hints_) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index f79af378951..e8ec4b3db74 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -53,7 +53,7 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( MergeTreeReaderSettings reader_settings = { - /// This is hack + /// bytes to use AIO (this is hack) .min_bytes_to_use_direct_io = read_with_direct_io ? 1UL : std::numeric_limits::max(), .max_read_buffer_size = DBMS_DEFAULT_BUFFER_SIZE, .save_marks_in_cache = false diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index baaac34ee79..4081f572d2e 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -1,12 +1,4 @@ #include -#include -#include -#include -#include -#include -#include -#include -#include #include @@ -29,7 +21,8 @@ MergedBlockOutputStream::MergedBlockOutputStream( { MergeTreeWriterSettings writer_settings(data_part->storage.global_context.getSettings(), data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); - writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); + + writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, std::move(writer_settings)); init(); } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 57361487e98..0665a2c58de 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -20,7 +20,8 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( writer_settings.filename_suffix = filename_suffix; writer_settings.skip_offsets = skip_offsets_; - writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, default_codec, writer_settings, index_granularity); + writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, + default_codec,std::move(writer_settings), index_granularity); writer->setWrittenOffsetColumns(offset_columns_); writer->initSkipIndices(); } diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index 0833c89fde9..cdcf9164882 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -1,8 +1,6 @@ #pragma once #include -#include -#include namespace DB { diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index 012e0aecc5d..1039395a8ff 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -24,6 +24,7 @@ namespace DB namespace ErrorCodes { extern const int CORRUPTED_DATA; + extern const int UNKNOWN_PART_TYPE; } From 750b9d9e55b042a25a78fa6675e30c410e0d7a12 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 17 Jan 2020 17:39:18 +0300 Subject: [PATCH 0116/2007] Fix very strange bug --- dbms/src/Interpreters/MutationsInterpreter.cpp | 11 +++++++++++ .../MergeTree/MergeTreeBaseSelectProcessor.cpp | 1 + .../Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 3 ++- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 3 ++- dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp | 2 +- .../Storages/MergeTree/StorageFromMergeTreeDataPart.h | 1 + 6 files changed, 18 insertions(+), 3 deletions(-) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index ad680c71a0f..d5577df3ad2 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -178,6 +178,7 @@ MutationsInterpreter::MutationsInterpreter( std::cerr << "Mutations ast:" << queryToString(mutation_ast) << std::endl; SelectQueryOptions limits = SelectQueryOptions().analyze(!can_execute).ignoreLimits(); select_interpreter = std::make_unique(mutation_ast, context, storage, limits); + std::cerr << "HEADER:" << select_interpreter->getSampleBlock().dumpStructure() << std::endl; } static NameSet getKeyColumns(const StoragePtr & storage) @@ -405,6 +406,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); } + std::cerr << "AFFECTED INDICES COLUMN:" << affected_indices_columns.size() << std::endl; /// We cares about affected indices because we also need to rewrite them /// when one of index columns updated or filtered with delete if (!affected_indices_columns.empty()) @@ -422,6 +424,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) } const ASTPtr select_query = prepareInterpreterSelectQuery(stages_copy, /* dry_run = */ true); + std::cerr << "SELECT query for index:" << queryToString(select_query) << std::endl; InterpreterSelectQuery interpreter{select_query, context, storage, SelectQueryOptions().analyze(/* dry_run = */ false).ignoreLimits()}; auto first_stage_header = interpreter.getSampleBlock(); @@ -431,8 +434,11 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) /// Special step to recalculate affected indices. stages.emplace_back(context); for (const auto & column : affected_indices_columns) + { + std::cerr << "AFFECTED COLUMN:" << column << std::endl; stages.back().column_to_updated.emplace( column, std::make_shared(column)); + } } is_prepared = true; @@ -611,9 +617,14 @@ BlockInputStreamPtr MutationsInterpreter::execute(TableStructureReadLockHolder & throw Exception("Cannot execute mutations interpreter because can_execute flag set to false", ErrorCodes::LOGICAL_ERROR); BlockInputStreamPtr in = select_interpreter->execute().in; + std::cerr << "INNNNNNNN HEADER:" << in->getHeader().dumpStructure() << std::endl; auto result_stream = addStreamsForLaterStages(stages, in); + std::cerr << "RESULTTTTT HEADER:" << result_stream->getHeader().dumpStructure() << std::endl; if (!updated_header) + { + std::cerr << "Saving updated header\n"; updated_header = std::make_unique(result_stream->getHeader()); + } return result_stream; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 183623babf4..7c85c5ce862 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -44,6 +44,7 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( save_marks_in_cache(save_marks_in_cache_), virt_column_names(virt_column_names_) { + std::cerr << "Processor header:" << header.dumpStructure() << std::endl; std::cerr << "HEADER IN SELECT PROCESSOR:" << getPort().getHeader().dumpStructure() << std::endl; //std::cerr << "STACK:" << StackTrace().toString() << std::endl; header_without_virtual_columns = getPort().getHeader(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 36eeb634556..2e50a54f2ab 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1061,8 +1061,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (!indices_to_recalc.empty()) { + std::cerr << "Updated header:" << updated_header.dumpStructure() << std::endl; auto indices_recalc_syntax - = SyntaxAnalyzer(context, {}).analyze(indices_recalc_expr_list, updated_header.getNamesAndTypesList()); + = SyntaxAnalyzer(context, {}).analyze(indices_recalc_expr_list, in->getHeader().getNamesAndTypesList()); auto indices_recalc_expr = ExpressionAnalyzer( indices_recalc_expr_list, indices_recalc_syntax, context).getActions(false); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 043e389458a..4661acdc613 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -184,6 +184,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( for (const String & name : column_names_to_return) { + std::cerr << "Column name to return:" << name << std::endl; if (name == "_part") { part_column_queried = true; @@ -644,7 +645,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( } else { - std::cerr << "Spreading marks among streams\n"; + std::cerr << "Spreading marks among streams columns size:" << column_names_to_read.size() << std::endl; res = spreadMarkRangesAmongStreams( std::move(parts_with_ranges), num_streams, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp index e22b10b8319..6530891b306 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -155,7 +155,7 @@ MarkRanges MergeTreeReadPool::getRestMarks(const MergeTreeDataPart & part, const Block MergeTreeReadPool::getHeader() const { - //////////////////std::cerr << "COLUMN NAMES IN POOL:" << column_names.front() << std::endl; + ////std::cerr << "COLUMN NAMES IN POOL:" << column_names.front() << std::endl; return data.getSampleBlockForColumns(column_names); } diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 2ad4abf607b..594ac27f408 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -29,6 +29,7 @@ public: size_t max_block_size, unsigned num_streams) override { + std::cerr << "StorageFromMergeTreeDataPart: columns names size:" << column_names.size() << std::endl; auto pipes = MergeTreeDataSelectExecutor(part->storage).readFromParts( {part}, column_names, query_info, context, max_block_size, num_streams); From d2e9ed3898d03634ada1dee4774dfc30f87a12f5 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 17 Jan 2020 17:48:56 +0300 Subject: [PATCH 0117/2007] Commented tons of debug prints --- .../src/Interpreters/MutationsInterpreter.cpp | 30 +++++++------- .../MergeTreeBaseSelectProcessor.cpp | 14 +++---- .../MergeTree/MergeTreeDataMergerMutator.cpp | 12 +++--- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 10 ++--- .../Storages/MergeTree/MergeTreeReader.cpp | 40 +++++++++--------- .../MergeTreeSequentialBlockInputStream.cpp | 8 ++-- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 4 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 4 +- .../Storages/StorageReplicatedMergeTree.cpp | 41 ++++++++----------- 9 files changed, 76 insertions(+), 87 deletions(-) diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index d5577df3ad2..caf448b6ba2 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -173,12 +173,12 @@ MutationsInterpreter::MutationsInterpreter( , context(context_) , can_execute(can_execute_) { - std::cerr << "STORAGE IS NULLPTR:" << (storage == nullptr) << std::endl; + //std::cerr << "STORAGE IS NULLPTR:" << (storage == nullptr) << std::endl; mutation_ast = prepare(!can_execute); - std::cerr << "Mutations ast:" << queryToString(mutation_ast) << std::endl; + //std::cerr << "Mutations ast:" << queryToString(mutation_ast) << std::endl; SelectQueryOptions limits = SelectQueryOptions().analyze(!can_execute).ignoreLimits(); select_interpreter = std::make_unique(mutation_ast, context, storage, limits); - std::cerr << "HEADER:" << select_interpreter->getSampleBlock().dumpStructure() << std::endl; + //std::cerr << "HEADER:" << select_interpreter->getSampleBlock().dumpStructure() << std::endl; } static NameSet getKeyColumns(const StoragePtr & storage) @@ -263,20 +263,20 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) if (commands.empty()) throw Exception("Empty mutation commands list", ErrorCodes::LOGICAL_ERROR); - std::cerr << "PREPARING\n"; + //std::cerr << "PREPARING\n"; const ColumnsDescription & columns_desc = storage->getColumns(); const IndicesDescription & indices_desc = storage->getIndices(); - std::cerr << "COLUMNS RECEIVED:" << columns_desc.toString() << std::endl; + //std::cerr << "COLUMNS RECEIVED:" << columns_desc.toString() << std::endl; NamesAndTypesList all_columns = columns_desc.getAllPhysical(); - std::cerr << "COMMANDS SIZE:" << commands.size() << std::endl; + //std::cerr << "COMMANDS SIZE:" << commands.size() << std::endl; NameSet updated_columns; for (const MutationCommand & command : commands) { for (const auto & kv : command.column_to_update_expression) { - std::cerr << "COLUMN:" << kv.first << std::endl; + //std::cerr << "COLUMN:" << kv.first << std::endl; updated_columns.insert(kv.first); } } @@ -322,7 +322,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) /// First, break a sequence of commands into stages. for (const auto & command : commands) { - std::cerr << "Processing command:" << command.ast << std::endl; + //std::cerr << "Processing command:" << command.ast << std::endl; if (command.type == MutationCommand::DELETE) { if (stages.empty() || !stages.back().column_to_updated.empty()) @@ -406,7 +406,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); } - std::cerr << "AFFECTED INDICES COLUMN:" << affected_indices_columns.size() << std::endl; + //std::cerr << "AFFECTED INDICES COLUMN:" << affected_indices_columns.size() << std::endl; /// We cares about affected indices because we also need to rewrite them /// when one of index columns updated or filtered with delete if (!affected_indices_columns.empty()) @@ -424,7 +424,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) } const ASTPtr select_query = prepareInterpreterSelectQuery(stages_copy, /* dry_run = */ true); - std::cerr << "SELECT query for index:" << queryToString(select_query) << std::endl; + //std::cerr << "SELECT query for index:" << queryToString(select_query) << std::endl; InterpreterSelectQuery interpreter{select_query, context, storage, SelectQueryOptions().analyze(/* dry_run = */ false).ignoreLimits()}; auto first_stage_header = interpreter.getSampleBlock(); @@ -435,7 +435,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) stages.emplace_back(context); for (const auto & column : affected_indices_columns) { - std::cerr << "AFFECTED COLUMN:" << column << std::endl; + //std::cerr << "AFFECTED COLUMN:" << column << std::endl; stages.back().column_to_updated.emplace( column, std::make_shared(column)); } @@ -449,7 +449,7 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & prepared_stages, bool dry_run) { NamesAndTypesList all_columns = storage->getColumns().getAllPhysical(); - std::cerr << "Prepare interpreter storage columns:" << all_columns.toString() << std::endl; + //std::cerr << "Prepare interpreter storage columns:" << all_columns.toString() << std::endl; /// Next, for each stage calculate columns changed by this and previous stages. @@ -617,12 +617,12 @@ BlockInputStreamPtr MutationsInterpreter::execute(TableStructureReadLockHolder & throw Exception("Cannot execute mutations interpreter because can_execute flag set to false", ErrorCodes::LOGICAL_ERROR); BlockInputStreamPtr in = select_interpreter->execute().in; - std::cerr << "INNNNNNNN HEADER:" << in->getHeader().dumpStructure() << std::endl; + //std::cerr << "INNNNNNNN HEADER:" << in->getHeader().dumpStructure() << std::endl; auto result_stream = addStreamsForLaterStages(stages, in); - std::cerr << "RESULTTTTT HEADER:" << result_stream->getHeader().dumpStructure() << std::endl; + //std::cerr << "RESULTTTTT HEADER:" << result_stream->getHeader().dumpStructure() << std::endl; if (!updated_header) { - std::cerr << "Saving updated header\n"; + //std::cerr << "Saving updated header\n"; updated_header = std::make_unique(result_stream->getHeader()); } return result_stream; diff --git a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp index 7c85c5ce862..816efdae014 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeBaseSelectProcessor.cpp @@ -44,9 +44,9 @@ MergeTreeBaseSelectProcessor::MergeTreeBaseSelectProcessor( save_marks_in_cache(save_marks_in_cache_), virt_column_names(virt_column_names_) { - std::cerr << "Processor header:" << header.dumpStructure() << std::endl; - std::cerr << "HEADER IN SELECT PROCESSOR:" << getPort().getHeader().dumpStructure() << std::endl; - //std::cerr << "STACK:" << StackTrace().toString() << std::endl; + //std::cerr << "Processor header:" << header.dumpStructure() << std::endl; + //std::cerr << "HEADER IN SELECT PROCESSOR:" << getPort().getHeader().dumpStructure() << std::endl; + ////std::cerr << "STACK:" << StackTrace().toString() << std::endl; header_without_virtual_columns = getPort().getHeader(); for (auto it = virt_column_names.rbegin(); it != virt_column_names.rend(); ++it) @@ -104,7 +104,7 @@ void MergeTreeBaseSelectProcessor::initializeRangeReaders(MergeTreeReadTask & cu Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() { - std::cerr << "Reading from part impl\n"; + //std::cerr << "Reading from part impl\n"; if (task->size_predictor) task->size_predictor->startBlock(); @@ -153,11 +153,7 @@ Chunk MergeTreeBaseSelectProcessor::readFromPartImpl() UInt64 rows_to_read = std::max(UInt64(1), std::min(current_max_block_size_rows, recommended_rows)); auto read_result = task->range_reader.read(rows_to_read, task->mark_ranges); - std::cerr << "Read result:\n"; - for (const auto & column : read_result.columns) - { - std::cerr << "Column:" << column->dumpStructure() << std::endl; - } + //std::cerr << "Read result:\n"; /// All rows were filtered. Repeat. if (read_result.num_rows == 0) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 2e50a54f2ab..6250ed5b3cc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1061,7 +1061,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor if (!indices_to_recalc.empty()) { - std::cerr << "Updated header:" << updated_header.dumpStructure() << std::endl; + //std::cerr << "Updated header:" << updated_header.dumpStructure() << std::endl; auto indices_recalc_syntax = SyntaxAnalyzer(context, {}).analyze(indices_recalc_expr_list, in->getHeader().getNamesAndTypesList()); auto indices_recalc_expr = ExpressionAnalyzer( @@ -1148,7 +1148,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor continue; Poco::Path destination(new_part_tmp_path); - std::cerr << "HARDLINKING FROM:" << dir_it.path().toString() << " TO " << destination.toString() << std::endl; + //std::cerr << "HARDLINKING FROM:" << dir_it.path().toString() << " TO " << destination.toString() << std::endl; destination.append(dir_it.name()); createHardLink(dir_it.path().toString(), destination.toString()); @@ -1159,7 +1159,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor new_data_part->checksums = source_part->checksums; if (in) { - std::cerr << "Updated header:" << updated_header.dumpStructure() << std::endl; + //std::cerr << "Updated header:" << updated_header.dumpStructure() << std::endl; IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( data, @@ -1182,8 +1182,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor { out.write(block); - std::cerr << "Block readed:" << block.dumpStructure() << std::endl; - std::cerr << "Block rows:" << block.rows() << std::endl; + //std::cerr << "Block readed:" << block.dumpStructure() << std::endl; + //std::cerr << "Block rows:" << block.rows() << std::endl; merge_entry->rows_written += block.rows(); merge_entry->bytes_written_uncompressed += block.bytes(); } @@ -1196,7 +1196,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor } else { - std::cerr << "Updated header empty\n"; + //std::cerr << "Updated header empty\n"; } for (const String & removed_file : remove_files) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 4661acdc613..78282abb93e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -184,7 +184,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( for (const String & name : column_names_to_return) { - std::cerr << "Column name to return:" << name << std::endl; + //std::cerr << "Column name to return:" << name << std::endl; if (name == "_part") { part_column_queried = true; @@ -432,7 +432,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( if (upper_limit_rational < size_of_universum) has_upper_limit = true; - /*std::cerr << std::fixed << std::setprecision(100) + /*//std::cerr << std::fixed << std::setprecision(100) << "relative_sample_size: " << relative_sample_size << "\n" << "relative_sample_offset: " << relative_sample_offset << "\n" << "lower_limit_float: " << lower_limit_rational << "\n" @@ -645,7 +645,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts( } else { - std::cerr << "Spreading marks among streams columns size:" << column_names_to_read.size() << std::endl; + //std::cerr << "Spreading marks among streams columns size:" << column_names_to_read.size() << std::endl; res = spreadMarkRangesAmongStreams( std::move(parts_with_ranges), num_streams, @@ -765,7 +765,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( num_streams, sum_marks, min_marks_for_concurrent_read, parts, data, query_info.prewhere_info, true, column_names, MergeTreeReadPool::BackoffSettings(settings), settings.preferred_block_size_bytes, false); - std::cerr << "POOL HEADER:" << pool->getHeader().dumpStructure() << std::endl; + //std::cerr << "POOL HEADER:" << pool->getHeader().dumpStructure() << std::endl; /// Let's estimate total number of rows for progress bar. LOG_TRACE(log, "Reading approx. " << total_rows << " rows with " << num_streams << " streams"); @@ -793,7 +793,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreams( { RangesInDataPart & part = parts[part_index]; - std::cerr << "Creating sequential stream from part:" << part_index << std::endl; + //std::cerr << "Creating sequential stream from part:" << part_index << std::endl; auto source = std::make_shared( data, part.data_part, max_block_size, settings.preferred_block_size_bytes, settings.preferred_max_column_in_block_size_bytes, column_names, part.ranges, use_uncompressed_cache, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp index 5b115d2ee7a..964433fd385 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReader.cpp @@ -56,7 +56,7 @@ MergeTreeReader::MergeTreeReader( , mmap_threshold(mmap_threshold_) , max_read_buffer_size(max_read_buffer_size_) { - //std::cerr << "Merge tree reader created for part:" << data_part->name << std::endl; + ////std::cerr << "Merge tree reader created for part:" << data_part->name << std::endl; try { for (const NameAndTypePair & column_from_part : data_part->columns) @@ -66,22 +66,22 @@ MergeTreeReader::MergeTreeReader( for (const NameAndTypePair & column : columns) { - //std::cerr << "Column name to read:" << column.name << std::endl; + ////std::cerr << "Column name to read:" << column.name << std::endl; if (columns_from_part.count(column.name)) { - //std::cerr << "With type:" << columns_from_part[column.name]->getName() << std::endl; - //std::cerr << "Original type:" << column.type->getName() << std::endl; + ////std::cerr << "With type:" << columns_from_part[column.name]->getName() << std::endl; + ////std::cerr << "Original type:" << column.type->getName() << std::endl; addStreams(column.name, *columns_from_part[column.name], profile_callback_, clock_type_); } else { - //std::cerr << "Original type:" << column.type->getName() << std::endl; + ////std::cerr << "Original type:" << column.type->getName() << std::endl; addStreams(column.name, *column.type, profile_callback_, clock_type_); } } - //std::cerr << "COLUMNS IN CONSTRUCTOR:" << columns.toString() << std::endl; + ////std::cerr << "COLUMNS IN CONSTRUCTOR:" << columns.toString() << std::endl; } catch (...) { @@ -241,7 +241,7 @@ void MergeTreeReader::readData( size_t from_mark, bool continue_reading, size_t max_rows_to_read, bool with_offsets) { - //std::cerr << "READ DATA:" << name << " with type:" << type.getName() << std::endl; + ////std::cerr << "READ DATA:" << name << " with type:" << type.getName() << std::endl; auto get_stream_getter = [&](bool stream_for_prefix) -> IDataType::InputStreamGetter { return [&, stream_for_prefix](const IDataType::SubstreamPath & substream_path) -> ReadBuffer * @@ -393,7 +393,7 @@ void MergeTreeReader::fillMissingColumns(Columns & res_columns, bool & should_ev void MergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns & res_columns) { - std::cerr << "EVALUATING\n"; + //std::cerr << "EVALUATING\n"; try { size_t num_columns = columns.size(); @@ -414,9 +414,9 @@ void MergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns additional_columns.insert({res_columns[pos], name_and_type->type, name_and_type->name}); } - std::cerr << "additional columns before:" << additional_columns.dumpStructure() << std::endl; + //std::cerr << "additional columns before:" << additional_columns.dumpStructure() << std::endl; DB::evaluateMissingDefaults(additional_columns, columns, storage.getColumns().getDefaults(), storage.global_context); - std::cerr << "additional columns after:" << additional_columns.dumpStructure() << std::endl; + //std::cerr << "additional columns after:" << additional_columns.dumpStructure() << std::endl; /// Move columns from block. name_and_type = columns.begin(); @@ -451,32 +451,32 @@ void MergeTreeReader::performRequiredConversions(Columns & res_columns) Block copy_block; auto name_and_type = columns.begin(); - //std::cerr << "DATAPART NAMES AND TYPES:" << data_part->columns.toString() << std::endl; - //std::cerr << "REQUIRED COLUMNS NAMES AND TYPES:" << columns.toString() << std::endl; - //std::cerr << "RES COLUMNS SIZE:" << res_columns.size() << std::endl; - //std::cerr << "RES COLUMNS STRUCTURE:\n"; + ////std::cerr << "DATAPART NAMES AND TYPES:" << data_part->columns.toString() << std::endl; + ////std::cerr << "REQUIRED COLUMNS NAMES AND TYPES:" << columns.toString() << std::endl; + ////std::cerr << "RES COLUMNS SIZE:" << res_columns.size() << std::endl; + ////std::cerr << "RES COLUMNS STRUCTURE:\n"; //for (const auto & column : res_columns) //{ - // std::cerr << column->dumpStructure() << std::endl; + // //std::cerr << column->dumpStructure() << std::endl; //} for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type) { - //std::cerr << "POS:" << pos << std::endl; + ////std::cerr << "POS:" << pos << std::endl; if (res_columns[pos] == nullptr) continue; - //std::cerr << "POS NAME:" << name_and_type->name << std::endl; - //std::cerr << "POS TYPE:" << name_and_type->type->getName() << std::endl; + ////std::cerr << "POS NAME:" << name_and_type->name << std::endl; + ////std::cerr << "POS TYPE:" << name_and_type->type->getName() << std::endl; if (columns_from_part.count(name_and_type->name)) copy_block.insert({res_columns[pos], columns_from_part[name_and_type->name], name_and_type->name}); else copy_block.insert({res_columns[pos], name_and_type->type, name_and_type->name}); } - //std::cerr << "Copy block: " << copy_block.dumpStructure() << std::endl; + ////std::cerr << "Copy block: " << copy_block.dumpStructure() << std::endl; DB::performRequiredConversions(copy_block, columns, storage.global_context); - //std::cerr << "Result copy block: " << copy_block.dumpStructure() << std::endl; + ////std::cerr << "Result copy block: " << copy_block.dumpStructure() << std::endl; /// Move columns from block. name_and_type = columns.begin(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp index ce58da673e9..e0bcd54c9b3 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeSequentialBlockInputStream.cpp @@ -42,13 +42,13 @@ MergeTreeSequentialBlockInputStream::MergeTreeSequentialBlockInputStream( NamesAndTypesList columns_for_reader; if (take_column_types_from_storage) { - std::cerr << "Taking columns from storage\n"; + //std::cerr << "Taking columns from storage\n"; const NamesAndTypesList & physical_columns = storage.getColumns().getAllPhysical(); columns_for_reader = physical_columns.addTypes(columns_to_read); } else { - std::cerr << "Taking columns from data part\n"; + //std::cerr << "Taking columns from data part\n"; /// take columns from data_part columns_for_reader = data_part->columns.addTypes(columns_to_read); } @@ -88,7 +88,7 @@ Block MergeTreeSequentialBlockInputStream::getHeader() const Block MergeTreeSequentialBlockInputStream::readImpl() try { - //std::cerr << "READING\n"; + ////std::cerr << "READING\n"; Block res; if (!isCancelled() && current_row < data_part->rows_count) { @@ -106,7 +106,7 @@ try bool should_evaluate_missing_defaults = false; reader->fillMissingColumns(columns, should_evaluate_missing_defaults, rows_readed); - //std::cerr << "Should evaluate missing defaults:" << should_evaluate_missing_defaults << std::endl; + ////std::cerr << "Should evaluate missing defaults:" << should_evaluate_missing_defaults << std::endl; if (should_evaluate_missing_defaults) { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 7c6cfa42873..f314bacdc53 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -164,10 +164,10 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) in >> required_mutation_znode >> "\nfinish\n"; } - std::cerr << "Read backn\n"; + //std::cerr << "Read backn\n"; in >> "\n"; - std::cerr << "Readed\n"; + //std::cerr << "Readed\n"; /// Optional field. if (!in.eof()) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 9af93996c0c..2707d44c8f6 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -517,7 +517,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C } catch (...) { - std::cerr << "DIE HERE:\n"; + //std::cerr << "DIE HERE:\n"; tryLogCurrentException(log); /// If it fails, the data in RAM is incorrect. In order to avoid possible further corruption of data in ZK, we will kill ourselves. /// This is possible only if there is an unknown logical error. @@ -1012,7 +1012,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (entry.type == LogEntry::FINISH_ALTER) { - std::cerr << "Entry finish alter\n"; + //std::cerr << "Entry finish alter\n"; if (mutations_by_znode.count(entry.required_mutation_znode) && !mutations_by_znode.at(entry.required_mutation_znode).is_done) { String reason = "Not altering storage because mutation " + entry.required_mutation_znode + " is not ready yet (mutation is beeing processed)."; LOG_TRACE(log, reason); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 806ce71a206..b35a10fd9f6 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1158,7 +1158,7 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & /*entry*/) { - std::cerr << "Trying to finish alter\n"; + ////std::cerr << "Trying to finish alter\n"; auto zookeeper = getZooKeeper(); String columns_path = zookeeper_path + "/columns"; @@ -1179,20 +1179,20 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer const bool changed_columns_version = (columns_version_zk != this->columns_version); const bool changed_metadata_version = (metadata_version_zk != this->metadata_version); - std::cerr << "Versions changed: columns:" << changed_columns_version << " metadata:" << changed_metadata_version << std::endl; + ////std::cerr << "Versions changed: columns:" << changed_columns_version << " metadata:" << changed_metadata_version << std::endl; if (!(changed_columns_version || changed_metadata_version)) { - std::cerr << "Nothing changed\n"; + ////std::cerr << "Nothing changed\n"; return true; } - std::cerr << "Receiving metadata from zookeeper\n"; + ////std::cerr << "Receiving metadata from zookeeper\n"; auto columns_in_zk = ColumnsDescription::parse(columns_str); auto metadata_in_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); - std::cerr << "Metadata received\n"; + ////std::cerr << "Metadata received\n"; MergeTreeData::DataParts parts; @@ -1219,29 +1219,22 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Applied changes to the metadata of the table."); } - std::cerr << "Columns version before:" << columns_version << std::endl; - std::cerr << "Columns version after:" << columns_version_zk << std::endl; + ////std::cerr << "Columns version before:" << columns_version << std::endl; + ////std::cerr << "Columns version after:" << columns_version_zk << std::endl; columns_version = columns_version_zk; metadata_version = metadata_version_zk; - std::cerr << "Recalculating columns sizes\n"; + ////std::cerr << "Recalculating columns sizes\n"; recalculateColumnSizes(); /// Update metadata ZK nodes for a specific replica. if (changed_columns_version) - { zookeeper->set(replica_path + "/columns", columns_str); - } - else - { - } if (changed_metadata_version) - { zookeeper->set(replica_path + "/metadata", metadata_str); - } - std::cerr << "Nodes in zk updated\n"; + ////std::cerr << "Nodes in zk updated\n"; } - std::cerr << "Done\n"; + ////std::cerr << "Done\n"; return true; } @@ -3336,7 +3329,7 @@ void StorageReplicatedMergeTree::alter( return queryToString(query); }; - std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; + //std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; /// /columns and /metadata nodes std::vector changed_nodes; @@ -3399,7 +3392,7 @@ void StorageReplicatedMergeTree::alter( entry.type = LogEntry::FINISH_ALTER; entry.source_replica = replica_name; - //std::cerr << " Columns before mutation:" << getColumns().getAllPhysical().toString() << std::endl; + ////std::cerr << " Columns before mutation:" << getColumns().getAllPhysical().toString() << std::endl; entry.new_part_name = ""; entry.create_time = time(nullptr); @@ -3407,18 +3400,18 @@ void StorageReplicatedMergeTree::alter( String path_created = getZooKeeper()->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - std::cerr << "Waiting for replicas\n"; + ////std::cerr << "Waiting for replicas\n"; auto unwaited = waitForAllReplicasToProcessLogEntry(entry, false); - std::cerr << "Replicas done"; + ////std::cerr << "Replicas done"; if (!maybe_mutation_commands.empty()) { - std::cerr << "We have mutation commands:" << maybe_mutation_commands.size() << std::endl; + ////std::cerr << "We have mutation commands:" << maybe_mutation_commands.size() << std::endl; Context copy_context = query_context; copy_context.getSettingsRef().mutations_sync = 2; ReplicatedMergeTreeMutationEntry mutation_entry = mutateImpl(maybe_mutation_commands, copy_context); - std::cerr << "Mutation finished\n"; + ////std::cerr << "Mutation finished\n"; } if (!unwaited.empty()) @@ -4500,7 +4493,7 @@ ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const Mu else if (query_context.getSettingsRef().mutations_sync == 1) /// just wait for ourself replicas.push_back(replica_path); - std::cerr << "Waiting for mutation on replicas:" << replicas.size() << std::endl; + //std::cerr << "Waiting for mutation on replicas:" << replicas.size() << std::endl; waitMutationToFinishOnReplicas(replicas, entry.znode_name); } From 795f36b110d7ba749e02c18ae370494caff7966b Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 17 Jan 2020 18:46:26 +0300 Subject: [PATCH 0118/2007] Better defaults --- dbms/src/Storages/AlterCommands.cpp | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/AlterCommands.cpp b/dbms/src/Storages/AlterCommands.cpp index 62e8398a98a..7b7a4d19b78 100644 --- a/dbms/src/Storages/AlterCommands.cpp +++ b/dbms/src/Storages/AlterCommands.cpp @@ -760,11 +760,18 @@ void AlterCommands::prepare(const StorageInMemoryMetadata & metadata, const Cont command->default_expression->clone(), std::make_shared(explicit_type->getName())); - //TODO(alesap) - //command->ast = std::make_shared(); - //command->type = ASTAlterCommand::MODIFY_COLUMN; - //command->col_decl = std::make_shared(); - //command->col_decl->name = column.name; + if (!command->ast) + { + //TODO(alesap) Understand how we get heere + auto ast = std::make_shared(); + ast->type = ASTAlterCommand::MODIFY_COLUMN; + auto col_decl = std::make_shared(); + col_decl->name = column.name; + col_decl->default_specifier = toString(command->default_kind); + col_decl->default_expression = command->default_expression->clone(); + ast->col_decl = col_decl; + command->ast = ast; + } } } else @@ -796,6 +803,7 @@ void AlterCommands::validate(const StorageInMemoryMetadata & metadata, const Con if (!metadata.columns.has(column_name)) if (!command.if_exists) throw Exception{"Wrong column name. Cannot find column " + column_name + " to modify", ErrorCodes::ILLEGAL_COLUMN}; + } } From fa5aa5fe6aa9147d8ffb972fed8fa1fe4aa7f9c7 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 17 Jan 2020 22:09:22 +0300 Subject: [PATCH 0119/2007] Remove debug print --- dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 594ac27f408..8f16bd53e7e 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -29,7 +29,7 @@ public: size_t max_block_size, unsigned num_streams) override { - std::cerr << "StorageFromMergeTreeDataPart: columns names size:" << column_names.size() << std::endl; + //std::cerr << "StorageFromMergeTreeDataPart: columns names size:" << column_names.size() << std::endl; auto pipes = MergeTreeDataSelectExecutor(part->storage).readFromParts( {part}, column_names, query_info, context, max_block_size, num_streams); From be5fb9a517f475129b57be4d1413a3a3fd5fda38 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 17 Jan 2020 23:55:31 +0300 Subject: [PATCH 0120/2007] Less comments --- dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp | 7 ------- dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp | 1 - 2 files changed, 8 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp index 69fcd0f6116..3a43614d3e9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPart.cpp @@ -464,13 +464,6 @@ void MergeTreeDataPart::remove() const LOG_ERROR(storage.log, "Cannot quickly remove directory " << to << " by removing files; fallback to recursive removal. Reason: " << getCurrentExceptionMessage(false)); - std::vector files; - to_dir.list(files); - for (const auto & f : files) - { - std::cerr << "NOT REMOVED FILE:" << f << std::endl; - } - to_dir.remove(true); } } diff --git a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp index 6faca2c2c3f..b78f2bc49eb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -594,7 +594,6 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar merge_tree_reader->fillMissingColumns(columns, should_evaluate_missing_defaults, num_rows); } - std::cerr << "SHOULD eVALUATE:" << should_evaluate_missing_defaults << std::endl; if (!columns.empty() && should_evaluate_missing_defaults) { auto block = prev_reader->sample_block.cloneWithColumns(read_result.columns); From 1dab2b756d08c4a4e0f0d8ba7bc2d0a21dce52f5 Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 18 Jan 2020 12:16:34 +0800 Subject: [PATCH 0121/2007] fix code style --- dbms/src/Functions/array/arrayScalarProduct.h | 104 ++++++++---------- dbms/src/Functions/array/auc.cpp | 53 +++++---- 2 files changed, 75 insertions(+), 82 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index 9d25f551483..82e73b8c17a 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -1,24 +1,23 @@ -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include #include +#include +#include +#include +#include namespace DB { - namespace ErrorCodes { extern const int LOGICAL_ERROR; @@ -32,7 +31,7 @@ class FunctionArrayScalarProduct : public IFunction public: static constexpr auto name = Name::name; static FunctionPtr create(const Context & context) { return std::make_shared(context); } - FunctionArrayScalarProduct(const Context & context_): context(context_) {} + FunctionArrayScalarProduct(const Context & context_) : context(context_) {} private: using ResultColumnType = ColumnVector; @@ -40,16 +39,11 @@ private: template bool executeNumber(Block & block, const ColumnNumbers & arguments, size_t result) { - return executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result) - || executeNumberNumber(block, arguments, result); + return executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result) + || executeNumberNumber(block, arguments, result) || executeNumberNumber(block, arguments, result); } @@ -58,29 +52,31 @@ private: { ColumnPtr col1 = block.getByPosition(arguments[0]).column->convertToFullColumnIfConst(); ColumnPtr col2 = block.getByPosition(arguments[1]).column->convertToFullColumnIfConst(); - if (! col1 || ! col2) + if (!col1 || !col2) return false; - const ColumnArray* col_array1 = checkAndGetColumn(col1.get()); - const ColumnArray* col_array2 = checkAndGetColumn(col2.get()); - if (! col_array1 || ! col_array2) + const ColumnArray * col_array1 = checkAndGetColumn(col1.get()); + const ColumnArray * col_array2 = checkAndGetColumn(col2.get()); + if (!col_array1 || !col_array2) return false; const ColumnVector * col_nested1 = checkAndGetColumn>(col_array1->getData()); const ColumnVector * col_nested2 = checkAndGetColumn>(col_array2->getData()); - if (! col_nested1 || ! col_nested2) + if (!col_nested1 || !col_nested2) return false; auto col_res = ResultColumnType::create(); - vector(col_nested1->getData(), col_array1->getOffsets(), - col_nested2->getData(), col_array2->getOffsets(), col_res->getData()); + vector(col_nested1->getData(), col_array1->getOffsets(), col_nested2->getData(), col_array2->getOffsets(), col_res->getData()); block.getByPosition(result).column = std::move(col_res); return true; } template - static void vector(const PaddedPODArray & data1, const ColumnArray::Offsets & offsets1, - const PaddedPODArray & data2, const ColumnArray::Offsets & offsets2, + static void vector( + const PaddedPODArray & data1, + const ColumnArray::Offsets & offsets1, + const PaddedPODArray & data2, + const ColumnArray::Offsets & offsets2, PaddedPODArray & result) { size_t size = offsets1.size(); @@ -88,7 +84,8 @@ private: ColumnArray::Offset current_offset1 = 0; ColumnArray::Offset current_offset2 = 0; - for (size_t i = 0; i < size; ++i) { + for (size_t i = 0; i < size; ++i) + { size_t array1_size = offsets1[i] - current_offset1; size_t array2_size = offsets2[i] - current_offset2; result[i] = Method::apply(data1, current_offset1, array1_size, data2, current_offset2, array2_size); @@ -100,10 +97,7 @@ private: public: /// Get function name. - String getName() const override - { - return name; - } + String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } @@ -111,20 +105,19 @@ public: { // Basic type check std::vector nested_types(2, nullptr); - for (size_t i = 0; i < getNumberOfArguments(); ++i) + for (size_t i = 0; i < getNumberOfArguments(); ++i) { const DataTypeArray * array_type = checkAndGetDataType(arguments[i].get()); if (!array_type) - throw Exception("All argument for function " + getName() + " must be an array.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - + throw Exception("All argument for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + auto & nested_type = array_type->getNestedType(); WhichDataType which(nested_type); bool is_number = which.isNativeInt() || which.isNativeUInt() || which.isFloat(); - if (! is_number) + if (!is_number) { - throw Exception(getName() + " cannot process values of type " + nested_type->getName(), - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + getName() + " cannot process values of type " + nested_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } nested_types[i] = nested_type; } @@ -135,19 +128,14 @@ public: void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result, size_t /* input_rows_count */) override { - if (!(executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result) - || executeNumber(block, arguments, result))) - throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() - + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN}; - + if (!(executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result) + || executeNumber(block, arguments, result) || executeNumber(block, arguments, result))) + throw Exception{"Illegal column " + block.getByPosition(arguments[0]).column->getName() + " of first argument of function " + + getName(), + ErrorCodes::ILLEGAL_COLUMN}; } private: diff --git a/dbms/src/Functions/array/auc.cpp b/dbms/src/Functions/array/auc.cpp index 2ac7d85a501..7091452ea0b 100644 --- a/dbms/src/Functions/array/auc.cpp +++ b/dbms/src/Functions/array/auc.cpp @@ -1,19 +1,21 @@ -#include #include +#include #include #include "arrayScalarProduct.h" namespace DB { +struct NameAUC +{ + static constexpr auto name = "auc"; +}; -struct NameAUC { static constexpr auto name = "auc"; }; - -class AUCImpl +class AUCImpl { public: using ResultType = Float64; - struct ScoreLabel + struct ScoreLabel { ResultType score; UInt8 label; @@ -22,38 +24,41 @@ public: static DataTypePtr getReturnType(const DataTypePtr & /* nested_type1 */, const DataTypePtr & nested_type2) { WhichDataType which2(nested_type2); - if (! which2.isUInt8()) { - throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + if (!which2.isUInt8()) + { + throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared>(); } template - static ResultType apply(const PaddedPODArray & scores, ColumnArray::Offset score_offset, size_t score_len, - const PaddedPODArray & labels, ColumnArray::Offset label_offset, size_t label_len) + static ResultType apply( + const PaddedPODArray & scores, + ColumnArray::Offset score_offset, + size_t score_len, + const PaddedPODArray & labels, + ColumnArray::Offset label_offset, + size_t label_len) { - if (score_len != label_len) - throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), - ErrorCodes::LOGICAL_ERROR}; - if (score_len == 0) + if (score_len != label_len) + throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), ErrorCodes::LOGICAL_ERROR}; + if (score_len == 0) return {}; // Order pairs of score and lable by score ascending size_t num_pos = 0; size_t num_neg = 0; std::vector pairs(score_len); - for (size_t i = 0; i < score_len; ++i) + for (size_t i = 0; i < score_len; ++i) { pairs[i].score = scores[i + score_offset]; pairs[i].label = (labels[i + label_offset] ? 1 : 0); - if (pairs[i].label) + if (pairs[i].label) ++num_pos; - else + else ++num_neg; } - std::sort(pairs.begin(), pairs.end(), - [](const auto & lhs, const auto & rhs) {return lhs.score < rhs.score; }); + std::sort(pairs.begin(), pairs.end(), [](const auto & lhs, const auto & rhs) { return lhs.score < rhs.score; }); // Calculate AUC size_t curr_cnt = 0; @@ -61,18 +66,18 @@ public: size_t curr_sum = 0; ResultType last_score = -1; ResultType rank_sum = 0; - for (size_t i = 0; i < pairs.size(); ++i) + for (size_t i = 0; i < pairs.size(); ++i) { if (pairs[i].score == last_score) { curr_sum += i + 1; ++curr_cnt; - if (pairs[i].label) + if (pairs[i].label) ++curr_pos_cnt; } - else + else { - if (i > 0) + if (i > 0) rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; curr_sum = i + 1; curr_cnt = 1; @@ -81,7 +86,7 @@ public: last_score = pairs[i].score; } rank_sum += ResultType(curr_sum * curr_pos_cnt) / curr_cnt; - return (rank_sum - num_pos*(num_pos+1)/2)/(num_pos * num_neg); + return (rank_sum - num_pos * (num_pos + 1) / 2) / (num_pos * num_neg); } }; From 07f2d88ed4230ecb724fc4c3ba57b1910e677fcf Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 18 Jan 2020 14:46:06 +0800 Subject: [PATCH 0122/2007] fix bug --- dbms/tests/queries/0_stateless/01064_array_auc.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql index 33b54e7db45..8886426f098 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.sql +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -1 +1 @@ -select select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) \ No newline at end of file +select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) From c66fc0c8b58af73237c7fae99b96dc93cd86b32b Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 18 Jan 2020 18:03:14 +0800 Subject: [PATCH 0123/2007] fix bug --- dbms/src/Functions/array/arrayScalarProduct.h | 2 +- dbms/src/Functions/array/auc.cpp | 2 +- dbms/tests/queries/0_stateless/01064_array_auc.reference | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index 82e73b8c17a..c450f41f48a 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -142,4 +142,4 @@ private: const Context & context; }; -} \ No newline at end of file +} diff --git a/dbms/src/Functions/array/auc.cpp b/dbms/src/Functions/array/auc.cpp index 7091452ea0b..7a2daf0fdd0 100644 --- a/dbms/src/Functions/array/auc.cpp +++ b/dbms/src/Functions/array/auc.cpp @@ -99,4 +99,4 @@ void registerFunctionAUC(FunctionFactory & factory) } -} \ No newline at end of file +} diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.reference b/dbms/tests/queries/0_stateless/01064_array_auc.reference index d6e1fa0f619..39c64a9f26e 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.reference +++ b/dbms/tests/queries/0_stateless/01064_array_auc.reference @@ -1 +1 @@ -0.75 \ No newline at end of file +0.75 From 47578b8dc10101b36169c0b5bcc8ff3730fafbed Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 20 Jan 2020 17:28:51 +0300 Subject: [PATCH 0124/2007] Remove some garbage --- dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp | 1 - dbms/src/Storages/StorageReplicatedMergeTree.cpp | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 2707d44c8f6..eef2710ea20 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -517,7 +517,6 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C } catch (...) { - //std::cerr << "DIE HERE:\n"; tryLogCurrentException(log); /// If it fails, the data in RAM is incorrect. In order to avoid possible further corruption of data in ZK, we will kill ourselves. /// This is possible only if there is an unknown logical error. diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 9087ab6db08..e76549bfc03 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1188,8 +1188,8 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer throw Exception(metadata_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); int32_t metadata_version_zk = metadata_znode_stat.version; - const bool changed_columns_version = (columns_version_zk != this->columns_version); - const bool changed_metadata_version = (metadata_version_zk != this->metadata_version); + const bool changed_columns_version = (columns_version_zk != columns_version); + const bool changed_metadata_version = (metadata_version_zk != metadata_version); ////std::cerr << "Versions changed: columns:" << changed_columns_version << " metadata:" << changed_metadata_version << std::endl; From d0731878c990add9710b7b58f614d62ed8de3bd2 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 20 Jan 2020 21:58:56 +0300 Subject: [PATCH 0125/2007] fix mutations --- .../MergeTree/IMergedBlockOutputStream.cpp | 2 - .../MergeTree/MergeTreeDataMergerMutator.cpp | 2 +- .../0_stateless/01055_compact_parts.reference | 47 +++++++++++++++++++ .../0_stateless/01055_compact_parts.sh | 46 ++++++++++++++++++ 4 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01055_compact_parts.reference create mode 100755 dbms/tests/queries/0_stateless/01055_compact_parts.sh diff --git a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index f8b32d86a9e..72b82e29e9b 100644 --- a/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -32,6 +32,4 @@ Block IMergedBlockOutputStream::getBlockAndPermute(const Block & block, const Na return result; } -/// Implementation of IMergedBlockOutputStream::ColumnStream. - } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index d514c007c7b..ed51affb1ae 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1021,7 +1021,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor MergeStageProgress stage_progress(1.0); in->setProgressCallback(MergeProgressCallback(merge_entry, watch_prev_elapsed, stage_progress)); - if (updated_header.columns() == all_columns.size()) + if (source_part->getColumns().isSubsetOf(updated_header.getNamesAndTypesList())) { /// All columns are modified, proceed to write a new part from scratch. if (data.hasPrimaryKey() || data.hasSkipIndices()) diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts.reference b/dbms/tests/queries/0_stateless/01055_compact_parts.reference new file mode 100644 index 00000000000..85bd59af53d --- /dev/null +++ b/dbms/tests/queries/0_stateless/01055_compact_parts.reference @@ -0,0 +1,47 @@ +0 0 1167657 [0,0,0] ['a','b','c'] baz +1 1 2072334 [0,0,0] ['a','b','c'] bar +2 4 843568 [0,0,0] ['a','b','c'] baz +3 9 1748245 [0,0,0] ['a','b','c'] bar +4 16 519479 [0,0,0] ['a','b','c'] baz +5 25 1424156 [0,0,0] ['a','b','c'] bar +6 36 195390 [0,0,0] ['a','b','c'] baz +7 49 1100067 [0,0,0] ['a','b','c'] bar +8 64 2004744 [0,0,0] ['a','b','c'] baz +9 81 775978 [0,0,0] ['a','b','c'] bar +===================== +Compact +Compact 7 +Wide 3 +0 0 1167657 [1,2] ['',''] 0 +0 0 1167657 [0,0,0] ['a','b','c'] baz +0 0 1748245 [1,2] ['',''] 3 +1 1 2072334 [1,2] ['',''] 1 +1 1 2072334 [0,0,0] ['a','b','c'] bar +1 1 519479 [1,2] ['',''] 4 +2 4 843568 [0,0,0] ['a','b','c'] baz +2 4 843568 [1,2] ['',''] 2 +3 9 1748245 [0,0,0] ['a','b','c'] bar +4 16 519479 [0,0,0] ['a','b','c'] baz +===================== +0 0 1167657 [1,2] ['qwqw'] 0 +0 0 1167657 [0,0,0] ['qwqw'] baz +0 0 1748245 [1,2] ['qwqw'] 3 +1 1 2072334 [1,2] ['qwqw'] 1 +1 1 2072334 [0,0,0] ['qwqw'] bar +1 1 519479 [1,2] ['qwqw'] 4 +2 4 843568 [0,0,0] ['qwqw'] baz +2 4 843568 [1,2] ['qwqw'] 2 +3 9 1748245 [0,0,0] ['qwqw'] bar +4 16 519479 [0,0,0] ['qwqw'] baz +===================== +2 42 843568 [1,2] ['qwqw'] 2 +2 42 843568 [0,0,0] ['qwqw'] baz +3 42 1748245 [0,0,0] ['qwqw'] bar +4 42 519479 [0,0,0] ['qwqw'] baz +5 42 1424156 [0,0,0] ['qwqw'] bar +6 42 195390 [0,0,0] ['qwqw'] baz +7 42 1100067 [0,0,0] ['qwqw'] bar +8 42 2004744 [0,0,0] ['qwqw'] baz +9 42 775978 [0,0,0] ['qwqw'] bar +10 42 1680655 [0,0,0] ['qwqw'] baz +===================== diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts.sh b/dbms/tests/queries/0_stateless/01055_compact_parts.sh new file mode 100755 index 00000000000..1aaf82289ca --- /dev/null +++ b/dbms/tests/queries/0_stateless/01055_compact_parts.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +. $CURDIR/mergetree_mutations.lib + +# Testing basic functionality with compact parts +${CLICKHOUSE_CLIENT} --query="drop table if exists mt_compact;" + +${CLICKHOUSE_CLIENT} --query="create table mt_compact(a UInt64, b UInt64 DEFAULT a * a, s String, n Nested(x UInt32, y String), lc LowCardinality(String)) \ + engine = MergeTree \ + order by a partition by a % 10 \ + settings index_granularity = 8, \ + min_rows_for_wide_part = 10;" + +${CLICKHOUSE_CLIENT} --query="insert into mt_compact (a, s, n.y, lc) select number, toString((number * 2132214234 + 5434543) % 2133443), ['a', 'b', 'c'], number % 2 ? 'bar' : 'baz' from numbers(90);" + +${CLICKHOUSE_CLIENT} --query="select * from mt_compact order by a limit 10;" +${CLICKHOUSE_CLIENT} --query="select '=====================';" + +${CLICKHOUSE_CLIENT} --query="select distinct part_type from system.parts where database = currentDatabase() and table = 'mt_compact' and active;" + +${CLICKHOUSE_CLIENT} --query="insert into mt_compact (a, s, n.x, lc) select number % 3, toString((number * 2132214234 + 5434543) % 2133443), [1, 2], toString(number) from numbers(5);" + +${CLICKHOUSE_CLIENT} --query="optimize table mt_compact final;" + +${CLICKHOUSE_CLIENT} --query="select part_type, count() from system.parts where database = currentDatabase() and table = 'mt_compact' and active group by part_type; " +${CLICKHOUSE_CLIENT} --query="select * from mt_compact order by a, s limit 10;" +${CLICKHOUSE_CLIENT} --query="select '=====================';" + +${CLICKHOUSE_CLIENT} --query="alter table mt_compact drop column n.y;" +${CLICKHOUSE_CLIENT} --query="alter table mt_compact add column n.y Array(String) DEFAULT ['qwqw'] after n.x;" +${CLICKHOUSE_CLIENT} --query="select * from mt_compact order by a, s limit 10;" +${CLICKHOUSE_CLIENT} --query="select '=====================';" + +${CLICKHOUSE_CLIENT} --query="alter table mt_compact update b = 42 where 1;" + +sleep 0.5 +mutation_id=`${CLICKHOUSE_CLIENT} --query="SELECT max(mutation_id) FROM system.mutations WHERE table='mt_compact'"` +wait_for_mutation "mt_compact" "$mutation_id" + +${CLICKHOUSE_CLIENT} --query="select * from mt_compact where a > 1 order by a, s limit 10;" +${CLICKHOUSE_CLIENT} --query="select '=====================';" + +${CLICKHOUSE_CLIENT} --query="drop table if exists mt_compact;" From 2fdd211635d3d6d7800ce0cdd63d6b70d7ba26b8 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 21 Jan 2020 16:32:35 +0800 Subject: [PATCH 0126/2007] rename auc to arrayAUC --- .../src/Functions/array/auc.cpp => array_auc.cpp | 16 ++++++++-------- .../queries/0_stateless/01064_array_auc.sql | 2 +- .../query_language/functions/array_functions.md | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) rename dbms/src/Functions/array/auc.cpp => array_auc.cpp (84%) diff --git a/dbms/src/Functions/array/auc.cpp b/array_auc.cpp similarity index 84% rename from dbms/src/Functions/array/auc.cpp rename to array_auc.cpp index 7a2daf0fdd0..41739446e46 100644 --- a/dbms/src/Functions/array/auc.cpp +++ b/array_auc.cpp @@ -5,12 +5,12 @@ namespace DB { -struct NameAUC +struct NameArrayAUC { - static constexpr auto name = "auc"; + static constexpr auto name = "arrayAUC"; }; -class AUCImpl +class ArrayAUCImpl { public: using ResultType = Float64; @@ -26,7 +26,7 @@ public: WhichDataType which2(nested_type2); if (!which2.isUInt8()) { - throw Exception(std::string(NameAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception(std::string(NameArrayAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } return std::make_shared>(); } @@ -41,7 +41,7 @@ public: size_t label_len) { if (score_len != label_len) - throw Exception{"Unmatched length of arrays in " + std::string(NameAUC::name), ErrorCodes::LOGICAL_ERROR}; + throw Exception{"Unmatched length of arrays in " + std::string(NameArrayAUC::name), ErrorCodes::LOGICAL_ERROR}; if (score_len == 0) return {}; @@ -91,11 +91,11 @@ public: }; /// auc(array_score, array_label) - Calculate AUC with array of score and label -using FunctionAUC = FunctionArrayScalarProduct; +using FunctionArrayAUC = FunctionArrayScalarProduct; -void registerFunctionAUC(FunctionFactory & factory) +void registerFunctionArrayAUC(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); } diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql index 8886426f098..ca270937f63 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.sql +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -1 +1 @@ -select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 64f631a84e4..8c9b2a7c151 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -896,7 +896,7 @@ Result: │ [('a','d'),('b','e'),('c','f')] │ └────────────────────────────────────────────┘ ``` -## auc(arr_scores, arr_labels) +## arrayAUC(arr_scores, arr_labels) Returns AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc); From b9076ef247b7765e2839c170debe6e10f4f202b4 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 21 Jan 2020 16:59:09 +0800 Subject: [PATCH 0127/2007] mv file --- array_auc.cpp => dbms/src/Functions/array/array_auc.cpp | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename array_auc.cpp => dbms/src/Functions/array/array_auc.cpp (100%) diff --git a/array_auc.cpp b/dbms/src/Functions/array/array_auc.cpp similarity index 100% rename from array_auc.cpp rename to dbms/src/Functions/array/array_auc.cpp From 3d9f49c0baff29d6d43d332cfb42e8d6c44d5e01 Mon Sep 17 00:00:00 2001 From: liyang Date: Tue, 21 Jan 2020 17:03:35 +0800 Subject: [PATCH 0128/2007] commitagin --- dbms/src/Functions/array/registerFunctionsArray.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbms/src/Functions/array/registerFunctionsArray.cpp b/dbms/src/Functions/array/registerFunctionsArray.cpp index 96f19f547c0..cc360d80979 100644 --- a/dbms/src/Functions/array/registerFunctionsArray.cpp +++ b/dbms/src/Functions/array/registerFunctionsArray.cpp @@ -1,6 +1,5 @@ namespace DB { - class FunctionFactory; void registerFunctionArray(FunctionFactory & factory); @@ -33,7 +32,7 @@ void registerFunctionArrayDistinct(FunctionFactory & factory); void registerFunctionArrayFlatten(FunctionFactory & factory); void registerFunctionArrayWithConstant(FunctionFactory & factory); void registerFunctionArrayZip(FunctionFactory & factory); -void registerFunctionAUC(FunctionFactory &); +void registerFunctionArrayAUC(FunctionFactory &); void registerFunctionsArray(FunctionFactory & factory) { @@ -67,8 +66,7 @@ void registerFunctionsArray(FunctionFactory & factory) registerFunctionArrayFlatten(factory); registerFunctionArrayWithConstant(factory); registerFunctionArrayZip(factory); - registerFunctionAUC(factory); + registerFunctionArrayAUC(factory); } } - From 6a29525fa34b16a4bc1b8c230d6fc30bdc063b21 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 21 Jan 2020 14:56:01 +0300 Subject: [PATCH 0129/2007] add some comments --- dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp | 3 ++- dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 1 + dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp | 1 - .../src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp | 4 +++- dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp | 1 - 5 files changed, 6 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp index c369430b954..bc0bf5cc161 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp @@ -131,7 +131,8 @@ static void fillIndexGranularityImpl( { size_t rows_left_in_block = rows_in_block - current_row; - /// FIXME need comment + /// Try to extend last granule if it's needed and block is large enough + /// or it shouldn't be first in granule (index_offset != 0). if (need_finish_last_granule && rows_left_in_block < index_granularity_for_block && (rows_in_block >= index_granularity_for_block || index_offset != 0)) { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index ed51affb1ae..62b546ff830 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -1021,6 +1021,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor MergeStageProgress stage_progress(1.0); in->setProgressCallback(MergeProgressCallback(merge_entry, watch_prev_elapsed, stage_progress)); + /// All columns from part are changed and may be some more that were missing before in part if (source_part->getColumns().isSubsetOf(updated_header.getNamesAndTypesList())) { /// All columns are modified, proceed to write a new part from scratch. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 1eff2a33cf0..931a365371c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -43,7 +43,6 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( const ValueSizeMap & avg_value_size_hints, const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { - /// FIXME maybe avoid shared_from_this return std::make_unique( shared_from_this(), columns_to_read, uncompressed_cache, mark_cache, mark_ranges, reader_settings, diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index daad489be58..522359ad635 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -77,7 +77,8 @@ void MergeTreeDataPartWriterCompact::write( if (rows_in_buffer < last_mark_rows) { - /// FIXME need comment + /// If it's not enough rows for granule, accumulate blocks + /// and save how much rows we already have. next_index_offset = last_mark_rows - rows_in_buffer; return; } @@ -114,6 +115,7 @@ void MergeTreeDataPartWriterCompact::writeBlock(const Block & block) size_t rows_written = total_rows - current_row; current_row += rows_to_write; + /// Correct last mark as it should contain exact amount of rows. if (current_row >= total_rows && rows_written != rows_to_write) { rows_to_write = rows_written; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 1d4f4df11de..00318e59fd7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -280,7 +280,6 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch { if (!serialization_states.empty()) { - /// FIXME maybe we need skip_offsets=false in some cases serialize_settings.getter = createStreamGetter(it->name, written_offset_columns ? *written_offset_columns : offset_columns); it->type->serializeBinaryBulkStateSuffix(serialize_settings, serialization_states[it->name]); } From 13709875eaf1bc89ebf74c990f27d1d626313781 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 21 Jan 2020 14:56:46 +0300 Subject: [PATCH 0130/2007] add some tests --- .../01055_compact_parts_1.reference | 0 .../0_stateless/01055_compact_parts_1.sql | 10 +++++++++ .../01055_compact_parts_granularity.reference | 2 ++ .../01055_compact_parts_granularity.sql | 21 +++++++++++++++++++ 4 files changed, 33 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01055_compact_parts_1.reference create mode 100644 dbms/tests/queries/0_stateless/01055_compact_parts_1.sql create mode 100644 dbms/tests/queries/0_stateless/01055_compact_parts_granularity.reference create mode 100644 dbms/tests/queries/0_stateless/01055_compact_parts_granularity.sql diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts_1.reference b/dbms/tests/queries/0_stateless/01055_compact_parts_1.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts_1.sql b/dbms/tests/queries/0_stateless/01055_compact_parts_1.sql new file mode 100644 index 00000000000..a3725ea16cf --- /dev/null +++ b/dbms/tests/queries/0_stateless/01055_compact_parts_1.sql @@ -0,0 +1,10 @@ +drop table if exists mt_compact; + +create table mt_compact (a Int, s String) engine = MergeTree order by a partition by a +settings index_granularity_bytes = 0; +alter table mt_compact modify setting min_rows_for_wide_part = 1000; -- { serverError 48 } + +create table mt_compact_2 (a Int, s String) engine = MergeTree order by a partition by a +settings min_rows_for_wide_part = 1000; +insert into mt_compact_2 values (1, 'a'); +alter table mt_compact attach partition 1 from mt_compact_2; -- { serverError 36 } diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts_granularity.reference b/dbms/tests/queries/0_stateless/01055_compact_parts_granularity.reference new file mode 100644 index 00000000000..543593fb9b0 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01055_compact_parts_granularity.reference @@ -0,0 +1,2 @@ +100 +1 9 diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts_granularity.sql b/dbms/tests/queries/0_stateless/01055_compact_parts_granularity.sql new file mode 100644 index 00000000000..8828ffc2da8 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01055_compact_parts_granularity.sql @@ -0,0 +1,21 @@ +drop table if exists mt_compact; + +-- Checks that granularity correctly computed from small parts. + +create table mt_compact(a Int, s String) engine = MergeTree order by a +settings min_rows_for_wide_part = 1000, +index_granularity = 14; + +system stop merges mt_compact; +set max_block_size = 1; +set min_insert_block_size_rows=1; +insert into mt_compact select number, 'aaa' from numbers(100); + +select count() from system.parts where table = 'mt_compact' and database = currentDatabase() and active; + +system start merges mt_compact; +optimize table mt_compact final; + +select count(), sum(marks) from system.parts where table = 'mt_compact' and database = currentDatabase() and active; + +drop table mt_compact; From 8183a791005d597c7fcc90a540c60895f2ee7759 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Wed, 22 Jan 2020 13:37:16 +0300 Subject: [PATCH 0131/2007] make all parts wide by default --- dbms/src/Storages/MergeTree/MergeTreeSettings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeSettings.h b/dbms/src/Storages/MergeTree/MergeTreeSettings.h index 9b4373eb666..6fbfae61c43 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeSettings.h @@ -30,7 +30,7 @@ struct MergeTreeSettings : public SettingsCollection \ /** Data storing format settigns. */ \ M(SettingUInt64, min_bytes_for_wide_part, 0, "Minimal uncompressed size in bytes to create part in wide format instead of compact", 0) \ - M(SettingUInt64, min_rows_for_wide_part, 10000000000, "Minimal number of rows to create part in wide format instead of compact", 0) \ + M(SettingUInt64, min_rows_for_wide_part, 0, "Minimal number of rows to create part in wide format instead of compact", 0) \ \ /** Merge settings. */ \ M(SettingUInt64, merge_max_block_size, DEFAULT_MERGE_BLOCK_SIZE, "How many rows in blocks should be formed for merge operations.", 0) \ From 771e429d35d267544baf4d6b2967934d3438bf8e Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 22 Jan 2020 17:10:35 +0300 Subject: [PATCH 0132/2007] fix tests --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 10 +-------- .../MergeTree/MergedBlockOutputStream.cpp | 4 +--- .../0_stateless/01055_compact_parts.reference | 22 +++++++++---------- .../0_stateless/01055_compact_parts.sh | 4 ++-- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 71614043a31..39293cdcba1 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -520,7 +520,7 @@ void IMergeTreeDataPart::loadRowsCount() { rows_count = 0; } - else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + else if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || part_type == Type::COMPACT) { if (!Poco::File(path).exists()) throw Exception("No count.txt in part " + name, ErrorCodes::NO_FILE_IN_DATA_PART); @@ -531,14 +531,6 @@ void IMergeTreeDataPart::loadRowsCount() } else { - if (Poco::File(path).exists()) - { - ReadBufferFromFile file = openForReading(path); - readIntText(rows_count, file); - assertEOF(file); - return; - } - for (const NameAndTypePair & column : columns) { ColumnPtr column_col = column.type->createColumn(); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 4081f572d2e..86c396b4955 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -98,7 +98,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (!total_column_list) total_column_list = &columns_list; - if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part)) { new_part->partition.store(storage, part_path, checksums); if (new_part->minmax_idx.initialized) @@ -106,9 +106,7 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( else if (rows_count) throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); - } - { WriteBufferFromFile count_out(part_path + "count.txt", 4096); HashingWriteBuffer count_out_hashing(count_out); writeIntText(rows_count, count_out_hashing); diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts.reference b/dbms/tests/queries/0_stateless/01055_compact_parts.reference index 85bd59af53d..c839ca13281 100644 --- a/dbms/tests/queries/0_stateless/01055_compact_parts.reference +++ b/dbms/tests/queries/0_stateless/01055_compact_parts.reference @@ -12,29 +12,29 @@ Compact Compact 7 Wide 3 -0 0 1167657 [1,2] ['',''] 0 0 0 1167657 [0,0,0] ['a','b','c'] baz -0 0 1748245 [1,2] ['',''] 3 -1 1 2072334 [1,2] ['',''] 1 +0 0 645645 [1,2] ['',''] 0 +0 0 804292 [1,2] ['',''] 3 +1 1 1409675 [1,2] ['',''] 1 +1 1 1568322 [1,2] ['',''] 4 1 1 2072334 [0,0,0] ['a','b','c'] bar -1 1 519479 [1,2] ['',''] 4 +2 4 40262 [1,2] ['',''] 2 2 4 843568 [0,0,0] ['a','b','c'] baz -2 4 843568 [1,2] ['',''] 2 3 9 1748245 [0,0,0] ['a','b','c'] bar 4 16 519479 [0,0,0] ['a','b','c'] baz ===================== -0 0 1167657 [1,2] ['qwqw'] 0 0 0 1167657 [0,0,0] ['qwqw'] baz -0 0 1748245 [1,2] ['qwqw'] 3 -1 1 2072334 [1,2] ['qwqw'] 1 +0 0 645645 [1,2] ['qwqw'] 0 +0 0 804292 [1,2] ['qwqw'] 3 +1 1 1409675 [1,2] ['qwqw'] 1 +1 1 1568322 [1,2] ['qwqw'] 4 1 1 2072334 [0,0,0] ['qwqw'] bar -1 1 519479 [1,2] ['qwqw'] 4 +2 4 40262 [1,2] ['qwqw'] 2 2 4 843568 [0,0,0] ['qwqw'] baz -2 4 843568 [1,2] ['qwqw'] 2 3 9 1748245 [0,0,0] ['qwqw'] bar 4 16 519479 [0,0,0] ['qwqw'] baz ===================== -2 42 843568 [1,2] ['qwqw'] 2 +2 42 40262 [1,2] ['qwqw'] 2 2 42 843568 [0,0,0] ['qwqw'] baz 3 42 1748245 [0,0,0] ['qwqw'] bar 4 42 519479 [0,0,0] ['qwqw'] baz diff --git a/dbms/tests/queries/0_stateless/01055_compact_parts.sh b/dbms/tests/queries/0_stateless/01055_compact_parts.sh index 1aaf82289ca..dbe138cd9ce 100755 --- a/dbms/tests/queries/0_stateless/01055_compact_parts.sh +++ b/dbms/tests/queries/0_stateless/01055_compact_parts.sh @@ -21,11 +21,11 @@ ${CLICKHOUSE_CLIENT} --query="select '=====================';" ${CLICKHOUSE_CLIENT} --query="select distinct part_type from system.parts where database = currentDatabase() and table = 'mt_compact' and active;" -${CLICKHOUSE_CLIENT} --query="insert into mt_compact (a, s, n.x, lc) select number % 3, toString((number * 2132214234 + 5434543) % 2133443), [1, 2], toString(number) from numbers(5);" +${CLICKHOUSE_CLIENT} --query="insert into mt_compact (a, s, n.x, lc) select number % 3, toString((number * 75434535 + 645645) % 2133443), [1, 2], toString(number) from numbers(5);" ${CLICKHOUSE_CLIENT} --query="optimize table mt_compact final;" -${CLICKHOUSE_CLIENT} --query="select part_type, count() from system.parts where database = currentDatabase() and table = 'mt_compact' and active group by part_type; " +${CLICKHOUSE_CLIENT} --query="select part_type, count() from system.parts where database = currentDatabase() and table = 'mt_compact' and active group by part_type order by part_type; " ${CLICKHOUSE_CLIENT} --query="select * from mt_compact order by a, s limit 10;" ${CLICKHOUSE_CLIENT} --query="select '=====================';" From 14f0b9e137fccc57c83da8f496760cd7c2eac427 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 22 Jan 2020 16:24:20 +0300 Subject: [PATCH 0133/2007] add query 'ALTER ... MATERIALIZE TTL' --- .../Interpreters/InterpreterAlterQuery.cpp | 7 ++++ .../src/Interpreters/MutationsInterpreter.cpp | 18 +++++++++++ dbms/src/Parsers/ASTAlterQuery.cpp | 5 +++ dbms/src/Parsers/ASTAlterQuery.h | 1 + dbms/src/Parsers/ParserAlterQuery.cpp | 5 +++ dbms/src/Storages/IStorage.h | 9 ++++++ dbms/src/Storages/MergeTree/MergeTreeData.cpp | 32 +++++++++++++++++++ dbms/src/Storages/MergeTree/MergeTreeData.h | 3 ++ .../MergeTree/MergeTreeDataMergerMutator.cpp | 32 +++++++++++++++++++ .../MergeTree/StorageFromMergeTreeDataPart.h | 4 +++ dbms/src/Storages/MutationCommands.cpp | 7 ++++ dbms/src/Storages/MutationCommands.h | 3 +- dbms/src/Storages/StorageMergeTree.cpp | 2 ++ .../Storages/StorageReplicatedMergeTree.cpp | 1 + .../01070_materialize_ttl.reference | 3 ++ .../0_stateless/01070_materialize_ttl.sh | 26 +++++++++++++++ 16 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/01070_materialize_ttl.reference create mode 100755 dbms/tests/queries/0_stateless/01070_materialize_ttl.sh diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index e821b56de2e..64de3c191d1 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int ILLEGAL_COLUMN; extern const int SUPPORT_IS_DISABLED; + extern const int INCORRECT_QUERY; } @@ -65,7 +66,13 @@ BlockIO InterpreterAlterQuery::execute() partition_commands.emplace_back(std::move(*partition_command)); } else if (auto mut_command = MutationCommand::parse(command_ast)) + { + if (mut_command->type == MutationCommand::MATERIALIZE_TTL && !table->hasAnyTTL()) + throw Exception("Cannot MATERIALIZE TTL as there is no TTL set for table " + + table->getStorageID().getNameForLogs(), ErrorCodes::INCORRECT_QUERY); + mutation_commands.emplace_back(std::move(*mut_command)); + } else if (auto live_view_command = LiveViewCommand::parse(command_ast)) live_view_commands.emplace_back(std::move(*live_view_command)); else diff --git a/dbms/src/Interpreters/MutationsInterpreter.cpp b/dbms/src/Interpreters/MutationsInterpreter.cpp index 8ff10e92dee..88c2d4c9752 100644 --- a/dbms/src/Interpreters/MutationsInterpreter.cpp +++ b/dbms/src/Interpreters/MutationsInterpreter.cpp @@ -381,6 +381,24 @@ ASTPtr MutationsInterpreter::prepare(bool dry_run) const auto required_columns = syntax_result->requiredSourceColumns(); affected_indices_columns.insert(std::cbegin(required_columns), std::cend(required_columns)); } + else if (command.type == MutationCommand::MATERIALIZE_TTL) + { + if (stages.empty() || !stages.back().column_to_updated.empty()) + stages.emplace_back(context); + if (stages.size() == 1) /// First stage only supports filtering and can't update columns. + stages.emplace_back(context); + + auto required_columns_for_ttl = storage->getColumnsRequiredForTTL(); + for (const auto & column_name : required_columns_for_ttl) + stages.back().column_to_updated.emplace(column_name, std::make_shared(column_name)); + + auto updated_columns_by_ttl = storage->getColumnsUpdatedByTTL(); + for (const auto & column_name : updated_columns_by_ttl) + { + stages.back().column_to_updated.emplace(column_name, std::make_shared(column_name)); + affected_indices_columns.insert(column_name); + } + } else throw Exception("Unknown mutation command type: " + DB::toString(command.type), ErrorCodes::UNKNOWN_MUTATION_COMMAND); } diff --git a/dbms/src/Parsers/ASTAlterQuery.cpp b/dbms/src/Parsers/ASTAlterQuery.cpp index fed9c92d630..842758626b4 100644 --- a/dbms/src/Parsers/ASTAlterQuery.cpp +++ b/dbms/src/Parsers/ASTAlterQuery.cpp @@ -261,6 +261,11 @@ void ASTAlterCommand::formatImpl( settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY TTL " << (settings.hilite ? hilite_none : ""); ttl->formatImpl(settings, state, frame); } + else if (type == ASTAlterCommand::MATERIALIZE_TTL) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MATERIALIZE TTL" + << (settings.hilite ? hilite_none : ""); + } else if (type == ASTAlterCommand::MODIFY_SETTING) { settings.ostr << (settings.hilite ? hilite_keyword : "") << indent_str << "MODIFY SETTING " << (settings.hilite ? hilite_none : ""); diff --git a/dbms/src/Parsers/ASTAlterQuery.h b/dbms/src/Parsers/ASTAlterQuery.h index 25688e3890f..44aa57dbc95 100644 --- a/dbms/src/Parsers/ASTAlterQuery.h +++ b/dbms/src/Parsers/ASTAlterQuery.h @@ -31,6 +31,7 @@ public: COMMENT_COLUMN, MODIFY_ORDER_BY, MODIFY_TTL, + MATERIALIZE_TTL, MODIFY_SETTING, ADD_INDEX, diff --git a/dbms/src/Parsers/ParserAlterQuery.cpp b/dbms/src/Parsers/ParserAlterQuery.cpp index 61bf4d2075d..afb44702372 100644 --- a/dbms/src/Parsers/ParserAlterQuery.cpp +++ b/dbms/src/Parsers/ParserAlterQuery.cpp @@ -29,6 +29,7 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected ParserKeyword s_comment_column("COMMENT COLUMN"); ParserKeyword s_modify_order_by("MODIFY ORDER BY"); ParserKeyword s_modify_ttl("MODIFY TTL"); + ParserKeyword s_materialize_ttl("MATERIALIZE TTL"); ParserKeyword s_modify_setting("MODIFY SETTING"); ParserKeyword s_add_index("ADD INDEX"); @@ -455,6 +456,10 @@ bool ParserAlterCommand::parseImpl(Pos & pos, ASTPtr & node, Expected & expected return false; command->type = ASTAlterCommand::MODIFY_TTL; } + else if (s_materialize_ttl.ignore(pos, expected)) + { + command->type = ASTAlterCommand::MATERIALIZE_TTL; + } else if (s_modify_setting.ignore(pos, expected)) { if (!parser_settings.parse(pos, command->settings_changes, expected)) diff --git a/dbms/src/Storages/IStorage.h b/dbms/src/Storages/IStorage.h index dcb805b141e..e55f759b1a6 100644 --- a/dbms/src/Storages/IStorage.h +++ b/dbms/src/Storages/IStorage.h @@ -116,6 +116,9 @@ public: virtual bool hasEvenlyDistributedRead() const { return false; } + /// Returns true if there is set table TTL, any column TTL or any move TTL + virtual bool hasAnyTTL() const { return false; } + /// Optional size information of each physical column. /// Currently it's only used by the MergeTree family for query optimizations. using ColumnSizeByName = std::unordered_map; @@ -440,6 +443,12 @@ public: /// Returns names of primary key + secondary sorting columns virtual Names getSortingKeyColumns() const { return {}; } + /// Returns columns required for calculation of TTL expression. + virtual Names getColumnsRequiredForTTL() const { return {}; } + + /// Returns columns that could be updated by applying TTL rules + virtual Names getColumnsUpdatedByTTL() const { return {}; } + /// Returns storage policy if storage supports it virtual StoragePolicyPtr getStoragePolicy() const { return {}; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 146440b11c3..bb4ea0e1467 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -3905,4 +3905,36 @@ bool MergeTreeData::moveParts(CurrentlyMovingPartsTagger && moving_tagger) return true; } +Names MergeTreeData::getColumnsRequiredForTTL() const +{ + NameSet res; + auto add_columns_to_set = [&](const auto & expression) + { + auto columns_vec = expression->getRequiredColumns(); + res.insert(columns_vec.begin(), columns_vec.end()); + }; + + if (hasTableTTL()) + add_columns_to_set(ttl_table_entry.expression); + + for (const auto & [_, entry] : column_ttl_entries_by_name) + add_columns_to_set(entry.expression); + + for (const auto & entry : move_ttl_entries) + add_columns_to_set(entry.expression); + + return Names(res.begin(), res.end()); +} + +Names MergeTreeData::getColumnsUpdatedByTTL() const +{ + if (hasTableTTL()) + return getColumns().getAllPhysical().getNames(); + + Names res; + for (const auto & [name, _] : column_ttl_entries_by_name) + res.push_back(name); + return res; +} + } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index ab5644749ee..3578cdb6577 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -354,6 +354,8 @@ public: Names getColumnsRequiredForSampling() const override { return columns_required_for_sampling; } Names getColumnsRequiredForFinal() const override { return sorting_key_expr->getRequiredColumns(); } Names getSortingKeyColumns() const override { return sorting_key_columns; } + Names getColumnsRequiredForTTL() const override; + Names getColumnsUpdatedByTTL() const override; StoragePolicyPtr getStoragePolicy() const override { return storage_policy; } @@ -580,6 +582,7 @@ public: bool hasAnyColumnTTL() const { return !column_ttl_entries_by_name.empty(); } bool hasAnyMoveTTL() const { return !move_ttl_entries.empty(); } bool hasRowsTTL() const { return !rows_ttl_entry.isEmpty(); } + bool hasAnyTTL() const override { return hasRowsTTL() || hasAnyMoveTTL() || hasAnyColumnTTL(); } /// Check that the part is not broken and calculate the checksums for it if they are not present. MutableDataPartPtr loadPartAndFixMetadata(const DiskPtr & disk, const String & relative_path); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 05db73ce215..ea983419831 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -936,6 +936,16 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor throw Exception("Trying to mutate " + toString(future_part.parts.size()) + " parts, not one. " "This is a bug.", ErrorCodes::LOGICAL_ERROR); + bool need_remove_expired_values = false; + for (const auto & command : commands) + { + if (command.type == MutationCommand::MATERIALIZE_TTL) + { + need_remove_expired_values = true; + break; + } + } + CurrentMetrics::Increment num_mutations{CurrentMetrics::PartMutation}; const auto & source_part = future_part.parts[0]; auto storage_from_source_part = StorageFromMergeTreeDataPart::create(source_part); @@ -966,6 +976,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared( data, space_reservation->getDisk(), future_part.name, future_part.part_info); + new_data_part->relative_path = "tmp_mut_" + future_part.name; new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; @@ -1004,6 +1015,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor in = std::make_shared( std::make_shared(in, data.primary_key_and_skip_indices_expr)); + if (need_remove_expired_values) + in = std::make_shared(in, data, new_data_part, time(nullptr), true); + MergeTreeDataPart::MinMaxIndex minmax_idx; MergedBlockOutputStream out(data, new_part_tmp_path, all_columns, compression_codec); @@ -1085,12 +1099,16 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor IDataType::SubstreamPath stream_path; entry.type->enumerateStreams(callback, stream_path); } + for (const auto & index : indices_to_recalc) { files_to_skip.insert(index->getFileName() + ".idx"); files_to_skip.insert(index->getFileName() + mrk_extension); } + if (!need_remove_expired_values) + files_to_skip.insert("ttl.txt"); + Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator dir_it(source_part->getFullPath()); dir_it != dir_end; ++dir_it) { @@ -1105,6 +1123,9 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor merge_entry->columns_written = all_columns.size() - updated_header.columns(); + if (need_remove_expired_values) + in = std::make_shared(in, data, new_data_part, time(nullptr), true); + IMergedBlockOutputStream::WrittenOffsetColumns unused_written_offsets; MergedColumnOnlyOutputStream out( data, @@ -1136,6 +1157,17 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor new_data_part->checksums = source_part->checksums; new_data_part->checksums.add(std::move(changed_checksums)); + + if (!new_data_part->ttl_infos.empty()) + { + /// Write a file with ttl infos in json format. + WriteBufferFromFile out_ttl(new_part_tmp_path + "ttl.txt", 4096); + HashingWriteBuffer out_hashing(out_ttl); + new_data_part->ttl_infos.write(out_hashing); + new_data_part->checksums.files["ttl.txt"].file_size = out_hashing.count(); + new_data_part->checksums.files["ttl.txt"].file_hash = out_hashing.getHash(); + } + { /// Write file with checksums. WriteBufferFromFile out_checksums(new_part_tmp_path + "checksums.txt", 4096); diff --git a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 1326c61b1f6..e2acd4dd64e 100644 --- a/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -47,6 +47,10 @@ public: return part->storage.mayBenefitFromIndexForIn(left_in_operand, query_context); } + Names getColumnsRequiredForTTL() const override { return part->storage.getColumnsRequiredForTTL(); } + Names getColumnsUpdatedByTTL() const override { return part->storage.getColumnsUpdatedByTTL(); } + bool hasAnyTTL() const override { return part->storage.hasAnyTTL(); } + protected: StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(getIDFromPart(part_), part_->storage.getVirtuals()) diff --git a/dbms/src/Storages/MutationCommands.cpp b/dbms/src/Storages/MutationCommands.cpp index f8bc781f166..58f745e0a77 100644 --- a/dbms/src/Storages/MutationCommands.cpp +++ b/dbms/src/Storages/MutationCommands.cpp @@ -55,6 +55,13 @@ std::optional MutationCommand::parse(ASTAlterCommand * command) res.index_name = command->index->as().name; return res; } + else if (command->type == ASTAlterCommand::MATERIALIZE_TTL) + { + MutationCommand res; + res.ast = command->ptr(); + res.type = MATERIALIZE_TTL; + return res; + } else return {}; } diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index 96ebd30f254..d9eec1f19e0 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -25,7 +25,8 @@ struct MutationCommand EMPTY, /// Not used. DELETE, UPDATE, - MATERIALIZE_INDEX + MATERIALIZE_INDEX, + MATERIALIZE_TTL }; Type type = EMPTY; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d8b25627a7e..c680e7836d6 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -778,6 +778,8 @@ bool StorageMergeTree::tryMutatePart() tagger->reserved_space, table_lock_holder); renameTempPartAndReplace(new_part); + removeEmptyColumnsFromPart(new_part); + tagger->is_successful = true; write_part_log({}); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index db113624f68..3866049e1c1 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1238,6 +1238,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM { new_part = merger_mutator.mutatePartToTemporaryPart(future_mutated_part, commands, *merge_entry, global_context, reserved_space, table_lock); renameTempPartAndReplace(new_part, nullptr, &transaction); + removeEmptyColumnsFromPart(new_part); try { diff --git a/dbms/tests/queries/0_stateless/01070_materialize_ttl.reference b/dbms/tests/queries/0_stateless/01070_materialize_ttl.reference new file mode 100644 index 00000000000..90986a38644 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_materialize_ttl.reference @@ -0,0 +1,3 @@ +Cannot MATERIALIZE TTL +2100-10-10 3 +2100-10-10 4 diff --git a/dbms/tests/queries/0_stateless/01070_materialize_ttl.sh b/dbms/tests/queries/0_stateless/01070_materialize_ttl.sh new file mode 100755 index 00000000000..053464cae6c --- /dev/null +++ b/dbms/tests/queries/0_stateless/01070_materialize_ttl.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +. $CURDIR/mergetree_mutations.lib + +${CLICKHOUSE_CLIENT} -q "drop table if exists ttl;" + +${CLICKHOUSE_CLIENT} -q "create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDayOfMonth(d);" +${CLICKHOUSE_CLIENT} -q "insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1);" +${CLICKHOUSE_CLIENT} -q "insert into ttl values (toDateTime('2000-10-10 00:00:00'), 2);" +${CLICKHOUSE_CLIENT} -q "insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3);" +${CLICKHOUSE_CLIENT} -q "insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4);" + +${CLICKHOUSE_CLIENT} -q "alter table ttl materialize ttl" --server_logs_file=/dev/null 2>&1 | grep -o "Cannot MATERIALIZE TTL" + +${CLICKHOUSE_CLIENT} -q "alter table ttl modify ttl d + interval 1 day" + +${CLICKHOUSE_CLIENT} -q "alter table ttl materialize ttl" + +wait_for_mutation "ttl" "mutation_5.txt" + +${CLICKHOUSE_CLIENT} -q "select * from ttl order by a" + +${CLICKHOUSE_CLIENT} -q "drop table if exists ttl" From a5d9975e377c57e58545dc19451a272b511d7f85 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 27 Jan 2020 14:53:50 +0300 Subject: [PATCH 0134/2007] better checking if table has TTL --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 6 +++--- dbms/src/Storages/StorageMergeTree.cpp | 2 +- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index bb4ea0e1467..b88fe8c3615 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -3914,8 +3914,8 @@ Names MergeTreeData::getColumnsRequiredForTTL() const res.insert(columns_vec.begin(), columns_vec.end()); }; - if (hasTableTTL()) - add_columns_to_set(ttl_table_entry.expression); + if (hasRowsTTL()) + add_columns_to_set(rows_ttl_entry.expression); for (const auto & [_, entry] : column_ttl_entries_by_name) add_columns_to_set(entry.expression); @@ -3928,7 +3928,7 @@ Names MergeTreeData::getColumnsRequiredForTTL() const Names MergeTreeData::getColumnsUpdatedByTTL() const { - if (hasTableTTL()) + if (hasRowsTTL()) return getColumns().getAllPhysical().getNames(); Names res; diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index c680e7836d6..5783cbcfe7e 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -652,7 +652,7 @@ bool StorageMergeTree::merge( { /// Force filter by TTL in 'OPTIMIZE ... FINAL' query to remove expired values from old parts /// without TTL infos or with outdated TTL infos, e.g. after 'ALTER ... MODIFY TTL' query. - bool force_ttl = (final && (hasRowsTTL() || hasAnyColumnTTL())); + bool force_ttl = (final && hasAnyTTL()); new_part = merger_mutator.mergePartsToTemporaryPart( future_part, *merge_entry, table_lock_holder, time(nullptr), diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 3866049e1c1..fd4d00b6ec9 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3133,7 +3133,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p return false; }; - bool force_ttl = (final && (hasRowsTTL() || hasAnyColumnTTL())); + bool force_ttl = (final && hasAnyTTL()); const auto storage_settings_ptr = getSettings(); if (!partition && final) From 974e9a6b7ac5f57634688de1f18678ee761bf44b Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 28 Jan 2020 20:15:22 +0300 Subject: [PATCH 0135/2007] Broke things --- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 15 ++++--- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 17 ++++---- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 24 +++++++---- .../MergeTree/ReplicatedMergeTreeQueue.h | 4 ++ dbms/src/Storages/MutationCommands.h | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 41 ++++++++----------- 6 files changed, 57 insertions(+), 46 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index f314bacdc53..ce8d873b972 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -65,10 +65,12 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const << new_part_name; break; - case FINISH_ALTER: /// Just make local /metadata and /columns consistent with global + case ALTER_METADATA: /// Just make local /metadata and /columns consistent with global out << "alter\n"; - out << required_mutation_znode << "\n"; - out << "finish\n"; + out << "sync_mode\n"; + out << alter_sync_mode << "\n"; + out << "mutatation_commands\n"; + out << mutation_commands << '\n'; break; default: @@ -160,8 +162,11 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) } else if (type_str == "alter") { - type = FINISH_ALTER; - in >> required_mutation_znode >> "\nfinish\n"; + type = ALTER_METADATA; + in >> "sync_mode\n"; + in >> alter_sync_mode; + in >> "mutatation_commands\n"; + in >> mutation_commands; } //std::cerr << "Read backn\n"; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index f7744d5a110..0e836ec7c27 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -37,7 +37,7 @@ struct ReplicatedMergeTreeLogEntryData CLEAR_INDEX, /// Drop specific index from specified partition. REPLACE_RANGE, /// Drop certain range of partitions and replace them by new ones MUTATE_PART, /// Apply one or several mutations to the part. - FINISH_ALTER, /// Apply one or several alter modifications to part + ALTER_METADATA, /// Apply alter modification according to global /metadata and /columns paths }; static String typeToString(Type type) @@ -51,7 +51,7 @@ struct ReplicatedMergeTreeLogEntryData case ReplicatedMergeTreeLogEntryData::CLEAR_INDEX: return "CLEAR_INDEX"; case ReplicatedMergeTreeLogEntryData::REPLACE_RANGE: return "REPLACE_RANGE"; case ReplicatedMergeTreeLogEntryData::MUTATE_PART: return "MUTATE_PART"; - case ReplicatedMergeTreeLogEntryData::FINISH_ALTER: return "FINISH_ALTER"; + case ReplicatedMergeTreeLogEntryData::ALTER_METADATA: return "ALTER_METADATA"; default: throw Exception("Unknown log entry type: " + DB::toString(type), ErrorCodes::LOGICAL_ERROR); } @@ -66,6 +66,7 @@ struct ReplicatedMergeTreeLogEntryData void readText(ReadBuffer & in); String toString() const; + /// log-xxx String znode_name; Type type = EMPTY; @@ -83,15 +84,12 @@ struct ReplicatedMergeTreeLogEntryData String index_name; /// Force filter by TTL in 'OPTIMIZE ... FINAL' query to remove expired values from old parts - /// without TTL infos or with outdated TTL infos, e.g. after 'ALTER ... MODIFY TTL' query. + /// without TTL infos or with outdated TTL infos, e.g. after 'ALTER ... MODIFY TTL' query. bool force_ttl = false; /// For DROP_RANGE, true means that the parts need not be deleted, but moved to the `detached` directory. bool detach = false; - /// For ALTER TODO(alesap) - String required_mutation_znode; - /// REPLACE PARTITION FROM command struct ReplaceRangeEntry { @@ -110,12 +108,17 @@ struct ReplicatedMergeTreeLogEntryData std::shared_ptr replace_range_entry; + /// Should alter be processed sychronously, or asynchronously. + size_t alter_sync_mode; + /// Mutation commands for alter if any. + String mutation_commands; + /// Returns a set of parts that will appear after executing the entry + parts to block /// selection of merges. These parts are added to queue.virtual_parts. Strings getVirtualPartNames() const { /// Doesn't produce any part - if (type == FINISH_ALTER) + if (type == ALTER_METADATA) return {}; /// DROP_RANGE does not add a real part, but we must disable merges in that range diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index eef2710ea20..f9622009325 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -149,6 +149,10 @@ void ReplicatedMergeTreeQueue::insertUnlocked( min_unprocessed_insert_time_changed = min_unprocessed_insert_time; } } + if (entry->type == LogEntry::ALTER_METADATA) + { + alter_znodes_in_queue.push_back(entry->znode_name); + } } @@ -219,6 +223,16 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( current_parts.remove(drop_range_part_name); virtual_parts.remove(drop_range_part_name); } + + if (entry->type == LogEntry::ALTER_METADATA) + { + if (alter_znodes_in_queue.front() != entry->znode_name) + { + /// TODO(alesap) Better + throw Exception("Processed incorrect alter.", ErrorCodes::LOGICAL_ERROR); + } + alter_znodes_in_queue.pop_front(); + } } else { @@ -1009,15 +1023,9 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( } } - if (entry.type == LogEntry::FINISH_ALTER) + if (entry.type == LogEntry::ALTER_METADATA) { - //std::cerr << "Entry finish alter\n"; - if (mutations_by_znode.count(entry.required_mutation_znode) && !mutations_by_znode.at(entry.required_mutation_znode).is_done) { - String reason = "Not altering storage because mutation " + entry.required_mutation_znode + " is not ready yet (mutation is beeing processed)."; - LOG_TRACE(log, reason); - out_postpone_reason = reason; - return false; - } + return entry.znode_name == alter_znodes_in_queue.front(); } return true; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index cad23df6f46..299b4ed1c48 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -77,6 +77,9 @@ private: time_t last_queue_update = 0; + /// This vector is used for sequential execution of alters + std::deque alter_znodes_in_queue; + /// parts that will appear as a result of actions performed right now by background threads (these actions are not in the queue). /// Used to block other actions on parts in the range covered by future_parts. using FuturePartsSet = std::map; @@ -161,6 +164,7 @@ private: /// Put a set of (already existing) parts in virtual_parts. void addVirtualParts(const MergeTreeData::DataParts & parts); + /// Insert new entry from log into queue void insertUnlocked( const LogEntryPtr & entry, std::optional & min_unprocessed_insert_time_changed, std::lock_guard & state_lock); diff --git a/dbms/src/Storages/MutationCommands.h b/dbms/src/Storages/MutationCommands.h index 265c5ae1a6f..7f7f31b6315 100644 --- a/dbms/src/Storages/MutationCommands.h +++ b/dbms/src/Storages/MutationCommands.h @@ -46,7 +46,7 @@ struct MutationCommand /// For reads, drops and etc. String column_name; - DataTypePtr data_type; + DataTypePtr data_type; /// Maybe empty if we just want to drop column /// If from_zookeeper, than consider more Alter commands as mutation commands static std::optional parse(ASTAlterCommand * command, bool from_zookeeper=false); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 323b939e737..574b73cfd74 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -988,7 +988,7 @@ bool StorageReplicatedMergeTree::executeLogEntry(LogEntry & entry) { do_fetch = !tryExecutePartMutation(entry); } - else if (entry.type == LogEntry::FINISH_ALTER) + else if (entry.type == LogEntry::ALTER_METADATA) { executeMetadataAlter(entry); } @@ -3345,6 +3345,7 @@ void StorageReplicatedMergeTree::alter( //std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; + ReplicatedMergeTreeLogEntryData entry; /// /columns and /metadata nodes std::vector changed_nodes; { @@ -3392,42 +3393,32 @@ void StorageReplicatedMergeTree::alter( for (const auto & node : changed_nodes) ops.emplace_back(zkutil::makeSetRequest(node.shared_path, node.new_value, -1)); + entry.type = LogEntry::ALTER_METADATA; + entry.source_replica = replica_name; + + WriteBufferFromString wb(entry.mutation_commands); + maybe_mutation_commands.writeText(wb); + wb.finalize(); + + entry.create_time = time(nullptr); + + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + Coordination::Responses results = getZooKeeper()->multi(ops); for (size_t i = 0; i < changed_nodes.size(); ++i) changed_nodes[i].new_version = dynamic_cast(*results[i]).stat.version; + + String path_created = dynamic_cast(*results.back()).path_created; + entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); } LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); table_lock_holder.release(); - ReplicatedMergeTreeLogEntryData entry; - entry.type = LogEntry::FINISH_ALTER; - entry.source_replica = replica_name; - - ////std::cerr << " Columns before mutation:" << getColumns().getAllPhysical().toString() << std::endl; - - entry.new_part_name = ""; - entry.create_time = time(nullptr); - - String path_created = getZooKeeper()->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); - entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - - ////std::cerr << "Waiting for replicas\n"; auto unwaited = waitForAllReplicasToProcessLogEntry(entry, false); - ////std::cerr << "Replicas done"; - - if (!maybe_mutation_commands.empty()) - { - ////std::cerr << "We have mutation commands:" << maybe_mutation_commands.size() << std::endl; - Context copy_context = query_context; - copy_context.getSettingsRef().mutations_sync = 2; - ReplicatedMergeTreeMutationEntry mutation_entry = mutateImpl(maybe_mutation_commands, copy_context); - ////std::cerr << "Mutation finished\n"; - } - if (!unwaited.empty()) { throw Exception("Some replicas doesn't finish alter", ErrorCodes::UNFINISHED); From 0893f7d49c9fece48657e44c4e5d713bca787ce1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 29 Jan 2020 20:28:39 +0300 Subject: [PATCH 0136/2007] Mutate from alter metadata task --- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 4 +- .../Storages/StorageReplicatedMergeTree.cpp | 51 ++++++++++++------- .../src/Storages/StorageReplicatedMergeTree.h | 3 +- .../0_stateless/01062_alter_on_mutataion.sql | 1 - 4 files changed, 37 insertions(+), 22 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index ce8d873b972..5bc8901ea35 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -70,7 +70,7 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const out << "sync_mode\n"; out << alter_sync_mode << "\n"; out << "mutatation_commands\n"; - out << mutation_commands << '\n'; + out << mutation_commands; break; default: @@ -165,7 +165,7 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) type = ALTER_METADATA; in >> "sync_mode\n"; in >> alter_sync_mode; - in >> "mutatation_commands\n"; + in >> "\nmutatation_commands\n"; in >> mutation_commands; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 574b73cfd74..2cd34472dbb 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1168,9 +1168,8 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } -bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & /*entry*/) +bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & entry) { - ////std::cerr << "Trying to finish alter\n"; auto zookeeper = getZooKeeper(); String columns_path = zookeeper_path + "/columns"; @@ -1191,7 +1190,6 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer const bool changed_columns_version = (columns_version_zk != columns_version); const bool changed_metadata_version = (metadata_version_zk != metadata_version); - ////std::cerr << "Versions changed: columns:" << changed_columns_version << " metadata:" << changed_metadata_version << std::endl; if (!(changed_columns_version || changed_metadata_version)) { @@ -1246,7 +1244,18 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer ////std::cerr << "Nodes in zk updated\n"; } - ////std::cerr << "Done\n"; + if (!entry.mutation_commands.empty()) + { + MutationCommands commands; + ReadBufferFromString in(entry.mutation_commands); + commands.readText(in); + if (is_leader) + { + auto mutation_entry = mutateImpl(commands); + waitMutation(mutation_entry, entry.alter_sync_mode); + } + } + return true; } @@ -3395,6 +3404,7 @@ void StorageReplicatedMergeTree::alter( entry.type = LogEntry::ALTER_METADATA; entry.source_replica = replica_name; + entry.alter_sync_mode = query_context.getSettingsRef().replication_alter_partitions_sync; WriteBufferFromString wb(entry.mutation_commands); maybe_mutation_commands.writeText(wb); @@ -4390,10 +4400,27 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const Context & query_context) { - mutateImpl(commands, query_context); + auto entry = mutateImpl(commands); + waitMutation(entry, query_context.getSettingsRef().mutations_sync); } -ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands, const Context & query_context) +void StorageReplicatedMergeTree::waitMutation(const ReplicatedMergeTreeMutationEntry & entry, size_t mutations_sync) const +{ + auto zookeeper = getZooKeeper(); + /// we have to wait + if (mutations_sync != 0) + { + Strings replicas; + if (mutations_sync == 2) /// wait for all replicas + replicas = zookeeper->getChildren(zookeeper_path + "/replicas"); + else if (mutations_sync == 1) /// just wait for ourself + replicas.push_back(replica_name); + + waitMutationToFinishOnReplicas(replicas, entry.znode_name); + } +} + +ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands) { /// Overview of the mutation algorithm. /// @@ -4497,18 +4524,6 @@ ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const Mu throw Coordination::Exception("Unable to create a mutation znode", rc); } - /// we have to wait - if (query_context.getSettingsRef().mutations_sync != 0) - { - Strings replicas; - if (query_context.getSettingsRef().mutations_sync == 2) /// wait for all replicas - replicas = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); - else if (query_context.getSettingsRef().mutations_sync == 1) /// just wait for ourself - replicas.push_back(replica_path); - - //std::cerr << "Waiting for mutation on replicas:" << replicas.size() << std::endl; - waitMutationToFinishOnReplicas(replicas, entry.znode_name); - } return entry; } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 9e46ee431d3..b03bb38c703 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -109,7 +109,8 @@ public: void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; void mutate(const MutationCommands & commands, const Context & context) override; - ReplicatedMergeTreeMutationEntry mutateImpl(const MutationCommands & commands, const Context & context); + ReplicatedMergeTreeMutationEntry mutateImpl(const MutationCommands & commands); + void waitMutation(const ReplicatedMergeTreeMutationEntry & entry, size_t mutation_sync) const; std::vector getMutationsStatus() const override; CancellationCode killMutation(const String & mutation_id) override; diff --git a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql index 32e9b9f22bc..a14e7f75aed 100644 --- a/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql +++ b/dbms/tests/queries/0_stateless/01062_alter_on_mutataion.sql @@ -54,7 +54,6 @@ ALTER TABLE test_alter_on_mutation MODIFY COLUMN value UInt64; SELECT sum(value) from test_alter_on_mutation; - DROP TABLE IF EXISTS test_alter_on_mutation; DROP TABLE IF EXISTS nested_alter; From 4b2eff855102081ba547dde1c3d797e142eb7a79 Mon Sep 17 00:00:00 2001 From: Vladimir Chebotarev Date: Thu, 30 Jan 2020 13:21:40 +0300 Subject: [PATCH 0137/2007] Added passing ttl info between replicas. --- .../Storages/MergeTree/DataPartsExchange.cpp | 36 ++++++++++++++----- dbms/tests/integration/test_ttl_move/test.py | 33 +++++++++++++++++ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index e459de6fa58..22e04e3628e 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -36,6 +36,8 @@ namespace static constexpr auto REPLICATION_PROTOCOL_VERSION_WITHOUT_PARTS_SIZE = "0"; static constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE = "1"; +static constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS = "2"; + std::string getEndpointId(const std::string & node_id) { @@ -53,10 +55,11 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo { String client_protocol_version = params.get("client_protocol_version", REPLICATION_PROTOCOL_VERSION_WITHOUT_PARTS_SIZE); - String part_name = params.get("part"); - if (client_protocol_version != REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE && client_protocol_version != REPLICATION_PROTOCOL_VERSION_WITHOUT_PARTS_SIZE) + if (client_protocol_version != REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS + && client_protocol_version != REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE + && client_protocol_version != REPLICATION_PROTOCOL_VERSION_WITHOUT_PARTS_SIZE) throw Exception("Unsupported fetch protocol version", ErrorCodes::UNKNOWN_PROTOCOL); const auto data_settings = data.getSettings(); @@ -75,7 +78,7 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo response.setChunkedTransferEncoding(false); return; } - response.addCookie({"server_protocol_version", REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE}); + response.addCookie({"server_protocol_version", REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS}); ++total_sends; SCOPE_EXIT({--total_sends;}); @@ -103,10 +106,16 @@ void Service::processQuery(const Poco::Net::HTMLForm & params, ReadBuffer & /*bo MergeTreeData::DataPart::Checksums data_checksums; - - if (client_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) + if (client_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE || client_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS) writeBinary(checksums.getTotalSizeOnDisk(), out); + if (client_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS) + { + WriteBufferFromOwnString ttl_infos_buffer; + part->ttl_infos.write(ttl_infos_buffer); + writeBinary(ttl_infos_buffer.str(), out); + } + writeBinary(checksums.files.size(), out); for (const auto & it : checksums.files) { @@ -192,7 +201,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( { {"endpoint", getEndpointId(replica_path)}, {"part", part_name}, - {"client_protocol_version", REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE}, + {"client_protocol_version", REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS}, {"compress", "false"} }); @@ -218,11 +227,22 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( ReservationPtr reservation; - if (server_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) + if (server_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE || server_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS) { size_t sum_files_size; readBinary(sum_files_size, in); - reservation = data.reserveSpace(sum_files_size); + if (server_protocol_version == REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS) + { + MergeTreeDataPart::TTLInfos ttl_infos; + String ttl_infos_string; + readBinary(ttl_infos_string, in); + ReadBufferFromString ttl_infos_buffer(ttl_infos_string); + assertString("ttl format version: 1\n", ttl_infos_buffer); + ttl_infos.read(ttl_infos_buffer); + reservation = data.reserveSpacePreferringTTLRules(sum_files_size, ttl_infos, std::time(nullptr)); + } + else + reservation = data.reserveSpace(sum_files_size); } else { diff --git a/dbms/tests/integration/test_ttl_move/test.py b/dbms/tests/integration/test_ttl_move/test.py index b498178e4d7..19f1dbe6430 100644 --- a/dbms/tests/integration/test_ttl_move/test.py +++ b/dbms/tests/integration/test_ttl_move/test.py @@ -332,6 +332,39 @@ def test_moves_to_disk_eventually_work(started_cluster, name, engine): node1.query("DROP TABLE IF EXISTS {}".format(name)) +def test_replicated_download_ttl_info(started_cluster): + name = "test_replicated_ttl_info" + engine = "ReplicatedMergeTree('/clickhouse/test_replicated_download_ttl_info', '{replica}')" + try: + for i, node in enumerate((node1, node2), start=1): + node.query(""" + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = {engine} + ORDER BY tuple() + TTL d1 TO DISK 'external' + SETTINGS storage_policy='small_jbod_with_external' + """.format(name=name, engine=engine)) + + node1.query("SYSTEM STOP MOVES {}".format(name)) + + node2.query("INSERT INTO {} (s1, d1) VALUES ('{}', toDateTime({}))".format(name, get_random_string(1024 * 1024), time.time()-100)) + + assert set(get_used_disks_for_table(node2, name)) == {"external"} + time.sleep(1) + + assert node1.query("SELECT count() FROM {}".format(name)).splitlines() == ["1"] + assert set(get_used_disks_for_table(node1, name)) == {"external"} + + finally: + for node in (node1, node2): + try: + node.query("DROP TABLE IF EXISTS {}".format(name)) + except: + continue + + @pytest.mark.skip(reason="Flappy test") @pytest.mark.parametrize("name,engine,positive", [ ("mt_test_merges_to_disk_do_not_work","MergeTree()",0), From 453be4c587a4a34ed8e7510204e3d400db041d64 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jan 2020 13:30:35 +0300 Subject: [PATCH 0138/2007] Removed changed node struct --- .../Storages/StorageReplicatedMergeTree.cpp | 32 +++---------------- 1 file changed, 4 insertions(+), 28 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 2cd34472dbb..9c40991ef86 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3330,22 +3330,6 @@ void StorageReplicatedMergeTree::alter( } - struct ChangedNode - { - ChangedNode(const String & table_path_, String name_, String new_value_) - : table_path(table_path_), name(std::move(name_)), shared_path(table_path + "/" + name), new_value(std::move(new_value_)) - { - } - - const String & table_path; - String name; - - String shared_path; - - String new_value; - int32_t new_version = -1; /// Initialization is to suppress (useless) false positive warning found by cppcheck. - }; - auto ast_to_str = [](ASTPtr query) -> String { if (!query) return ""; @@ -3355,8 +3339,8 @@ void StorageReplicatedMergeTree::alter( //std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; ReplicatedMergeTreeLogEntryData entry; + Coordination::Requests ops; /// /columns and /metadata nodes - std::vector changed_nodes; { /// Just to read current structure. Alter will be done in separate thread. auto table_lock = lockStructureForShare(false, query_context.getCurrentQueryId()); @@ -3368,8 +3352,9 @@ void StorageReplicatedMergeTree::alter( params.apply(metadata); String new_columns_str = metadata.columns.toString(); + /// TODO(alesap) maybe check correct version if (new_columns_str != getColumns().toString()) - changed_nodes.emplace_back(zookeeper_path, "columns", new_columns_str); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1)); ReplicatedMergeTreeTableMetadata new_metadata(*this); if (ast_to_str(metadata.order_by_ast) != ast_to_str(order_by_ast)) @@ -3388,20 +3373,14 @@ void StorageReplicatedMergeTree::alter( String new_metadata_str = new_metadata.toString(); if (new_metadata_str != ReplicatedMergeTreeTableMetadata(*this).toString()) - changed_nodes.emplace_back(zookeeper_path, "metadata", new_metadata_str); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, -1)); /// Perform settings update locally - auto old_metadata = getInMemoryMetadata(); old_metadata.settings_ast = metadata.settings_ast; changeSettings(metadata.settings_ast, table_lock_holder); global_context.getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, old_metadata); - /// Modify shared metadata nodes in ZooKeeper. - Coordination::Requests ops; - for (const auto & node : changed_nodes) - ops.emplace_back(zkutil::makeSetRequest(node.shared_path, node.new_value, -1)); - entry.type = LogEntry::ALTER_METADATA; entry.source_replica = replica_name; entry.alter_sync_mode = query_context.getSettingsRef().replication_alter_partitions_sync; @@ -3416,9 +3395,6 @@ void StorageReplicatedMergeTree::alter( Coordination::Responses results = getZooKeeper()->multi(ops); - for (size_t i = 0; i < changed_nodes.size(); ++i) - changed_nodes[i].new_version = dynamic_cast(*results[i]).stat.version; - String path_created = dynamic_cast(*results.back()).path_created; entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); } From 236f461420c121172783ffa63a369aa527c06582 Mon Sep 17 00:00:00 2001 From: millb Date: Thu, 30 Jan 2020 14:15:43 +0300 Subject: [PATCH 0139/2007] attempt --- dbms/src/Interpreters/Cluster.cpp | 24 +++++++++++++++++------- dbms/src/Interpreters/Cluster.h | 7 ++++--- 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 71bd89b2b6f..5473c3f7762 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -72,7 +72,8 @@ bool Cluster::Address::isLocal(UInt16 clickhouse_port) const } -Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix) +Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_number_, UInt32 replica_number_) : + shard_number(shard_number_), replica_number(replica_number_) { host_name = config.getString(config_prefix + ".host"); port = static_cast(config.getInt(config_prefix + ".port")); @@ -137,12 +138,13 @@ std::pair Cluster::Address::fromString(const String & host_port_ String Cluster::Address::toFullString() const { return - escapeForFileName(user) + - (password.empty() ? "" : (':' + escapeForFileName(password))) + '@' + + ((shard_number == 0) ? "" : "shard" + std::to_string(shard_number)) + + ((replica_number == 0) ? "" : "_replica" + std::to_string(replica_number)) + '@' + escapeForFileName(host_name) + ':' + std::to_string(port) + - (default_database.empty() ? "" : ('#' + escapeForFileName(default_database))) - + ((secure == Protocol::Secure::Enable) ? "+secure" : ""); + (default_database.empty() ? "" : ('#' + + escapeForFileName(default_database))) + + ((secure == Protocol::Secure::Enable) ? "+secure" : ""); } Cluster::Address Cluster::Address::fromFullString(const String & full_string) @@ -158,8 +160,14 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string) secure = Protocol::Secure::Enable; } + + const char * underscore = strchr(full_string.data(), '_'); + const char * slash = strchr(full_string.data(), '/'); const char * user_pw_end = strchr(full_string.data(), '@'); const char * colon = strchr(full_string.data(), ':'); + const bool has_shard = startsWith(full_string, "shard"); + if (has_shard && !slash) + throw Exception("Incorrect [shard{shard_number}[_replica{replica_number}]]/user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR); if (!user_pw_end || !colon) throw Exception("Incorrect user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR); @@ -175,9 +183,11 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string) address.secure = secure; address.port = parse(host_end + 1, port_end - (host_end + 1)); address.host_name = unescapeForFileName(std::string(user_pw_end + 1, host_end)); - address.user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end)); + address.user = unescapeForFileName(std::string(slash + 1, has_pw ? colon : user_pw_end)); address.password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string(); address.default_database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) : std::string(); + address.shard_number = has_shard ? parse(address_begin + 5) : 0; + address.replica_number = underscore ? parse(underscore + 8) : 0; return address; } @@ -309,7 +319,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting if (startsWith(replica_key, "replica")) { - replica_addresses.emplace_back(config, partial_prefix + replica_key); + replica_addresses.emplace_back(config, partial_prefix + replica_key, current_shard_num, current_replica_num); ++current_replica_num; if (!replica_addresses.back().is_local) diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h index ef12f9fe78f..3d769bf9c7b 100644 --- a/dbms/src/Interpreters/Cluster.h +++ b/dbms/src/Interpreters/Cluster.h @@ -57,6 +57,8 @@ public: UInt16 port; String user; String password; + UInt32 shard_number{}; + UInt32 replica_number{}; /// This database is selected when no database is specified for Distributed table String default_database; /// The locality is determined at the initialization, and is not changed even if DNS is changed @@ -67,9 +69,8 @@ public: Protocol::Secure secure = Protocol::Secure::Disable; Address() = default; - Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix); + Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_number_ = 0, UInt32 replica_number_ = 0); Address(const String & host_port_, const String & user_, const String & password_, UInt16 clickhouse_port, bool secure_ = false); - /// Returns 'escaped_host_name:port' String toString() const; @@ -80,7 +81,7 @@ public: static std::pair fromString(const String & host_port_string); - /// Retrurns escaped user:password@resolved_host_address:resolved_host_port#default_database + /// Retrurns escaped shard{shard_number}_replica{replica_number}@resolved_host_address:resolved_host_port#default_database String toFullString() const; static Address fromFullString(const String & address_full_string); From d0e5243ad857432e1255f8dee53faf34eb5ce82e Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 30 Jan 2020 15:54:52 +0300 Subject: [PATCH 0140/2007] Send updates in entry --- .../ReplicatedMergeTreeAlterThread.cpp | 406 +++++++++--------- .../ReplicatedMergeTreeAlterThread.h | 84 ++-- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 19 +- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 3 + .../ReplicatedMergeTreeTableMetadata.cpp | 7 + .../Storages/StorageReplicatedMergeTree.cpp | 164 +++---- .../src/Storages/StorageReplicatedMergeTree.h | 8 - 7 files changed, 336 insertions(+), 355 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp index d1a201df1be..d1a85087246 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp @@ -1,203 +1,203 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int NOT_FOUND_NODE; -} - -static const auto ALTER_ERROR_SLEEP_MS = 10 * 1000; - - -ReplicatedMergeTreeAlterThread::ReplicatedMergeTreeAlterThread(StorageReplicatedMergeTree & storage_) - : storage(storage_) - , zk_node_cache([&] { return storage.getZooKeeper(); }) - , log_name(storage.getStorageID().getFullTableName() + " (ReplicatedMergeTreeAlterThread)") - , log(&Logger::get(log_name)) -{ - task = storage_.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); -} - -void ReplicatedMergeTreeAlterThread::run() -{ - try - { - /** We have a description of columns in ZooKeeper, common for all replicas (Example: /clickhouse/tables/02-06/visits/columns), - * as well as a description of columns in local file with metadata (storage.getColumnsList()). - * - * If these descriptions are different - you need to do ALTER. - * - * If stored version of the node (columns_version) differs from the version in ZK, - * then the description of the columns in ZK does not necessarily differ from the local - * - this can happen with a loop from ALTER-s, which as a whole, does not change anything. - * In this case, you need to update the stored version number, - * and also check the structure of parts, and, if necessary, make ALTER. - * - * Recorded version number needs to be updated after updating the metadata, under lock. - * This version number is checked against the current one for INSERT. - * That is, we make sure to insert blocks with the correct structure. - * - * When the server starts, previous ALTER might not have been completed. - * Therefore, for the first time, regardless of the changes, we check the structure of all parts, - * (Example: /clickhouse/tables/02-06/visits/replicas/example02-06-1.yandex.ru/parts/20140806_20140831_131664_134988_3296/columns) - * and do ALTER if necessary. - * - * TODO: Too complicated, rewrite everything. - */ - - auto zookeeper = storage.getZooKeeper(); - - String columns_path = storage.zookeeper_path + "/columns"; - auto columns_znode = zk_node_cache.get(columns_path, task->getWatchCallback()); - if (!columns_znode.exists) - throw Exception(columns_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - int32_t columns_version = columns_znode.stat.version; - - String metadata_path = storage.zookeeper_path + "/metadata"; - auto metadata_znode = zk_node_cache.get(metadata_path, task->getWatchCallback()); - if (!metadata_znode.exists) - throw Exception(metadata_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - int32_t metadata_version = metadata_znode.stat.version; - - const bool changed_columns_version = (columns_version != storage.columns_version); - const bool changed_metadata_version = (metadata_version != storage.metadata_version); - - if (!(changed_columns_version || changed_metadata_version || force_recheck_parts)) - return; - - const String & columns_str = columns_znode.contents; - auto columns_in_zk = ColumnsDescription::parse(columns_str); - - const String & metadata_str = metadata_znode.contents; - auto metadata_in_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); - auto metadata_diff = ReplicatedMergeTreeTableMetadata(storage).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); - - /// If you need to lock table structure, then suspend merges and moves. - ActionLock merge_blocker = storage.merger_mutator.merges_blocker.cancel(); - ActionLock moves_blocker = storage.parts_mover.moves_blocker.cancel(); - - MergeTreeData::DataParts parts; - - /// If metadata nodes have changed, we will update table structure locally. - if (changed_columns_version || changed_metadata_version) - { - /// Temporarily cancel part checks to avoid locking for long time. - auto temporarily_stop_part_checks = storage.part_check_thread.temporarilyStop(); - - /// Temporarily cancel parts sending - ActionLock data_parts_exchange_blocker; - if (storage.data_parts_exchange_endpoint) - data_parts_exchange_blocker = storage.data_parts_exchange_endpoint->blocker.cancel(); - - /// Temporarily cancel part fetches - auto fetches_blocker = storage.fetcher.blocker.cancel(); - - LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); - - auto table_lock = storage.lockExclusively(RWLockImpl::NO_QUERY); - - if (columns_in_zk == storage.getColumns() && metadata_diff.empty()) - { - LOG_INFO(log, "Metadata nodes changed in ZooKeeper, but their contents didn't change. " - "Most probably it is a cyclic ALTER."); - } - else - { - LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); - - storage.setTableStructure(std::move(columns_in_zk), metadata_diff); - - LOG_INFO(log, "Applied changes to the metadata of the table."); - } - - /// You need to get a list of parts under table lock to avoid race condition with merge. - parts = storage.getDataParts(); - - storage.columns_version = columns_version; - storage.metadata_version = metadata_version; - } - - /// Update parts. - if (changed_columns_version || force_recheck_parts) - { - auto table_lock = storage.lockStructureForShare(false, RWLockImpl::NO_QUERY); - - if (changed_columns_version) - LOG_INFO(log, "ALTER-ing parts"); - - int changed_parts = 0; - - if (!changed_columns_version) - parts = storage.getDataParts(); - - const auto columns_for_parts = storage.getColumns().getAllPhysical(); - const auto indices_for_parts = storage.getIndices(); - - for (const MergeTreeData::DataPartPtr & part : parts) - { - /// Update the part and write result to temporary files. - /// TODO: You can skip checking for too large changes if ZooKeeper has, for example, - /// node /flags/force_alter. - MergeTreeData::AlterDataPartTransactionPtr transaction(new MergeTreeData::AlterDataPartTransaction(part)); - storage.alterDataPart(columns_for_parts, indices_for_parts.indices, false, transaction); - if (!transaction->isValid()) - continue; - - storage.updatePartHeaderInZooKeeperAndCommit(zookeeper, *transaction); - - ++changed_parts; - } - - /// Columns sizes could be quietly changed in case of MODIFY/ADD COLUMN - storage.recalculateColumnSizes(); - - if (changed_columns_version) - { - if (changed_parts != 0) - LOG_INFO(log, "ALTER-ed " << changed_parts << " parts"); - else - LOG_INFO(log, "No parts ALTER-ed"); - } - } - - /// Update metadata ZK nodes for a specific replica. - if (changed_columns_version || force_recheck_parts) - zookeeper->set(storage.replica_path + "/columns", columns_str); - if (changed_metadata_version || force_recheck_parts) - zookeeper->set(storage.replica_path + "/metadata", metadata_str); - - force_recheck_parts = false; - } - catch (const Coordination::Exception & e) - { - tryLogCurrentException(log, __PRETTY_FUNCTION__); - - if (e.code == Coordination::ZSESSIONEXPIRED) - return; - - force_recheck_parts = true; - task->scheduleAfter(ALTER_ERROR_SLEEP_MS); - } - catch (...) - { - tryLogCurrentException(log, __PRETTY_FUNCTION__); - - force_recheck_parts = true; - task->scheduleAfter(ALTER_ERROR_SLEEP_MS); - } -} - -} +//#include +//#include +//#include +//#include +//#include +//#include +//#include +//#include +//#include +// +//#include +// +// +//namespace DB +//{ +// +//namespace ErrorCodes +//{ +// extern const int NOT_FOUND_NODE; +//} +// +//static const auto ALTER_ERROR_SLEEP_MS = 10 * 1000; +// +// +//ReplicatedMergeTreeAlterThread::ReplicatedMergeTreeAlterThread(StorageReplicatedMergeTree & storage_) +// : storage(storage_) +// , zk_node_cache([&] { return storage.getZooKeeper(); }) +// , log_name(storage.getStorageID().getFullTableName() + " (ReplicatedMergeTreeAlterThread)") +// , log(&Logger::get(log_name)) +//{ +// task = storage_.global_context.getSchedulePool().createTask(log_name, [this]{ run(); }); +//} +// +//void ReplicatedMergeTreeAlterThread::run() +//{ +// try +// { +// /** We have a description of columns in ZooKeeper, common for all replicas (Example: /clickhouse/tables/02-06/visits/columns), +// * as well as a description of columns in local file with metadata (storage.getColumnsList()). +// * +// * If these descriptions are different - you need to do ALTER. +// * +// * If stored version of the node (columns_version) differs from the version in ZK, +// * then the description of the columns in ZK does not necessarily differ from the local +// * - this can happen with a loop from ALTER-s, which as a whole, does not change anything. +// * In this case, you need to update the stored version number, +// * and also check the structure of parts, and, if necessary, make ALTER. +// * +// * Recorded version number needs to be updated after updating the metadata, under lock. +// * This version number is checked against the current one for INSERT. +// * That is, we make sure to insert blocks with the correct structure. +// * +// * When the server starts, previous ALTER might not have been completed. +// * Therefore, for the first time, regardless of the changes, we check the structure of all parts, +// * (Example: /clickhouse/tables/02-06/visits/replicas/example02-06-1.yandex.ru/parts/20140806_20140831_131664_134988_3296/columns) +// * and do ALTER if necessary. +// * +// * TODO: Too complicated, rewrite everything. +// */ +// +// auto zookeeper = storage.getZooKeeper(); +// +// String columns_path = storage.zookeeper_path + "/columns"; +// auto columns_znode = zk_node_cache.get(columns_path, task->getWatchCallback()); +// if (!columns_znode.exists) +// throw Exception(columns_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); +// int32_t columns_version = columns_znode.stat.version; +// +// String metadata_path = storage.zookeeper_path + "/metadata"; +// auto metadata_znode = zk_node_cache.get(metadata_path, task->getWatchCallback()); +// if (!metadata_znode.exists) +// throw Exception(metadata_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); +// int32_t metadata_version = metadata_znode.stat.version; +// +// const bool changed_columns_version = (columns_version != storage.columns_version); +// const bool changed_metadata_version = (metadata_version != storage.metadata_version); +// +// if (!(changed_columns_version || changed_metadata_version || force_recheck_parts)) +// return; +// +// const String & columns_str = columns_znode.contents; +// auto columns_in_zk = ColumnsDescription::parse(columns_str); +// +// const String & metadata_str = metadata_znode.contents; +// auto metadata_in_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); +// auto metadata_diff = ReplicatedMergeTreeTableMetadata(storage).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); +// +// /// If you need to lock table structure, then suspend merges and moves. +// ActionLock merge_blocker = storage.merger_mutator.merges_blocker.cancel(); +// ActionLock moves_blocker = storage.parts_mover.moves_blocker.cancel(); +// +// MergeTreeData::DataParts parts; +// +// /// If metadata nodes have changed, we will update table structure locally. +// if (changed_columns_version || changed_metadata_version) +// { +// /// Temporarily cancel part checks to avoid locking for long time. +// auto temporarily_stop_part_checks = storage.part_check_thread.temporarilyStop(); +// +// /// Temporarily cancel parts sending +// ActionLock data_parts_exchange_blocker; +// if (storage.data_parts_exchange_endpoint) +// data_parts_exchange_blocker = storage.data_parts_exchange_endpoint->blocker.cancel(); +// +// /// Temporarily cancel part fetches +// auto fetches_blocker = storage.fetcher.blocker.cancel(); +// +// LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); +// +// auto table_lock = storage.lockExclusively(RWLockImpl::NO_QUERY); +// +// if (columns_in_zk == storage.getColumns() && metadata_diff.empty()) +// { +// LOG_INFO(log, "Metadata nodes changed in ZooKeeper, but their contents didn't change. " +// "Most probably it is a cyclic ALTER."); +// } +// else +// { +// LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); +// +// storage.setTableStructure(std::move(columns_in_zk), metadata_diff); +// +// LOG_INFO(log, "Applied changes to the metadata of the table."); +// } +// +// /// You need to get a list of parts under table lock to avoid race condition with merge. +// parts = storage.getDataParts(); +// +// storage.columns_version = columns_version; +// storage.metadata_version = metadata_version; +// } +// +// /// Update parts. +// if (changed_columns_version || force_recheck_parts) +// { +// auto table_lock = storage.lockStructureForShare(false, RWLockImpl::NO_QUERY); +// +// if (changed_columns_version) +// LOG_INFO(log, "ALTER-ing parts"); +// +// int changed_parts = 0; +// +// if (!changed_columns_version) +// parts = storage.getDataParts(); +// +// const auto columns_for_parts = storage.getColumns().getAllPhysical(); +// const auto indices_for_parts = storage.getIndices(); +// +// for (const MergeTreeData::DataPartPtr & part : parts) +// { +// /// Update the part and write result to temporary files. +// /// TODO: You can skip checking for too large changes if ZooKeeper has, for example, +// /// node /flags/force_alter. +// MergeTreeData::AlterDataPartTransactionPtr transaction(new MergeTreeData::AlterDataPartTransaction(part)); +// storage.alterDataPart(columns_for_parts, indices_for_parts.indices, false, transaction); +// if (!transaction->isValid()) +// continue; +// +// storage.updatePartHeaderInZooKeeperAndCommit(zookeeper, *transaction); +// +// ++changed_parts; +// } +// +// /// Columns sizes could be quietly changed in case of MODIFY/ADD COLUMN +// storage.recalculateColumnSizes(); +// +// if (changed_columns_version) +// { +// if (changed_parts != 0) +// LOG_INFO(log, "ALTER-ed " << changed_parts << " parts"); +// else +// LOG_INFO(log, "No parts ALTER-ed"); +// } +// } +// +// /// Update metadata ZK nodes for a specific replica. +// if (changed_columns_version || force_recheck_parts) +// zookeeper->set(storage.replica_path + "/columns", columns_str); +// if (changed_metadata_version || force_recheck_parts) +// zookeeper->set(storage.replica_path + "/metadata", metadata_str); +// +// force_recheck_parts = false; +// } +// catch (const Coordination::Exception & e) +// { +// tryLogCurrentException(log, __PRETTY_FUNCTION__); +// +// if (e.code == Coordination::ZSESSIONEXPIRED) +// return; +// +// force_recheck_parts = true; +// task->scheduleAfter(ALTER_ERROR_SLEEP_MS); +// } +// catch (...) +// { +// tryLogCurrentException(log, __PRETTY_FUNCTION__); +// +// force_recheck_parts = true; +// task->scheduleAfter(ALTER_ERROR_SLEEP_MS); +// } +//} +// +//} diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h index d5b8f6c2e67..d6c46dfcf68 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h @@ -1,42 +1,42 @@ -#pragma once - -#include -#include -#include -#include -#include -#include - - -namespace DB -{ - -class StorageReplicatedMergeTree; - - -/** Keeps track of changing the table structure in ZooKeeper and performs the necessary conversions. - * - * NOTE This has nothing to do with manipulating partitions, - * which are processed through the replication queue. - */ -class ReplicatedMergeTreeAlterThread -{ -public: - ReplicatedMergeTreeAlterThread(StorageReplicatedMergeTree & storage_); - - void start() { task->activateAndSchedule(); } - - void stop() { task->deactivate(); } - -private: - void run(); - - StorageReplicatedMergeTree & storage; - zkutil::ZooKeeperNodeCache zk_node_cache; - String log_name; - Logger * log; - BackgroundSchedulePool::TaskHolder task; - bool force_recheck_parts = true; -}; - -} +//#pragma once +// +//#include +//#include +//#include +//#include +//#include +//#include +// +// +//namespace DB +//{ +// +//class StorageReplicatedMergeTree; +// +// +///** Keeps track of changing the table structure in ZooKeeper and performs the necessary conversions. +// * +// * NOTE This has nothing to do with manipulating partitions, +// * which are processed through the replication queue. +// */ +//class ReplicatedMergeTreeAlterThread +//{ +//public: +// ReplicatedMergeTreeAlterThread(StorageReplicatedMergeTree & storage_); +// +// void start() { task->activateAndSchedule(); } +// +// void stop() { task->deactivate(); } +// +//private: +// void run(); +// +// StorageReplicatedMergeTree & storage; +// zkutil::ZooKeeperNodeCache zk_node_cache; +// String log_name; +// Logger * log; +// BackgroundSchedulePool::TaskHolder task; +// bool force_recheck_parts = true; +//}; +// +//} diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 5bc8901ea35..41fc42fc136 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -70,7 +71,13 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const out << "sync_mode\n"; out << alter_sync_mode << "\n"; out << "mutatation_commands\n"; - out << mutation_commands; + out << mutation_commands << "\n"; + out << "columns_str_size:\n"; + out << columns_str.size() << "\n"; + out << columns_str << "\n"; + out << "metadata_str_size:\n"; + out << metadata_str.size() << "\n"; + out << metadata_str; break; default: @@ -167,6 +174,16 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) in >> alter_sync_mode; in >> "\nmutatation_commands\n"; in >> mutation_commands; + in >> "\ncolumns_str_size:\n"; + size_t columns_size; + in >> columns_size >> "\n"; + columns_str.resize(columns_size); + in.readStrict(&columns_str[0], columns_size); + in >> "\nmetadata_str_size:\n"; + size_t metadata_size; + in >> metadata_size >> "\n"; + metadata_str.resize(metadata_size); + in.readStrict(&metadata_str[0], metadata_size); } //std::cerr << "Read backn\n"; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index 0e836ec7c27..b54ae11c2f0 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -113,6 +113,9 @@ struct ReplicatedMergeTreeLogEntryData /// Mutation commands for alter if any. String mutation_commands; + String columns_str; + String metadata_str; + /// Returns a set of parts that will appear after executing the entry + parts to block /// selection of merges. These parts are added to queue.virtual_parts. Strings getVirtualPartNames() const diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 4ebb51f0b41..b81108dcfe2 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -106,12 +106,19 @@ String ReplicatedMergeTreeTableMetadata::toString() const void ReplicatedMergeTreeTableMetadata::read(ReadBuffer & in) { in >> "metadata format version: 1\n"; + std::cerr << "READ METADATA1\n"; in >> "date column: " >> date_column >> "\n"; + std::cerr << "READ METADATA2\n"; in >> "sampling expression: " >> sampling_expression >> "\n"; + std::cerr << "READ METADATA3\n"; in >> "index granularity: " >> index_granularity >> "\n"; + std::cerr << "READ METADATA4\n"; in >> "mode: " >> merging_params_mode >> "\n"; + std::cerr << "READ METADATA5\n"; in >> "sign column: " >> sign_column >> "\n"; + std::cerr << "READ METADATA6\n"; in >> "primary key: " >> primary_key >> "\n"; + std::cerr << "READ METADATA\n"; if (in.eof()) data_format_version = 0; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 9c40991ef86..7e100dda03c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -463,11 +463,11 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo String metadata_str = zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); auto metadata_from_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); auto metadata_diff = old_metadata.checkAndFindDiff(metadata_from_zk, allow_alter); - metadata_version = metadata_stat.version; + //metadata_version = metadata_stat.version; Coordination::Stat columns_stat; auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &columns_stat)); - columns_version = columns_stat.version; + //columns_version = columns_stat.version; const ColumnsDescription & old_columns = getColumns(); if (columns_from_zk != old_columns || !metadata_diff.empty()) @@ -767,7 +767,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: part_name = part->name; check(part->columns); - int expected_columns_version = columns_version; + //int expected_columns_version = columns_version; auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( part->columns, part->checksums); @@ -839,8 +839,8 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: const auto storage_settings_ptr = getSettings(); String part_path = replica_path + "/parts/" + part_name; - ops.emplace_back(zkutil::makeCheckRequest( - zookeeper_path + "/columns", expected_columns_version)); + //ops.emplace_back(zkutil::makeCheckRequest( + // zookeeper_path + "/columns", expected_columns_version)); if (storage_settings_ptr->use_minimalistic_part_header_in_zookeeper) { @@ -1168,96 +1168,6 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) } -bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & entry) -{ - auto zookeeper = getZooKeeper(); - - String columns_path = zookeeper_path + "/columns"; - String columns_str; - Coordination::Stat columns_znode_stat; - if (!zookeeper->tryGet(columns_path, columns_str, &columns_znode_stat)) - throw Exception(columns_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - int32_t columns_version_zk = columns_znode_stat.version; - - String metadata_path = zookeeper_path + "/metadata"; - String metadata_str; - Coordination::Stat metadata_znode_stat; - - if (!zookeeper->tryGet(metadata_path, metadata_str, &metadata_znode_stat)) - throw Exception(metadata_path + " doesn't exist", ErrorCodes::NOT_FOUND_NODE); - int32_t metadata_version_zk = metadata_znode_stat.version; - - const bool changed_columns_version = (columns_version_zk != columns_version); - const bool changed_metadata_version = (metadata_version_zk != metadata_version); - - - if (!(changed_columns_version || changed_metadata_version)) - { - ////std::cerr << "Nothing changed\n"; - return true; - } - - ////std::cerr << "Receiving metadata from zookeeper\n"; - auto columns_in_zk = ColumnsDescription::parse(columns_str); - auto metadata_in_zk = ReplicatedMergeTreeTableMetadata::parse(metadata_str); - auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_in_zk, /* allow_alter = */ true); - - ////std::cerr << "Metadata received\n"; - - MergeTreeData::DataParts parts; - - /// If metadata nodes have changed, we will update table structure locally. - if (changed_columns_version || changed_metadata_version) - { - LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); - - auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); - - if (columns_in_zk == getColumns() && metadata_diff.empty()) - { - LOG_INFO( - log, - "Metadata nodes changed in ZooKeeper, but their contents didn't change. " - "Most probably it is a cyclic ALTER."); - } - else - { - LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); - - setTableStructure(std::move(columns_in_zk), metadata_diff); - - LOG_INFO(log, "Applied changes to the metadata of the table."); - } - - ////std::cerr << "Columns version before:" << columns_version << std::endl; - ////std::cerr << "Columns version after:" << columns_version_zk << std::endl; - columns_version = columns_version_zk; - metadata_version = metadata_version_zk; - - ////std::cerr << "Recalculating columns sizes\n"; - recalculateColumnSizes(); - /// Update metadata ZK nodes for a specific replica. - if (changed_columns_version) - zookeeper->set(replica_path + "/columns", columns_str); - if (changed_metadata_version) - zookeeper->set(replica_path + "/metadata", metadata_str); - - ////std::cerr << "Nodes in zk updated\n"; - } - if (!entry.mutation_commands.empty()) - { - MutationCommands commands; - ReadBufferFromString in(entry.mutation_commands); - commands.readText(in); - if (is_leader) - { - auto mutation_entry = mutateImpl(commands); - waitMutation(mutation_entry, entry.alter_sync_mode); - } - } - - return true; -} bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedMergeTree::LogEntry & entry) { @@ -3300,6 +3210,56 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p return true; } +bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMergeTree::LogEntry & entry) +{ + auto zookeeper = getZooKeeper(); + + auto columns_from_entry = ColumnsDescription::parse(entry.columns_str); + auto metadata_from_entry = ReplicatedMergeTreeTableMetadata::parse(entry.metadata_str); + auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_from_entry, /* allow_alter = */ true); + + ////std::cerr << "Metadata received\n"; + + MergeTreeData::DataParts parts; + + /// If metadata nodes have changed, we will update table structure locally. + { + LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); + + auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); + + LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); + + setTableStructure(std::move(columns_from_entry), metadata_diff); + + LOG_INFO(log, "Applied changes to the metadata of the table."); + + ////std::cerr << "Recalculating columns sizes\n"; + recalculateColumnSizes(); + /// Update metadata ZK nodes for a specific replica. + Coordination::Requests requests; + requests.emplace_back(zkutil::makeSetRequest(replica_path + "/columns", entry.columns_str, -1)); + requests.emplace_back(zkutil::makeSetRequest(replica_path + "/metadata", entry.metadata_str, -1)); + + zookeeper->multi(requests); + + std::cerr << "Nodes in zk updated\n"; + } + if (!entry.mutation_commands.empty()) + { + MutationCommands commands; + ReadBufferFromString in(entry.mutation_commands); + commands.readText(in); + if (is_leader) + { + auto mutation_entry = mutateImpl(commands); + waitMutation(mutation_entry, entry.alter_sync_mode); + } + } + + return true; +} + void StorageReplicatedMergeTree::alter( const AlterCommands & params, const Context & query_context, TableStructureWriteLockHolder & table_lock_holder) @@ -3384,6 +3344,8 @@ void StorageReplicatedMergeTree::alter( entry.type = LogEntry::ALTER_METADATA; entry.source_replica = replica_name; entry.alter_sync_mode = query_context.getSettingsRef().replication_alter_partitions_sync; + entry.metadata_str = new_metadata_str; + entry.columns_str = new_columns_str; WriteBufferFromString wb(entry.mutation_commands); maybe_mutation_commands.writeText(wb); @@ -3975,7 +3937,7 @@ void StorageReplicatedMergeTree::getStatus(Status & res, bool with_zk_fields) res.zookeeper_path = zookeeper_path; res.replica_name = replica_name; res.replica_path = replica_path; - res.columns_version = columns_version; + res.columns_version = -1; if (res.is_session_expired || !with_zk_fields) { @@ -4930,7 +4892,7 @@ void StorageReplicatedMergeTree::replacePartitionFrom(const StoragePtr & source_ entry_replace.new_part_names.emplace_back(part->name); for (const String & checksum : part_checksums) entry_replace.part_names_checksums.emplace_back(checksum); - entry_replace.columns_version = columns_version; + entry_replace.columns_version = -1; } /// We are almost ready to commit changes, remove fetches and merges from drop range @@ -5117,7 +5079,7 @@ void StorageReplicatedMergeTree::movePartitionToTable(const StoragePtr & dest_ta entry_replace.new_part_names.emplace_back(part->name); for (const String & checksum : part_checksums) entry_replace.part_names_checksums.emplace_back(checksum); - entry_replace.columns_version = columns_version; + entry_replace.columns_version = -1; } queue.removePartProducingOpsInRange(zookeeper, drop_range, entry); @@ -5223,9 +5185,9 @@ void StorageReplicatedMergeTree::getCommitPartOps( /// Information about the part, in the replica - ops.emplace_back(zkutil::makeCheckRequest( - zookeeper_path + "/columns", - columns_version)); + //ops.emplace_back(zkutil::makeCheckRequest( + // zookeeper_path + "/columns", + // columns_version)); if (storage_settings_ptr->use_minimalistic_part_header_in_zookeeper) { diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index b03bb38c703..228702b2a5f 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -218,14 +218,6 @@ private: */ zkutil::EphemeralNodeHolderPtr replica_is_active_node; - /** Version of the /columns node in ZooKeeper corresponding to the current data.columns. - * Read and modify along with the data.columns - under TableStructureLock. - */ - int columns_version = -1; - - /// Version of the /metadata node in ZooKeeper. - int metadata_version = -1; - /// Used to delay setting table structure till startup() in case of an offline ALTER. std::function set_table_structure_at_startup; From 3407d99ce365139fbf86c11e31c3d257687279c0 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 30 Jan 2020 23:12:00 +0300 Subject: [PATCH 0141/2007] Added draft version of random table function. Currently unimplemented data generators --- .../DataTypes/DataTypeAggregateFunction.cpp | 4 ++ .../src/DataTypes/DataTypeAggregateFunction.h | 1 + dbms/src/DataTypes/DataTypeArray.cpp | 7 ++ dbms/src/DataTypes/DataTypeArray.h | 1 + dbms/src/DataTypes/DataTypeDecimalBase.cpp | 7 ++ dbms/src/DataTypes/DataTypeDecimalBase.h | 1 + dbms/src/DataTypes/DataTypeEnum.h | 1 + dbms/src/DataTypes/DataTypeFixedString.cpp | 6 ++ dbms/src/DataTypes/DataTypeFixedString.h | 1 + dbms/src/DataTypes/DataTypeLowCardinality.cpp | 6 ++ dbms/src/DataTypes/DataTypeLowCardinality.h | 1 + dbms/src/DataTypes/DataTypeNothing.cpp | 8 +++ dbms/src/DataTypes/DataTypeNothing.h | 1 + dbms/src/DataTypes/DataTypeNullable.cpp | 5 ++ dbms/src/DataTypes/DataTypeNullable.h | 1 + dbms/src/DataTypes/DataTypeNumberBase.cpp | 7 ++ dbms/src/DataTypes/DataTypeNumberBase.h | 1 + dbms/src/DataTypes/DataTypeSet.h | 1 + dbms/src/DataTypes/DataTypeString.cpp | 5 ++ dbms/src/DataTypes/DataTypeString.h | 1 + dbms/src/DataTypes/DataTypeTuple.cpp | 8 +++ dbms/src/DataTypes/DataTypeTuple.h | 1 + dbms/src/DataTypes/IDataType.h | 4 ++ dbms/src/DataTypes/IDataTypeDummy.h | 5 ++ .../TableFunctions/TableFunctionRandom.cpp | 69 +++++++++++++++++++ dbms/src/TableFunctions/TableFunctionRandom.h | 20 ++++++ .../TableFunctions/registerTableFunctions.cpp | 1 + .../TableFunctions/registerTableFunctions.h | 1 + .../01072_random_table_function.sql | 1 + 29 files changed, 176 insertions(+) create mode 100644 dbms/src/TableFunctions/TableFunctionRandom.cpp create mode 100644 dbms/src/TableFunctions/TableFunctionRandom.h create mode 100644 dbms/tests/queries/0_stateless/01072_random_table_function.sql diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp index 8111b1de2fe..f3b26497912 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp @@ -304,6 +304,10 @@ MutableColumnPtr DataTypeAggregateFunction::createColumn() const return ColumnAggregateFunction::create(function); } +MutableColumnPtr DataTypeAggregateFunction::createColumnWithRandomData(size_t) const +{ + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} /// Create empty state Field DataTypeAggregateFunction::getDefault() const diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.h b/dbms/src/DataTypes/DataTypeAggregateFunction.h index 9ae7c67a803..e4c226b2917 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.h +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.h @@ -63,6 +63,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index e2c03805ea8..0500182c61a 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -487,6 +487,13 @@ MutableColumnPtr DataTypeArray::createColumn() const } +MutableColumnPtr DataTypeArray::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + Field DataTypeArray::getDefault() const { return Array(); diff --git a/dbms/src/DataTypes/DataTypeArray.h b/dbms/src/DataTypes/DataTypeArray.h index 1451f27dfbe..ccf269bd357 100644 --- a/dbms/src/DataTypes/DataTypeArray.h +++ b/dbms/src/DataTypes/DataTypeArray.h @@ -94,6 +94,7 @@ public: bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.cpp b/dbms/src/DataTypes/DataTypeDecimalBase.cpp index 7b9a391427c..a0f2bd7bd82 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.cpp +++ b/dbms/src/DataTypes/DataTypeDecimalBase.cpp @@ -41,6 +41,13 @@ MutableColumnPtr DataTypeDecimalBase::createColumn() const return ColumnType::create(0, scale); } +template +MutableColumnPtr DataTypeDecimalBase::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + template void DataTypeDecimalBase::serializeBinary(const Field & field, WriteBuffer & ostr) const { diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.h b/dbms/src/DataTypes/DataTypeDecimalBase.h index 11f7490e80a..d579b965412 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.h +++ b/dbms/src/DataTypes/DataTypeDecimalBase.h @@ -83,6 +83,7 @@ public: Field getDefault() const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return true; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeEnum.h b/dbms/src/DataTypes/DataTypeEnum.h index 2cb677984df..a0408df0279 100644 --- a/dbms/src/DataTypes/DataTypeEnum.h +++ b/dbms/src/DataTypes/DataTypeEnum.h @@ -111,6 +111,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override { return ColumnType::create(); } + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index d30f1003ca0..a148d0b2d22 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -268,6 +268,12 @@ MutableColumnPtr DataTypeFixedString::createColumn() const return ColumnFixedString::create(n); } +MutableColumnPtr DataTypeFixedString::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + Field DataTypeFixedString::getDefault() const { return String(); diff --git a/dbms/src/DataTypes/DataTypeFixedString.h b/dbms/src/DataTypes/DataTypeFixedString.h index 6d1f1c4db83..4f264d3ac86 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.h +++ b/dbms/src/DataTypes/DataTypeFixedString.h @@ -70,6 +70,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.cpp b/dbms/src/DataTypes/DataTypeLowCardinality.cpp index 5db32bd5380..24dc3af48c9 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.cpp +++ b/dbms/src/DataTypes/DataTypeLowCardinality.cpp @@ -934,6 +934,12 @@ MutableColumnPtr DataTypeLowCardinality::createColumn() const return ColumnLowCardinality::create(std::move(dictionary), std::move(indexes)); } +MutableColumnPtr DataTypeLowCardinality::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + Field DataTypeLowCardinality::getDefault() const { return dictionary_type->getDefault(); diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.h b/dbms/src/DataTypes/DataTypeLowCardinality.h index f8c314909b8..9b22acea7e3 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.h +++ b/dbms/src/DataTypes/DataTypeLowCardinality.h @@ -68,6 +68,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNothing.cpp b/dbms/src/DataTypes/DataTypeNothing.cpp index 79fbb002bff..ce4990748f9 100644 --- a/dbms/src/DataTypes/DataTypeNothing.cpp +++ b/dbms/src/DataTypes/DataTypeNothing.cpp @@ -14,6 +14,14 @@ MutableColumnPtr DataTypeNothing::createColumn() const return ColumnNothing::create(0); } + +MutableColumnPtr DataTypeNothing::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + void DataTypeNothing::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { size_t size = column.size(); diff --git a/dbms/src/DataTypes/DataTypeNothing.h b/dbms/src/DataTypes/DataTypeNothing.h index e9421fb15e8..5fbe0acc0a9 100644 --- a/dbms/src/DataTypes/DataTypeNothing.h +++ b/dbms/src/DataTypes/DataTypeNothing.h @@ -19,6 +19,7 @@ public: TypeIndex getTypeId() const override { return TypeIndex::Nothing; } MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; /// These methods read and write zero bytes just to allow to figure out size of column. void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index 397d5ba0a65..6f31e66a1e5 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -488,6 +488,11 @@ MutableColumnPtr DataTypeNullable::createColumn() const return ColumnNullable::create(nested_data_type->createColumn(), ColumnUInt8::create()); } +MutableColumnPtr DataTypeNullable::createColumnWithRandomData(size_t limit) const +{ + return ColumnNullable::create(nested_data_type->createColumnWithRandomData(limit), DataTypeUInt8().createColumnWithRandomData(limit)); +} + Field DataTypeNullable::getDefault() const { return Null(); diff --git a/dbms/src/DataTypes/DataTypeNullable.h b/dbms/src/DataTypes/DataTypeNullable.h index 1766b399c2a..83a76ae0410 100644 --- a/dbms/src/DataTypes/DataTypeNullable.h +++ b/dbms/src/DataTypes/DataTypeNullable.h @@ -76,6 +76,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNumberBase.cpp b/dbms/src/DataTypes/DataTypeNumberBase.cpp index 90356817730..937967d431a 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.cpp +++ b/dbms/src/DataTypes/DataTypeNumberBase.cpp @@ -239,6 +239,13 @@ MutableColumnPtr DataTypeNumberBase::createColumn() const return ColumnVector::create(); } +template +MutableColumnPtr DataTypeNumberBase::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + template bool DataTypeNumberBase::isValueRepresentedByInteger() const { diff --git a/dbms/src/DataTypes/DataTypeNumberBase.h b/dbms/src/DataTypes/DataTypeNumberBase.h index fb752ad5329..5a3dda5fe15 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.h +++ b/dbms/src/DataTypes/DataTypeNumberBase.h @@ -45,6 +45,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return false; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index 7ef0d931279..1d0d56c164b 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -21,6 +21,7 @@ public: // Used for expressions analysis. MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } + MutableColumnPtr createColumnWithRandomData(size_t) const override; // Used only for debugging, making it DUMPABLE Field getDefault() const override { return Tuple(); } diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index ef32fe33690..46478396a68 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -360,6 +360,11 @@ MutableColumnPtr DataTypeString::createColumn() const return ColumnString::create(); } +MutableColumnPtr DataTypeString::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} bool DataTypeString::equals(const IDataType & rhs) const { diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h index 28968eef3f1..4a2c6be42e1 100644 --- a/dbms/src/DataTypes/DataTypeString.h +++ b/dbms/src/DataTypes/DataTypeString.h @@ -54,6 +54,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 4d60177aa4d..5c912b89f2d 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -454,6 +454,14 @@ MutableColumnPtr DataTypeTuple::createColumn() const return ColumnTuple::create(std::move(tuple_columns)); } + +MutableColumnPtr DataTypeTuple::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + Field DataTypeTuple::getDefault() const { return Tuple(ext::map(elems, [] (const DataTypePtr & elem) { return elem->getDefault(); })); diff --git a/dbms/src/DataTypes/DataTypeTuple.h b/dbms/src/DataTypes/DataTypeTuple.h index 06f0f62026e..a3a8fb2847e 100644 --- a/dbms/src/DataTypes/DataTypeTuple.h +++ b/dbms/src/DataTypes/DataTypeTuple.h @@ -81,6 +81,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & reader, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/IDataType.h b/dbms/src/DataTypes/IDataType.h index 92d0c1057c5..04ad5896154 100644 --- a/dbms/src/DataTypes/IDataType.h +++ b/dbms/src/DataTypes/IDataType.h @@ -287,6 +287,10 @@ public: */ virtual MutableColumnPtr createColumn() const = 0; + /** Create column for corresponding type and fill with random values. + */ + virtual MutableColumnPtr createColumnWithRandomData(size_t size) const = 0; + /** Create ColumnConst for corresponding type, with specified size and value. */ ColumnPtr createColumnConst(size_t size, const Field & field) const; diff --git a/dbms/src/DataTypes/IDataTypeDummy.h b/dbms/src/DataTypes/IDataTypeDummy.h index f27359e5f74..e346689274f 100644 --- a/dbms/src/DataTypes/IDataTypeDummy.h +++ b/dbms/src/DataTypes/IDataTypeDummy.h @@ -42,6 +42,11 @@ public: throw Exception("Method createColumn() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); } + MutableColumnPtr createColumnWithRandomData(size_t) const override + { + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + Field getDefault() const override { throw Exception("Method getDefault() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp new file mode 100644 index 00000000000..f7ffe977698 --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -0,0 +1,69 @@ +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "registerTableFunctions.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; +} + +StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + ASTs & args_func = ast_function->children; + + if (args_func.size() != 1) + throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::LOGICAL_ERROR); + + ASTs & args = args_func.at(0)->children; + + if (args.size() > 2) + throw Exception("Table function '" + getName() + "' requires one or two arguments: structure (and limit).", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + /// Parsing first argument as table structure and creating a sample block + std::string structure = args[0]->as().value.safeGet(); + + UInt64 limit = 1; + /// Parsing second argument if present + if (args.size() == 2) + limit = args[1]->as().value.safeGet(); + + if (!limit) + throw Exception("Table function '" + getName() + "' limit should not be 0.", ErrorCodes::BAD_ARGUMENTS); + + ColumnsDescription columns = parseColumnsListFromString(structure, context); + + Block res_block; + for (const auto & name_type : columns.getOrdinary()) + Column c = name_type.type->createColumnWithRandomData(limit) ; + res_block.insert({ c, name_type.type, name_type.name }); + + auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); + res->startup(); + return res; +} + +void registerTableFunctionRandom(TableFunctionFactory & factory) +{ + factory.registerFunction(TableFunctionFactory::CaseInsensitive); +} + +} diff --git a/dbms/src/TableFunctions/TableFunctionRandom.h b/dbms/src/TableFunctions/TableFunctionRandom.h new file mode 100644 index 00000000000..c4f8e2bca37 --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionRandom.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace DB +{ +/* random(structure, limit) - creates a temporary storage filling columns with random data + * random is case-insensitive table function + */ +class TableFunctionRandom : public ITableFunction +{ +public: + static constexpr auto name = "generate"; + std::string getName() const override { return name; } +private: + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; +}; + + +} diff --git a/dbms/src/TableFunctions/registerTableFunctions.cpp b/dbms/src/TableFunctions/registerTableFunctions.cpp index 35021cd46d0..91b6b94440c 100644 --- a/dbms/src/TableFunctions/registerTableFunctions.cpp +++ b/dbms/src/TableFunctions/registerTableFunctions.cpp @@ -15,6 +15,7 @@ void registerTableFunctions() registerTableFunctionURL(factory); registerTableFunctionValues(factory); registerTableFunctionInput(factory); + registerTableFunctionRandom(factory); #if USE_AWS_S3 registerTableFunctionS3(factory); diff --git a/dbms/src/TableFunctions/registerTableFunctions.h b/dbms/src/TableFunctions/registerTableFunctions.h index 66f2dda90ea..8ae5ab339f4 100644 --- a/dbms/src/TableFunctions/registerTableFunctions.h +++ b/dbms/src/TableFunctions/registerTableFunctions.h @@ -12,6 +12,7 @@ void registerTableFunctionFile(TableFunctionFactory & factory); void registerTableFunctionURL(TableFunctionFactory & factory); void registerTableFunctionValues(TableFunctionFactory & factory); void registerTableFunctionInput(TableFunctionFactory & factory); +void registerTableFunctionRandom(TableFunctionFactory & factory); #if USE_AWS_S3 void registerTableFunctionS3(TableFunctionFactory & factory); diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.sql b/dbms/tests/queries/0_stateless/01072_random_table_function.sql new file mode 100644 index 00000000000..fb217befea5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.sql @@ -0,0 +1 @@ +SELECT * FROM random(3) From 3f4db956ca7e305378c22e3df3bbcbeb4a37c988 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 31 Jan 2020 09:36:29 +0300 Subject: [PATCH 0142/2007] fix --- dbms/src/TableFunctions/TableFunctionRandom.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index f7ffe977698..b68bde17550 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "registerTableFunctions.h" @@ -44,7 +45,7 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C UInt64 limit = 1; /// Parsing second argument if present if (args.size() == 2) - limit = args[1]->as().value.safeGet(); + limit = args[1]->as().value.safeGet(); if (!limit) throw Exception("Table function '" + getName() + "' limit should not be 0.", ErrorCodes::BAD_ARGUMENTS); @@ -53,8 +54,8 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C Block res_block; for (const auto & name_type : columns.getOrdinary()) - Column c = name_type.type->createColumnWithRandomData(limit) ; - res_block.insert({ c, name_type.type, name_type.name }); + MutableColumnPtr column = name_type.type->createColumnWithRandomData(limit); + res_block.insert({ column, name_type.type, name_type.name }); auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); From 5717d233bdf8ff7dae7e546f1791bfee591f3ac3 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 31 Jan 2020 10:58:41 +0300 Subject: [PATCH 0143/2007] build fix --- dbms/src/DataTypes/DataTypeEnum.cpp | 8 ++++++++ dbms/src/DataTypes/DataTypeSet.h | 5 ++++- dbms/src/TableFunctions/TableFunctionRandom.cpp | 4 +++- .../queries/0_stateless/01072_random_table_function.sql | 2 +- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index 5ca6296f43d..fcaf9f2c8a3 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -347,6 +347,14 @@ Field DataTypeEnum::castToValue(const Field & value_or_name) const } +template +MutableColumnPtr DataTypeEnum::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + /// Explicit instantiations. template class DataTypeEnum; template class DataTypeEnum; diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index 1d0d56c164b..f468881cfe9 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -21,7 +21,10 @@ public: // Used for expressions analysis. MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } - MutableColumnPtr createColumnWithRandomData(size_t) const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override + { + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } // Used only for debugging, making it DUMPABLE Field getDefault() const override { return Tuple(); } diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index b68bde17550..3d4bb1d3247 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -54,8 +54,10 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C Block res_block; for (const auto & name_type : columns.getOrdinary()) + { MutableColumnPtr column = name_type.type->createColumnWithRandomData(limit); - res_block.insert({ column, name_type.type, name_type.name }); + res_block.insert({std::move(column), name_type.type, name_type.name}); + } auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.sql b/dbms/tests/queries/0_stateless/01072_random_table_function.sql index fb217befea5..21f0925439d 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.sql +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.sql @@ -1 +1 @@ -SELECT * FROM random(3) +SELECT * FROM generate('id int', 3) From 240f1e3e96d41fadd14a39fd7315c9f5927c4cce Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 11:14:20 +0300 Subject: [PATCH 0144/2007] Processors support for StorageDictionary. --- dbms/src/Storages/StorageDictionary.cpp | 11 +++++++++-- dbms/src/Storages/StorageDictionary.h | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/StorageDictionary.cpp b/dbms/src/Storages/StorageDictionary.cpp index 51914b53ae6..dce2d85d3ff 100644 --- a/dbms/src/Storages/StorageDictionary.cpp +++ b/dbms/src/Storages/StorageDictionary.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include namespace DB @@ -43,7 +45,7 @@ StorageDictionary::StorageDictionary( } } -BlockInputStreams StorageDictionary::read( +Pipes StorageDictionary::readWithProcessors( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, @@ -52,7 +54,12 @@ BlockInputStreams StorageDictionary::read( const unsigned /*threads*/) { auto dictionary = context.getExternalDictionariesLoader().getDictionary(dictionary_name); - return BlockInputStreams{dictionary->getBlockInputStream(column_names, max_block_size)}; + auto stream = dictionary->getBlockInputStream(column_names, max_block_size); + auto source = std::make_shared(stream); + /// TODO: update dictionary interface for processors. + Pipes pipes; + pipes.emplace_back(std::move(source)); + return pipes; } NamesAndTypesList StorageDictionary::getNamesAndTypes(const DictionaryStructure & dictionary_structure) diff --git a/dbms/src/Storages/StorageDictionary.h b/dbms/src/Storages/StorageDictionary.h index 1e741a05094..7dda2dc4310 100644 --- a/dbms/src/Storages/StorageDictionary.h +++ b/dbms/src/Storages/StorageDictionary.h @@ -25,13 +25,15 @@ class StorageDictionary : public ext::shared_ptr_helper, publ public: std::string getName() const override { return "Dictionary"; } - BlockInputStreams read(const Names & column_names, + Pipes readWithProcessors(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size = DEFAULT_BLOCK_SIZE, unsigned threads = 1) override; + bool supportProcessorsPipeline() const override { return true; } + static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure); template From 0157de021ac9fdb94b194d9c455105d3497f9303 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 11:54:57 +0300 Subject: [PATCH 0145/2007] Processors support for StorageDistributed reading. --- .../ClusterProxy/IStreamFactory.h | 5 +++- .../ClusterProxy/SelectStreamFactory.cpp | 25 +++++++++++++------ .../ClusterProxy/SelectStreamFactory.h | 2 +- .../ClusterProxy/executeQuery.cpp | 5 ++-- .../Interpreters/ClusterProxy/executeQuery.h | 5 +++- dbms/src/Processors/QueryPipeline.cpp | 11 +++++--- .../Sources/SourceFromInputStream.cpp | 16 ++++++++++++ .../Sources/SourceFromInputStream.h | 13 +++++++--- dbms/src/Storages/StorageDistributed.cpp | 2 +- dbms/src/Storages/StorageDistributed.h | 4 ++- 10 files changed, 66 insertions(+), 22 deletions(-) diff --git a/dbms/src/Interpreters/ClusterProxy/IStreamFactory.h b/dbms/src/Interpreters/ClusterProxy/IStreamFactory.h index 783ab0020c0..dbeb4b9cfd2 100644 --- a/dbms/src/Interpreters/ClusterProxy/IStreamFactory.h +++ b/dbms/src/Interpreters/ClusterProxy/IStreamFactory.h @@ -12,6 +12,9 @@ class Context; class Cluster; class Throttler; +class Pipe; +using Pipes = std::vector; + namespace ClusterProxy { @@ -26,7 +29,7 @@ public: const Cluster::ShardInfo & shard_info, const String & query, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, - BlockInputStreams & res) = 0; + Pipes & res) = 0; }; } diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index d5c57db76c4..2ee516852dc 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -12,7 +12,10 @@ #include #include - +#include +#include +#include +#include namespace ProfileEvents { @@ -67,12 +70,12 @@ SelectStreamFactory::SelectStreamFactory( namespace { -BlockInputStreamPtr createLocalStream(const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage) +Pipe createLocalStream(const ASTPtr & query_ast, const Block & header, const Context & context, QueryProcessingStage::Enum processed_stage) { checkStackSize(); InterpreterSelectQuery interpreter{query_ast, context, SelectQueryOptions(processed_stage)}; - BlockInputStreamPtr stream = interpreter.execute().in; + Pipe pipe = interpreter.executeWithProcessors().getPipe(); /** Materialization is needed, since from remote servers the constants come materialized. * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, @@ -84,7 +87,12 @@ BlockInputStreamPtr createLocalStream(const ASTPtr & query_ast, const Block & he */ /// return std::make_shared(stream); - return std::make_shared(context, stream, header, ConvertingBlockInputStream::MatchColumnsMode::Name); + auto converting = std::make_shared( + pipe.getHeader(), header,ConvertingTransform::MatchColumnsMode::Name, context); + + pipe.addSimpleTransform(std::move(converting)); + + return pipe; } static String formattedAST(const ASTPtr & ast) @@ -102,7 +110,7 @@ void SelectStreamFactory::createForShard( const Cluster::ShardInfo & shard_info, const String &, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, - BlockInputStreams & res) + Pipes & res) { auto modified_query_ast = query_ast->clone(); if (has_virtual_shard_num_column) @@ -122,7 +130,8 @@ void SelectStreamFactory::createForShard( stream->setPoolMode(PoolMode::GET_MANY); if (!table_func_ptr) stream->setMainTable(main_table); - res.emplace_back(std::move(stream)); + + res.emplace_back(std::make_shared(std::move(stream))); }; const auto & settings = context.getSettingsRef(); @@ -250,7 +259,7 @@ void SelectStreamFactory::createForShard( } if (try_results.empty() || local_delay < max_remote_delay) - return createLocalStream(modified_query_ast, header, context, stage); + return std::make_shared(createLocalStream(modified_query_ast, header, context, stage)); else { std::vector connections; @@ -263,7 +272,7 @@ void SelectStreamFactory::createForShard( } }; - res.emplace_back(std::make_shared("LazyShardWithLocalReplica", header, lazily_create_stream)); + res.emplace_back(std::make_shared("LazyShardWithLocalReplica", header, lazily_create_stream)); } else emplace_remote_stream(); diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h index e2fbf422246..a800a22f128 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -35,7 +35,7 @@ public: const Cluster::ShardInfo & shard_info, const String & query, const ASTPtr & query_ast, const Context & context, const ThrottlerPtr & throttler, - BlockInputStreams & res) override; + Pipes & res) override; private: const Block header; diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp index 9a0494cca45..3074652a245 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -36,11 +37,11 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin return new_context; } -BlockInputStreams executeQuery( +Pipes executeQuery( IStreamFactory & stream_factory, const ClusterPtr & cluster, const ASTPtr & query_ast, const Context & context, const Settings & settings) { - BlockInputStreams res; + Pipes res; const std::string query = queryToString(query_ast); diff --git a/dbms/src/Interpreters/ClusterProxy/executeQuery.h b/dbms/src/Interpreters/ClusterProxy/executeQuery.h index e8704b73d56..c78d802e9fe 100644 --- a/dbms/src/Interpreters/ClusterProxy/executeQuery.h +++ b/dbms/src/Interpreters/ClusterProxy/executeQuery.h @@ -10,6 +10,9 @@ struct Settings; class Context; class Cluster; +class Pipe; +using Pipes = std::vector; + namespace ClusterProxy { @@ -22,7 +25,7 @@ Context removeUserRestrictionsFromSettings(const Context & context, const Settin /// Execute a distributed query, creating a vector of BlockInputStreams, from which the result can be read. /// `stream_factory` object encapsulates the logic of creating streams for a different type of query /// (currently SELECT, DESCRIBE). -BlockInputStreams executeQuery( +Pipes executeQuery( IStreamFactory & stream_factory, const ClusterPtr & cluster, const ASTPtr & query_ast, const Context & context, const Settings & settings); diff --git a/dbms/src/Processors/QueryPipeline.cpp b/dbms/src/Processors/QueryPipeline.cpp index 53c58bb5dd9..187631aa552 100644 --- a/dbms/src/Processors/QueryPipeline.cpp +++ b/dbms/src/Processors/QueryPipeline.cpp @@ -597,11 +597,14 @@ void QueryPipeline::calcRowsBeforeLimit() if (auto * source = typeid_cast(processor)) { - auto & info = source->getStream().getProfileInfo(); - if (info.hasAppliedLimit()) + if (auto & stream = source->getStream()) { - has_limit = visited_limit = true; - rows_before_limit_at_least += info.getRowsBeforeLimit(); + auto & info = stream->getProfileInfo(); + if (info.hasAppliedLimit()) + { + has_limit = visited_limit = true; + rows_before_limit_at_least += info.getRowsBeforeLimit(); + } } } } diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.cpp b/dbms/src/Processors/Sources/SourceFromInputStream.cpp index 06206dbb667..a9b4ca96c04 100644 --- a/dbms/src/Processors/Sources/SourceFromInputStream.cpp +++ b/dbms/src/Processors/Sources/SourceFromInputStream.cpp @@ -10,6 +10,19 @@ SourceFromInputStream::SourceFromInputStream(BlockInputStreamPtr stream_, bool f : ISourceWithProgress(stream_->getHeader()) , force_add_aggregating_info(force_add_aggregating_info_) , stream(std::move(stream_)) +{ + init(); +} + +SourceFromInputStream::SourceFromInputStream(String name, Block header, std::function stream_builder_) + : ISourceWithProgress(std::move(header)) + , stream_builder(std::move(stream_builder_)) + , source_name(std::move(name)) +{ + init(); +} + +void SourceFromInputStream::init() { auto & sample = getPort().getHeader(); for (auto & type : sample.getDataTypes()) @@ -104,6 +117,9 @@ Chunk SourceFromInputStream::generate() if (!is_stream_started) { + if (!stream) + stream = stream_builder(); + stream->readPrefix(); is_stream_started = true; } diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.h b/dbms/src/Processors/Sources/SourceFromInputStream.h index b5704fc521f..bb043c03006 100644 --- a/dbms/src/Processors/Sources/SourceFromInputStream.h +++ b/dbms/src/Processors/Sources/SourceFromInputStream.h @@ -12,14 +12,16 @@ class SourceFromInputStream : public ISourceWithProgress { public: explicit SourceFromInputStream(BlockInputStreamPtr stream_, bool force_add_aggregating_info_ = false); - String getName() const override { return "SourceFromInputStream"; } + /// Constructor which works like LazyBlockInputStream. First 'generate' method creates stream using callback. + SourceFromInputStream(String name, Block header, std::function stream_builder_); + String getName() const override { return source_name.empty() ? "SourceFromInputStream" : source_name; } Status prepare() override; void work() override; Chunk generate() override; - IBlockInputStream & getStream() { return *stream; } + BlockInputStreamPtr & getStream() { return stream; } void addTotalsPort(); @@ -35,9 +37,12 @@ protected: private: bool has_aggregate_functions = false; - bool force_add_aggregating_info; + bool force_add_aggregating_info = false; BlockInputStreamPtr stream; + std::function stream_builder; + String source_name; + Chunk totals; bool has_totals_port = false; bool has_totals = false; @@ -45,6 +50,8 @@ private: bool is_generating_finished = false; bool is_stream_finished = false; bool is_stream_started = false; + + void init(); }; } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index ddf727c21de..ce5a4f84f7e 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -340,7 +340,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(const Con : QueryProcessingStage::WithMergeableState; } -BlockInputStreams StorageDistributed::read( +Pipes StorageDistributed::readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, diff --git a/dbms/src/Storages/StorageDistributed.h b/dbms/src/Storages/StorageDistributed.h index a3fd6f3c6e1..34a3eba639a 100644 --- a/dbms/src/Storages/StorageDistributed.h +++ b/dbms/src/Storages/StorageDistributed.h @@ -68,7 +68,7 @@ public: QueryProcessingStage::Enum getQueryProcessingStage(const Context & context) const override; QueryProcessingStage::Enum getQueryProcessingStage(const Context & context, const ClusterPtr & cluster) const; - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -76,6 +76,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; void drop(TableStructureWriteLockHolder &) override {} From 165afabdd247150df6bb9763cf3015a546d16a94 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 31 Jan 2020 12:22:30 +0300 Subject: [PATCH 0146/2007] second attempt --- dbms/src/Interpreters/Cluster.cpp | 42 ++----------------- dbms/src/Interpreters/Cluster.h | 5 ++- .../Storages/Distributed/DirectoryMonitor.cpp | 19 ++------- 3 files changed, 10 insertions(+), 56 deletions(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 5473c3f7762..e8b95a4dc47 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -139,53 +139,17 @@ String Cluster::Address::toFullString() const { return ((shard_number == 0) ? "" : "shard" + std::to_string(shard_number)) + - ((replica_number == 0) ? "" : "_replica" + std::to_string(replica_number)) + '@' + - escapeForFileName(host_name) + ':' + - std::to_string(port) + - (default_database.empty() ? "" : ('#' + - escapeForFileName(default_database))) + - ((secure == Protocol::Secure::Enable) ? "+secure" : ""); + ((replica_number == 0) ? "" : "_replica" + std::to_string(replica_number)); } Cluster::Address Cluster::Address::fromFullString(const String & full_string) { const char * address_begin = full_string.data(); - const char * address_end = address_begin + full_string.size(); - Protocol::Secure secure = Protocol::Secure::Disable; - const char * secure_tag = "+secure"; - if (endsWith(full_string, secure_tag)) - { - address_end -= strlen(secure_tag); - secure = Protocol::Secure::Enable; - } - - - const char * underscore = strchr(full_string.data(), '_'); - const char * slash = strchr(full_string.data(), '/'); - const char * user_pw_end = strchr(full_string.data(), '@'); - const char * colon = strchr(full_string.data(), ':'); - const bool has_shard = startsWith(full_string, "shard"); - if (has_shard && !slash) - throw Exception("Incorrect [shard{shard_number}[_replica{replica_number}]]/user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR); - if (!user_pw_end || !colon) - throw Exception("Incorrect user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR); - - const bool has_pw = colon < user_pw_end; - const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon; - if (!host_end) - throw Exception("Incorrect address '" + full_string + "', it does not contain port", ErrorCodes::SYNTAX_ERROR); - - const char * has_db = strchr(full_string.data(), '#'); - const char * port_end = has_db ? has_db : address_end; + bool has_shard = startsWith("shard", full_string); + bool underscore = strchr(full_string.data(), '_'); Address address; - address.secure = secure; - address.port = parse(host_end + 1, port_end - (host_end + 1)); - address.host_name = unescapeForFileName(std::string(user_pw_end + 1, host_end)); - address.user = unescapeForFileName(std::string(slash + 1, has_pw ? colon : user_pw_end)); - address.password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string(); - address.default_database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) : std::string(); address.shard_number = has_shard ? parse(address_begin + 5) : 0; address.replica_number = underscore ? parse(underscore + 8) : 0; return address; diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h index 3d769bf9c7b..b1cbdb22e9a 100644 --- a/dbms/src/Interpreters/Cluster.h +++ b/dbms/src/Interpreters/Cluster.h @@ -57,8 +57,9 @@ public: UInt16 port; String user; String password; - UInt32 shard_number{}; - UInt32 replica_number{}; + UInt32 shard_number{}; /// shard serial number in configuration file + UInt32 replica_number{}; /// replica serial number in this shard + /// This database is selected when no database is specified for Distributed table String default_database; /// The locality is determined at the initialization, and is not changed even if DNS is changed diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 43a595cde6f..ea9a659314b 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -188,25 +188,14 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri { const auto & cluster = storage.getCluster(); const auto & shards_info = cluster->getShardsInfo(); - const auto & shards_addresses = cluster->getShardsAddresses(); + // const auto & shards_addresses = cluster->getShardsInfo(); /// existing connections pool have a higher priority - for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) - { - const Cluster::Addresses & replicas_addresses = shards_addresses[shard_index]; + return shards_info[address.shard_number].per_replica_pools[address.replica_number]; - for (size_t replica_index = 0; replica_index < replicas_addresses.size(); ++replica_index) - { - const Cluster::Address & replica_address = replicas_addresses[replica_index]; - - if (address == replica_address) - return shards_info[shard_index].per_replica_pools[replica_index]; - } - } - - return std::make_shared( + /*return std::make_shared( 1, address.host_name, address.port, address.default_database, address.user, address.password, - storage.getName() + '_' + address.user, Protocol::Compression::Enable, address.secure); + storage.getName() + '_' + address.user, Protocol::Compression::Enable, address.secure);*/ }; auto pools = createPoolsForAddresses(name, pool_factory); From 613b7314f64f8d4c23ce9dc985e1a0ed3fa43d33 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 31 Jan 2020 12:29:28 +0300 Subject: [PATCH 0147/2007] fixed bugs --- dbms/src/Interpreters/Cluster.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index e8b95a4dc47..18657bc94cf 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -147,7 +147,7 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string) const char * address_begin = full_string.data(); bool has_shard = startsWith("shard", full_string); - bool underscore = strchr(full_string.data(), '_'); + const char * underscore = strchr(full_string.data(), '_'); Address address; address.shard_number = has_shard ? parse(address_begin + 5) : 0; From 2dab300ad4b86c29942d77f51087db4217ef8396 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 31 Jan 2020 13:49:10 +0300 Subject: [PATCH 0148/2007] attempt --- dbms/src/Interpreters/Cluster.cpp | 47 +++++++++++++++++++++++++++---- dbms/src/Interpreters/Cluster.h | 10 ++++--- 2 files changed, 47 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 18657bc94cf..d390ce8cbaa 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -72,8 +72,8 @@ bool Cluster::Address::isLocal(UInt16 clickhouse_port) const } -Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_number_, UInt32 replica_number_) : - shard_number(shard_number_), replica_number(replica_number_) +Cluster::Address::Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_index_, UInt32 replica_index_) : + shard_index(shard_index_), replica_index(replica_index_) { host_name = config.getString(config_prefix + ".host"); port = static_cast(config.getInt(config_prefix + ".port")); @@ -145,13 +145,48 @@ String Cluster::Address::toFullString() const Cluster::Address Cluster::Address::fromFullString(const String & full_string) { const char * address_begin = full_string.data(); + const char * address_end = address_begin + full_string.size(); - bool has_shard = startsWith("shard", full_string); - const char * underscore = strchr(full_string.data(), '_'); + Protocol::Secure secure = Protocol::Secure::Disable; + const char * secure_tag = "+secure"; + if (endsWith(full_string, secure_tag)) + { + address_end -= strlen(secure_tag); + secure = Protocol::Secure::Enable; + } + + const char * user_pw_end = strchr(full_string.data(), '@'); + + /// parsing with format [shard{shard_number}[_replica{replica_number}]] + if (!user_pw_end && startsWith("shard", full_string)) + { + const char * underscore = strchr(full_string.data(), '_'); + + Address address; + address.shard_index = parse(address_begin + 5); + address.replica_index = underscore ? parse(underscore + 8) : 0; + return address; + } + + const char * colon = strchr(full_string.data(), ':'); + if (!user_pw_end || !colon) + throw Exception("Incorrect user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR); + + const bool has_pw = colon < user_pw_end; + const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon; + if (!host_end) + throw Exception("Incorrect address '" + full_string + "', it does not contain port", ErrorCodes::SYNTAX_ERROR); + + const char * has_db = strchr(full_string.data(), '#'); + const char * port_end = has_db ? has_db : address_end; Address address; - address.shard_number = has_shard ? parse(address_begin + 5) : 0; - address.replica_number = underscore ? parse(underscore + 8) : 0; + address.secure = secure; + address.port = parse(host_end + 1, port_end - (host_end + 1)); + address.host_name = unescapeForFileName(std::string(user_pw_end + 1, host_end)); + address.user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end)); + address.password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string(); + address.default_database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) : std::string(); return address; } diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h index b1cbdb22e9a..e1ee1994d65 100644 --- a/dbms/src/Interpreters/Cluster.h +++ b/dbms/src/Interpreters/Cluster.h @@ -57,8 +57,8 @@ public: UInt16 port; String user; String password; - UInt32 shard_number{}; /// shard serial number in configuration file - UInt32 replica_number{}; /// replica serial number in this shard + UInt32 shard_index{}; /// shard serial number in configuration file + UInt32 replica_index{}; /// replica serial number in this shard /// This database is selected when no database is specified for Distributed table String default_database; @@ -70,7 +70,7 @@ public: Protocol::Secure secure = Protocol::Secure::Disable; Address() = default; - Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_number_ = 0, UInt32 replica_number_ = 0); + Address(const Poco::Util::AbstractConfiguration & config, const String & config_prefix, UInt32 shard_index_ = 0, UInt32 replica_index_ = 0); Address(const String & host_port_, const String & user_, const String & password_, UInt16 clickhouse_port, bool secure_ = false); /// Returns 'escaped_host_name:port' String toString() const; @@ -82,8 +82,10 @@ public: static std::pair fromString(const String & host_port_string); - /// Retrurns escaped shard{shard_number}_replica{replica_number}@resolved_host_address:resolved_host_port#default_database + /// Returns escaped shard{shard_index}_replica{replica_index} String toFullString() const; + + /// Returns address with only shard index and replica index or full address without shard index and replica index static Address fromFullString(const String & address_full_string); /// Returns resolved address if it does resolve. From d05642575d50d9a4ca6d904b03e4110976e7b728 Mon Sep 17 00:00:00 2001 From: millb Date: Fri, 31 Jan 2020 14:16:46 +0300 Subject: [PATCH 0149/2007] fixed --- dbms/src/Interpreters/Cluster.cpp | 6 ++-- .../Storages/Distributed/DirectoryMonitor.cpp | 36 +++++++++++++++---- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index d390ce8cbaa..a972fe294d5 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -138,8 +138,8 @@ std::pair Cluster::Address::fromString(const String & host_port_ String Cluster::Address::toFullString() const { return - ((shard_number == 0) ? "" : "shard" + std::to_string(shard_number)) + - ((replica_number == 0) ? "" : "_replica" + std::to_string(replica_number)); + ((shard_index == 0) ? "" : "shard" + std::to_string(shard_index)) + + ((replica_index == 0) ? "" : "_replica" + std::to_string(replica_index)); } Cluster::Address Cluster::Address::fromFullString(const String & full_string) @@ -157,7 +157,7 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string) const char * user_pw_end = strchr(full_string.data(), '@'); - /// parsing with format [shard{shard_number}[_replica{replica_number}]] + /// parsing with format [shard{shard_index}[_replica{replica_index}]] if (!user_pw_end && startsWith("shard", full_string)) { const char * underscore = strchr(full_string.data(), '_'); diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index ea9a659314b..5d4c8d36254 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -188,21 +188,45 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri { const auto & cluster = storage.getCluster(); const auto & shards_info = cluster->getShardsInfo(); - // const auto & shards_addresses = cluster->getShardsInfo(); + const auto & shards_addresses = cluster->getShardsAddresses(); + + /// check format shard{shard_index}_number{number_index} + if (address.shard_index != 0) + { + return shards_info[address.shard_index].per_replica_pools[address.replica_index]; + } /// existing connections pool have a higher priority - return shards_info[address.shard_number].per_replica_pools[address.replica_number]; + for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) + { + const Cluster::Addresses & replicas_addresses = shards_addresses[shard_index]; - /*return std::make_shared( - 1, address.host_name, address.port, address.default_database, address.user, address.password, - storage.getName() + '_' + address.user, Protocol::Compression::Enable, address.secure);*/ + for (size_t replica_index = 0; replica_index < replicas_addresses.size(); ++replica_index) + { + const Cluster::Address & replica_address = replicas_addresses[replica_index]; + + if (address.user == replica_address.user && + address.password == replica_address.password && + address.host_name == replica_address.host_name && + address.port == replica_address.port && + address.default_database == replica_address.default_database && + address.secure == replica_address.secure) + { + return shards_info[shard_index].per_replica_pools[replica_index]; + } + } + } + + return std::make_shared( + 1, address.host_name, address.port, address.default_database, address.user, address.password, + storage.getName() + '_' + address.user, Protocol::Compression::Enable, address.secure); }; auto pools = createPoolsForAddresses(name, pool_factory); const auto settings = storage.global_context.getSettings(); return pools.size() == 1 ? pools.front() : std::make_shared(pools, LoadBalancing::RANDOM, - settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); + settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); } From 5c5b76d2b083194916eb1ffa5182640e187e6b04 Mon Sep 17 00:00:00 2001 From: Mikhail Korotov <55493615+millb@users.noreply.github.com> Date: Fri, 31 Jan 2020 14:28:39 +0300 Subject: [PATCH 0150/2007] Update DirectoryMonitor.cpp --- dbms/src/Storages/Distributed/DirectoryMonitor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 5d4c8d36254..2ef0d64e73c 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -226,7 +226,7 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri const auto settings = storage.global_context.getSettings(); return pools.size() == 1 ? pools.front() : std::make_shared(pools, LoadBalancing::RANDOM, - settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); + settings.distributed_replica_error_half_life.totalSeconds(), settings.distributed_replica_error_cap); } From 14acdffb0e8432f0164a3e5b2b7dfb505e1bbf0f Mon Sep 17 00:00:00 2001 From: Mikhail Korotov <55493615+millb@users.noreply.github.com> Date: Fri, 31 Jan 2020 14:29:21 +0300 Subject: [PATCH 0151/2007] Update DirectoryMonitor.cpp --- dbms/src/Storages/Distributed/DirectoryMonitor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 2ef0d64e73c..a9ebdb02076 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -218,8 +218,8 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri } return std::make_shared( - 1, address.host_name, address.port, address.default_database, address.user, address.password, - storage.getName() + '_' + address.user, Protocol::Compression::Enable, address.secure); + 1, address.host_name, address.port, address.default_database, address.user, address.password, + storage.getName() + '_' + address.user, Protocol::Compression::Enable, address.secure); }; auto pools = createPoolsForAddresses(name, pool_factory); From 01ff1c65e2b6c5ec45bc2f9f4a0ec4c6fd759bc1 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 31 Jan 2020 15:25:31 +0300 Subject: [PATCH 0152/2007] More flexible schema --- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 7 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 42 +++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 10 +- .../Storages/StorageReplicatedMergeTree.cpp | 166 ++++++++++++------ .../src/Storages/StorageReplicatedMergeTree.h | 7 +- 5 files changed, 153 insertions(+), 79 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index b54ae11c2f0..068fbc5c344 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -66,7 +66,6 @@ struct ReplicatedMergeTreeLogEntryData void readText(ReadBuffer & in); String toString() const; - /// log-xxx String znode_name; Type type = EMPTY; @@ -108,10 +107,8 @@ struct ReplicatedMergeTreeLogEntryData std::shared_ptr replace_range_entry; - /// Should alter be processed sychronously, or asynchronously. - size_t alter_sync_mode; - /// Mutation commands for alter if any. - String mutation_commands; + //TODO(alesap) + int alter_version; String columns_str; String metadata_str; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index f9622009325..8903f525e94 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -151,7 +151,12 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } if (entry->type == LogEntry::ALTER_METADATA) { - alter_znodes_in_queue.push_back(entry->znode_name); + alter_sequence.addMetadataAlter(entry->alter_version); + } + + if (entry->type == LogEntry::MUTATE_PART && entry->alter_version != -1) + { + alter_sequence.addDataAlterIfEmpty(entry->alter_version); } } @@ -226,12 +231,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (entry->type == LogEntry::ALTER_METADATA) { - if (alter_znodes_in_queue.front() != entry->znode_name) - { - /// TODO(alesap) Better - throw Exception("Processed incorrect alter.", ErrorCodes::LOGICAL_ERROR); - } - alter_znodes_in_queue.pop_front(); + alter_sequence.finishMetadataAlter(entry->alter_version); } } else @@ -1025,7 +1025,12 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (entry.type == LogEntry::ALTER_METADATA) { - return entry.znode_name == alter_znodes_in_queue.front(); + return alter_sequence.canExecuteMetadataAlter(entry.alter_version); + } + + if (entry.type == LogEntry::MUTATE_PART && entry.alter_version != -1) + { + return alter_sequence.canExecuteDataAlter(entry.alter_version); } return true; @@ -1304,6 +1309,8 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep { LOG_TRACE(log, "Marking mutation " << znode << " done because it is <= mutation_pointer (" << mutation_pointer << ")"); mutation.is_done = true; + if (mutation.entry->alter_version != -1) + alter_sequence.finishDataAlter(mutation.entry->alter_version); } else if (mutation.parts_to_do == 0) { @@ -1340,6 +1347,8 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep { LOG_TRACE(log, "Mutation " << entry->znode_name << " is done"); it->second.is_done = true; + if (entry->alter_version != -1) + alter_sequence.finishDataAlter(entry->alter_version); } } } @@ -1703,7 +1712,7 @@ bool ReplicatedMergeTreeMergePredicate::operator()( } -std::optional ReplicatedMergeTreeMergePredicate::getDesiredMutationVersion(const MergeTreeData::DataPartPtr & part) const +std::optional> ReplicatedMergeTreeMergePredicate::getDesiredMutationVersion(const MergeTreeData::DataPartPtr & part) const { /// Assigning mutations is easier than assigning merges because mutations appear in the same order as /// the order of their version numbers (see StorageReplicatedMergeTree::mutate). @@ -1726,11 +1735,22 @@ std::optional ReplicatedMergeTreeMergePredicate::getDesiredMutationVersio return {}; Int64 current_version = queue.getCurrentMutationVersionImpl(part->info.partition_id, part->info.getDataVersion(), lock); - Int64 max_version = in_partition->second.rbegin()->first; + Int64 max_version = in_partition->second.begin()->first; + + int alter_version = -1; + for (auto [mutation_version, mutation_status] : in_partition->second) + { + max_version = mutation_version; + if (mutations_status->entry->alter_version != -1) + { + alter_version = mutations_status->entry->alter_version; + break; + } + } if (current_version >= max_version) return {}; - return max_version; + return std::make_pair(max_version, alter_version); } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 299b4ed1c48..0506ab76c05 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -126,10 +127,10 @@ private: /// Znode ID of the latest mutation that is done. String mutation_pointer; - /// Provides only one simultaneous call to pullLogsToQueue. std::mutex pull_logs_to_queue_mutex; + AlterSequence alter_sequence; /// List of subscribers /// A subscriber callback is called when an entry queue is deleted @@ -374,6 +375,11 @@ public: void getInsertTimes(time_t & out_min_unprocessed_insert_time, time_t & out_max_processed_insert_time) const; std::vector getMutationsStatus() const; + + String getLatestMutation() const + { + return mutations_by_znode.rbegin()->first; + } }; class ReplicatedMergeTreeMergePredicate @@ -390,7 +396,7 @@ public: /// Return nonempty optional if the part can and should be mutated. /// Returned mutation version number is always the biggest possible. - std::optional getDesiredMutationVersion(const MergeTreeData::DataPartPtr & part) const; + std::optional> getDesiredMutationVersion(const MergeTreeData::DataPartPtr & part) const; bool isMutationFinished(const ReplicatedMergeTreeMutationEntry & mutation) const; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 7e100dda03c..79da80cfa84 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -404,6 +404,8 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() /// ALTERs of the metadata node. zookeeper->createIfNotExists(replica_path + "/metadata", String()); + + zookeeper->createIfNotExists(zookeeper_path + "/alters", String()); } @@ -469,6 +471,7 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo auto columns_from_zk = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &columns_stat)); //columns_version = columns_stat.version; + /// TODO(alesap) remove this trash const ColumnsDescription & old_columns = getColumns(); if (columns_from_zk != old_columns || !metadata_diff.empty()) { @@ -2296,11 +2299,11 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (part->bytes_on_disk > max_source_part_size_for_mutation) continue; - std::optional desired_mutation_version = merge_pred.getDesiredMutationVersion(part); + std::optional> desired_mutation_version = merge_pred.getDesiredMutationVersion(part); if (!desired_mutation_version) continue; - if (createLogEntryToMutatePart(*part, *desired_mutation_version)) + if (createLogEntryToMutatePart(*part, desired_mutation_version->first, desired_mutation_version->second)) { success = true; break; @@ -2401,7 +2404,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( } -bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const MergeTreeDataPart & part, Int64 mutation_version) +bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const MergeTreeDataPart & part, Int64 mutation_version, int alter_version) { auto zookeeper = getZooKeeper(); @@ -2431,6 +2434,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMutatePart(const MergeTreeDataP entry.source_parts.push_back(part.name); entry.new_part_name = new_part_name; entry.create_time = time(nullptr); + entry.alter_version = alter_version; zookeeper->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); return true; @@ -3245,18 +3249,57 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer std::cerr << "Nodes in zk updated\n"; } + if (!entry.mutation_commands.empty()) { + MutationCommands commands; ReadBufferFromString in(entry.mutation_commands); commands.readText(in); - if (is_leader) + + String mutation_znode; + + while (true) { - auto mutation_entry = mutateImpl(commands); - waitMutation(mutation_entry, entry.alter_sync_mode); + Coordination::Requests requests; + requests.emplace_back(zkutil::makeCreateRequest( + zookeeper_path + "/alters" + entry.znode_name, String(), zkutil::CreateMode::PersistentSequential)); + + prepareMutationEntry(zookeeper, commands, requests); + + std::cerr << replica_name << " - " << "REqeusts size:" << requests.size() << std::endl; + Coordination::Responses responses; + int32_t rc = zookeeper->tryMulti(requests, responses); + + if (rc == Coordination::ZOK) + { + const String & path_created = dynamic_cast(responses.back().get())->path_created; + mutation_znode = path_created.substr(path_created.find_last_of('/') + 1); + std::cerr << "Created mutation\n"; + LOG_TRACE(log, "Created mutation with ID " << mutation_znode); + break; + } + else if (rc == Coordination::ZNODEEXISTS) + { + queue.updateMutations(zookeeper); + + mutation_znode = queue.getLatestMutation(); + + std::cerr << replica_name << " - " << "Found mutation in queue:" << mutation_znode; + LOG_TRACE(log, "Already have mutation with ID " << mutation_znode); + break; + } + else if (rc != Coordination::ZBADVERSION) + { + throw Coordination::Exception("Cannot create mutation for alter", rc); + } + std::cerr << "LOOOPING\n"; } + + waitMutation(mutation_znode, entry.alter_sync_mode); } + return true; } @@ -3296,6 +3339,8 @@ void StorageReplicatedMergeTree::alter( return queryToString(query); }; + auto zookeeper = getZooKeeper(); + //std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; ReplicatedMergeTreeLogEntryData entry; @@ -3355,6 +3400,11 @@ void StorageReplicatedMergeTree::alter( ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + if (!entry.mutation_commands.empty()) + { + prepareMutationEntry(zookeeper, entry.mutation_commands, ops); + } + Coordination::Responses results = getZooKeeper()->multi(ops); String path_created = dynamic_cast(*results.back()).path_created; @@ -4338,11 +4388,15 @@ void StorageReplicatedMergeTree::fetchPartition(const ASTPtr & partition, const void StorageReplicatedMergeTree::mutate(const MutationCommands & commands, const Context & query_context) { - auto entry = mutateImpl(commands); - waitMutation(entry, query_context.getSettingsRef().mutations_sync); + auto zookeeper = getZooKeeper(); + Coordination::Requests requests; + + ReplicatedMergeTreeMutationEntry entry = prepareMutationEntry(zookeeper, commands, requests); + mutateImpl(zookeeper, requests, entry); + waitMutation(entry.znode_name, query_context.getSettingsRef().mutations_sync); } -void StorageReplicatedMergeTree::waitMutation(const ReplicatedMergeTreeMutationEntry & entry, size_t mutations_sync) const +void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t mutations_sync) const { auto zookeeper = getZooKeeper(); /// we have to wait @@ -4354,11 +4408,39 @@ void StorageReplicatedMergeTree::waitMutation(const ReplicatedMergeTreeMutationE else if (mutations_sync == 1) /// just wait for ourself replicas.push_back(replica_name); - waitMutationToFinishOnReplicas(replicas, entry.znode_name); + waitMutationToFinishOnReplicas(replicas, znode_name); } } -ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const MutationCommands & commands) + +ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::prepareMutationEntry( + zkutil::ZooKeeperPtr zookeeper, const MutationCommands & commands, Coordination::Requests & requests) const +{ + String mutations_path = zookeeper_path + "/mutations"; + + ReplicatedMergeTreeMutationEntry entry; + entry.source_replica = replica_name; + entry.commands = commands; + Coordination::Stat mutations_stat; + zookeeper->get(mutations_path, &mutations_stat); + + EphemeralLocksInAllPartitions block_number_locks(zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + + for (const auto & lock : block_number_locks.getLocks()) + entry.block_numbers[lock.partition_id] = lock.number; + + entry.create_time = time(nullptr); + + requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); + requests.emplace_back(zkutil::makeCreateRequest(mutations_path + "/", entry.toString(), zkutil::CreateMode::PersistentSequential)); + + return entry; +} + +void StorageReplicatedMergeTree::mutateImpl( + zkutil::ZooKeeperPtr zookeeper, + const Coordination::Requests & requests, + ReplicatedMergeTreeMutationEntry & entry) { /// Overview of the mutation algorithm. /// @@ -4413,57 +4495,25 @@ ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::mutateImpl(const Mu /// After all needed parts are mutated (i.e. all active parts have the mutation version greater than /// the version of this mutation), the mutation is considered done and can be deleted. - ReplicatedMergeTreeMutationEntry entry; - entry.source_replica = replica_name; - entry.commands = commands; - String mutations_path = zookeeper_path + "/mutations"; + Coordination::Responses responses; + int32_t rc = zookeeper->tryMulti(requests, responses); - /// Update the mutations_path node when creating the mutation and check its version to ensure that - /// nodes for mutations are created in the same order as the corresponding block numbers. - /// Should work well if the number of concurrent mutation requests is small. - while (true) + if (rc == Coordination::ZOK) { - auto zookeeper = getZooKeeper(); - - Coordination::Stat mutations_stat; - zookeeper->get(mutations_path, &mutations_stat); - - EphemeralLocksInAllPartitions block_number_locks( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); - - for (const auto & lock : block_number_locks.getLocks()) - entry.block_numbers[lock.partition_id] = lock.number; - - entry.create_time = time(nullptr); - - Coordination::Requests requests; - requests.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); - requests.emplace_back(zkutil::makeCreateRequest( - mutations_path + "/", entry.toString(), zkutil::CreateMode::PersistentSequential)); - - Coordination::Responses responses; - int32_t rc = zookeeper->tryMulti(requests, responses); - - if (rc == Coordination::ZOK) - { - const String & path_created = - dynamic_cast(responses[1].get())->path_created; - entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); - LOG_TRACE(log, "Created mutation with ID " << entry.znode_name); - break; - } - else if (rc == Coordination::ZBADVERSION) - { - LOG_TRACE(log, "Version conflict when trying to create a mutation node, retrying..."); - continue; - } - else - throw Coordination::Exception("Unable to create a mutation znode", rc); + const String & path_created = + dynamic_cast(responses[1].get())->path_created; + entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); + LOG_TRACE(log, "Created mutation with ID " << entry.znode_name); } - - - return entry; + else if (rc == Coordination::ZBADVERSION) + /// This error mean, that parallel mutation is created in mutations queue (/mutations) right now. + /// (NOTE: concurrent mutations execution is OK, but here we have case with concurrent mutation intention from client) + /// We can retry this error by ourself, but mutations is not designed for highly concurrent execution. + /// So, if client sure that he do what he want, than he should retry. + throw Coordination::Exception("Parallel mutation is creating right now. Client should retry.", rc); + else + throw Coordination::Exception("Unable to create a mutation znode", rc); } std::vector StorageReplicatedMergeTree::getMutationsStatus() const diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 228702b2a5f..488c60ed7c1 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -109,8 +109,9 @@ public: void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; void mutate(const MutationCommands & commands, const Context & context) override; - ReplicatedMergeTreeMutationEntry mutateImpl(const MutationCommands & commands); - void waitMutation(const ReplicatedMergeTreeMutationEntry & entry, size_t mutation_sync) const; + ReplicatedMergeTreeMutationEntry prepareMutationEntry(zkutil::ZooKeeperPtr zk, const MutationCommands & commands, Coordination::Requests & requests) const; + void mutateImpl(zkutil::ZooKeeperPtr zookeeper, const Coordination::Requests & requests, ReplicatedMergeTreeMutationEntry & entry); + void waitMutation(const String & znode_name, size_t mutation_sync) const; std::vector getMutationsStatus() const override; CancellationCode killMutation(const String & mutation_id) override; @@ -438,7 +439,7 @@ private: bool force_ttl, ReplicatedMergeTreeLogEntryData * out_log_entry = nullptr); - bool createLogEntryToMutatePart(const MergeTreeDataPart & part, Int64 mutation_version); + bool createLogEntryToMutatePart(const MergeTreeDataPart & part, Int64 mutation_version, int alter_version); /// Exchange parts. From 2d1f06a49f91c0bcb9575df0d2a524164bb86baf Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 16:12:11 +0300 Subject: [PATCH 0153/2007] Processors support for StorageFile reading. --- dbms/src/Storages/StorageFile.cpp | 216 +++++++++++++++++++----------- dbms/src/Storages/StorageFile.h | 6 +- 2 files changed, 142 insertions(+), 80 deletions(-) diff --git a/dbms/src/Storages/StorageFile.cpp b/dbms/src/Storages/StorageFile.cpp index 5e7177b3e25..334567d6c26 100644 --- a/dbms/src/Storages/StorageFile.cpp +++ b/dbms/src/Storages/StorageFile.cpp @@ -33,6 +33,8 @@ #include #include #include +#include +#include namespace fs = std::filesystem; @@ -200,16 +202,47 @@ StorageFile::StorageFile(CommonArguments args) setConstraints(args.constraints); } -class StorageFileBlockInputStream : public IBlockInputStream +class StorageFileSource : public SourceWithProgress { public: - StorageFileBlockInputStream(std::shared_ptr storage_, - const Context & context_, UInt64 max_block_size_, - std::string file_path_, bool need_path, bool need_file, - const CompressionMethod compression_method_, - BlockInputStreamPtr prepared_reader = nullptr) - : storage(std::move(storage_)), reader(std::move(prepared_reader)), - context(context_), max_block_size(max_block_size_), compression_method(compression_method_) + struct FilesInfo + { + std::vector files; + + std::atomic next_file_to_read = 0; + + bool need_path_column = false; + bool need_file_column = false; + }; + + using FilesInfoPtr = std::shared_ptr; + + static Block getHeader(StorageFile & storage, bool need_path_column, bool need_file_column) + { + auto header = storage.getSampleBlock(); + + /// Note: AddingDefaultsBlockInputStream doesn't change header. + + if (need_path_column) + header.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + if (need_file_column) + header.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + + return header; + } + + StorageFileSource( + std::shared_ptr storage_, + const Context & context_, + UInt64 max_block_size_, + FilesInfoPtr files_info_, + ColumnDefaults column_defaults_) + : SourceWithProgress(getHeader(*storage_, files_info_->need_path_column, files_info_->need_file_column)) + , storage(std::move(storage_)) + , files_info(std::move(files_info_)) + , column_defaults(std::move(column_defaults_)) + , context(context_) + , max_block_size(max_block_size_) { if (storage->use_table_fd) { @@ -233,9 +266,6 @@ public: else { shared_lock = std::shared_lock(storage->rwlock); - file_path = std::make_optional(file_path_); - with_file_column = need_file; - with_path_column = need_path; } } @@ -244,82 +274,111 @@ public: return storage->getName(); } - Block readImpl() override + Chunk generate() override { - /// Open file lazily on first read. This is needed to avoid too many open files from different streams. - if (!reader) + while (!finished_generate) { - read_buf = wrapReadBufferWithCompressionMethod(storage->use_table_fd - ? std::make_unique(storage->table_fd) - : std::make_unique(file_path.value()), - compression_method); - - reader = FormatFactory::instance().getInput(storage->format_name, *read_buf, storage->getSampleBlock(), context, max_block_size); - reader->readPrefix(); - } - - auto res = reader->read(); - - /// Enrich with virtual columns. - if (res && file_path) - { - if (with_path_column) - res.insert({DataTypeString().createColumnConst(res.rows(), file_path.value())->convertToFullColumnIfConst(), - std::make_shared(), "_path"}); /// construction with const is for probably generating less code - if (with_file_column) + /// Open file lazily on first read. This is needed to avoid too many open files from different streams. + if (!reader) { - size_t last_slash_pos = file_path.value().find_last_of('/'); - res.insert({DataTypeString().createColumnConst(res.rows(), file_path.value().substr( - last_slash_pos + 1))->convertToFullColumnIfConst(), - std::make_shared(), "_file"}); - } - } + if (!storage->use_table_fd) + { + auto current_file = files_info->next_file_to_read.fetch_add(1); + if (current_file >= files_info->files.size()) + return {}; - /// Close file prematurally if stream was ended. - if (!res) - { + current_path = files_info->files[current_file]; + + /// Special case for distributed format. Defaults are not needed here. + if (storage->format_name == "Distributed") + { + reader = StorageDistributedDirectoryMonitor::createStreamFromFile(current_path); + continue; + } + } + + std::unique_ptr nested_buffer; + CompressionMethod method; + + if (storage->use_table_fd) + { + nested_buffer = std::make_unique(storage->table_fd); + method = chooseCompressionMethod("", storage->compression_method); + } + else + { + nested_buffer = std::make_unique(current_path); + method = chooseCompressionMethod(current_path, storage->compression_method); + } + + read_buf = wrapReadBufferWithCompressionMethod(std::move(nested_buffer), method); + reader = FormatFactory::instance().getInput( + storage->format_name, *read_buf, storage->getSampleBlock(), context, max_block_size); + + if (!column_defaults.empty()) + reader = std::make_shared(reader, column_defaults, context); + + reader->readPrefix(); + } + + if (auto res = reader->read()) + { + Columns columns = res.getColumns(); + UInt64 num_rows = res.rows(); + + /// Enrich with virtual columns. + if (files_info->need_path_column) + { + auto column = DataTypeString().createColumnConst(num_rows, current_path); + columns.push_back(column->convertToFullColumnIfConst()); + } + + if (files_info->need_file_column) + { + size_t last_slash_pos = current_path.find_last_of('/'); + auto file_name = current_path.substr(last_slash_pos + 1); + + auto column = DataTypeString().createColumnConst(num_rows, std::move(file_name)); + columns.push_back(column->convertToFullColumnIfConst()); + } + + return Chunk(std::move(columns), num_rows); + } + + /// Read only once for file descriptor. + if (storage->use_table_fd) + finished_generate = true; + + /// Close file prematurely if stream was ended. reader->readSuffix(); reader.reset(); read_buf.reset(); } - return res; - } - - Block getHeader() const override - { - auto res = storage->getSampleBlock(); - - if (res && file_path) - { - if (with_path_column) - res.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); - if (with_file_column) - res.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); - } - - return res; + return {}; } private: std::shared_ptr storage; - std::optional file_path; - bool with_path_column = false; - bool with_file_column = false; + FilesInfoPtr files_info; + String current_path; Block sample_block; std::unique_ptr read_buf; BlockInputStreamPtr reader; + ColumnDefaults column_defaults; + const Context & context; /// TODO Untangle potential issues with context lifetime. UInt64 max_block_size; - const CompressionMethod compression_method; + + bool finished_generate = false; std::shared_lock shared_lock; std::unique_lock unique_lock; }; -BlockInputStreams StorageFile::read( +Pipes StorageFile::readWithProcessors( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, @@ -337,29 +396,30 @@ BlockInputStreams StorageFile::read( if (paths.size() == 1 && !Poco::File(paths[0]).exists()) throw Exception("File " + paths[0] + " doesn't exist", ErrorCodes::FILE_DOESNT_EXIST); - blocks_input.reserve(paths.size()); - bool need_path_column = false; - bool need_file_column = false; + + auto files_info = std::make_shared(); + files_info->files = paths; + for (const auto & column : column_names) { if (column == "_path") - need_path_column = true; + files_info->need_path_column = true; if (column == "_file") - need_file_column = true; + files_info->need_file_column = true; } - for (const auto & file_path : paths) - { - BlockInputStreamPtr cur_block; - if (format_name == "Distributed") - cur_block = StorageDistributedDirectoryMonitor::createStreamFromFile(file_path); - else - cur_block = std::make_shared( - std::static_pointer_cast(shared_from_this()), context, max_block_size, file_path, need_path_column, need_file_column, chooseCompressionMethod(file_path, compression_method)); + auto this_ptr = std::static_pointer_cast(shared_from_this()); - blocks_input.push_back(column_defaults.empty() ? cur_block : std::make_shared(cur_block, column_defaults, context)); - } - return narrowBlockInputStreams(blocks_input, num_streams); + if (num_streams < paths.size()) + num_streams = paths.size(); + + Pipes pipes; + pipes.reserve(num_streams); + + for (size_t i = 0; i < num_streams; ++i) + pipes.emplace_back(std::make_shared(this_ptr, context, max_block_size, files_info, column_defaults)); + + return pipes; } diff --git a/dbms/src/Storages/StorageFile.h b/dbms/src/Storages/StorageFile.h index 46c3f787db7..f6edc98b23e 100644 --- a/dbms/src/Storages/StorageFile.h +++ b/dbms/src/Storages/StorageFile.h @@ -24,7 +24,7 @@ class StorageFile : public ext::shared_ptr_helper, public IStorage public: std::string getName() const override { return "File"; } - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -32,6 +32,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + BlockOutputStreamPtr write( const ASTPtr & query, const Context & context) override; @@ -53,7 +55,7 @@ public: }; protected: - friend class StorageFileBlockInputStream; + friend class StorageFileSource; friend class StorageFileBlockOutputStream; /// From file descriptor From 687013271304d6fb8aafa20d71632cdb944b0856 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 17:06:43 +0300 Subject: [PATCH 0154/2007] Processors support for StorageHDFS reading. --- dbms/src/Storages/StorageHDFS.cpp | 171 ++++++++++++++++++------------ dbms/src/Storages/StorageHDFS.h | 4 +- 2 files changed, 108 insertions(+), 67 deletions(-) diff --git a/dbms/src/Storages/StorageHDFS.cpp b/dbms/src/Storages/StorageHDFS.cpp index 391119c3a30..3b424b1a6db 100644 --- a/dbms/src/Storages/StorageHDFS.cpp +++ b/dbms/src/Storages/StorageHDFS.cpp @@ -25,6 +25,8 @@ #include #include #include +#include +#include namespace DB { @@ -63,24 +65,46 @@ StorageHDFS::StorageHDFS(const String & uri_, namespace { -class HDFSBlockInputStream : public IBlockInputStream +class HDFSSource : public SourceWithProgress { public: - HDFSBlockInputStream(const String & uri, - bool need_path, - bool need_file, - const String & format, - const Block & sample_block, - const Context & context, - UInt64 max_block_size, - const CompressionMethod compression_method) + struct SourcesInfo + { + std::vector uris; + + std::atomic next_uri_to_read = 0; + + bool need_path_column = false; + bool need_file_column = false; + }; + + using SourcesInfoPtr = std::shared_ptr; + + static Block getHeader(Block header, bool need_path_column, bool need_file_column) + { + if (need_path_column) + header.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + if (need_file_column) + header.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + + return header; + } + + HDFSSource( + SourcesInfoPtr source_info_, + String uri_, + String format_, + Block sample_block_, + const Context & context_, + UInt64 max_block_size_) + : SourceWithProgress(getHeader(sample_block_, source_info_->need_path_column, source_info_->need_file_column)) + , source_info(std::move(source_info_)) + , uri(std::move(uri_)) + , format(std::move(format_)) + , max_block_size(max_block_size_) + , sample_block(std::move(sample_block_)) + , context(context_) { - auto read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(uri), compression_method); - file_path = uri; - with_file_column = need_file; - with_path_column = need_path; - auto input_stream = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); - reader = std::make_shared>(input_stream, std::move(read_buf)); } String getName() const override @@ -88,53 +112,65 @@ public: return "HDFS"; } - Block readImpl() override + Chunk generate() override { - auto res = reader->read(); - if (res) + while (true) { - if (with_path_column) - res.insert({DataTypeString().createColumnConst(res.rows(), file_path)->convertToFullColumnIfConst(), std::make_shared(), - "_path"}); /// construction with const is for probably generating less code - if (with_file_column) + if (!reader) { - size_t last_slash_pos = file_path.find_last_of('/'); - res.insert({DataTypeString().createColumnConst(res.rows(), file_path.substr( - last_slash_pos + 1))->convertToFullColumnIfConst(), std::make_shared(), - "_file"}); + auto pos = source_info->next_uri_to_read.fetch_add(1); + if (pos >= source_info->uris.size()) + return {}; + + auto path = source_info->uris[pos]; + current_path = uri + path; + + auto compression = chooseCompressionMethod(path, format); + auto read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(current_path), compression); + auto input_stream = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + + reader = std::make_shared>(input_stream, std::move(read_buf)); + reader->readPrefix(); } + + if (auto res = reader->read()) + { + Columns columns = res.getColumns(); + UInt64 num_rows = res.rows(); + + /// Enrich with virtual columns. + if (source_info->need_path_column) + { + auto column = DataTypeString().createColumnConst(num_rows, current_path); + columns.push_back(column->convertToFullColumnIfConst()); + } + + if (source_info->need_file_column) + { + size_t last_slash_pos = current_path.find_last_of('/'); + auto file_name = current_path.substr(last_slash_pos + 1); + + auto column = DataTypeString().createColumnConst(num_rows, std::move(file_name)); + columns.push_back(column->convertToFullColumnIfConst()); + } + + return Chunk(std::move(columns), num_rows); + } + + reader->readSuffix(); } - return res; - } - - Block getHeader() const override - { - auto res = reader->getHeader(); - if (res) - { - if (with_path_column) - res.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); - if (with_file_column) - res.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); - } - return res; - } - - void readPrefixImpl() override - { - reader->readPrefix(); - } - - void readSuffixImpl() override - { - reader->readSuffix(); } private: BlockInputStreamPtr reader; - String file_path; - bool with_path_column = false; - bool with_file_column = false; + SourcesInfoPtr source_info; + String uri; + String format; + String current_path; + + UInt64 max_block_size; + Block sample_block; + const Context & context; }; class HDFSBlockOutputStream : public IBlockOutputStream @@ -228,7 +264,7 @@ Strings LSWithRegexpMatching(const String & path_for_ls, const HDFSFSPtr & fs, c } -BlockInputStreams StorageHDFS::read( +Pipes StorageHDFS::readWithProcessors( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context_, @@ -243,24 +279,27 @@ BlockInputStreams StorageHDFS::read( HDFSBuilderPtr builder = createHDFSBuilder(uri_without_path + "/"); HDFSFSPtr fs = createHDFSFS(builder.get()); - const Strings res_paths = LSWithRegexpMatching("/", fs, path_from_uri); - BlockInputStreams result; - bool need_path_column = false; - bool need_file_column = false; + auto sources_info = std::make_shared(); + sources_info->uris = LSWithRegexpMatching("/", fs, path_from_uri); + for (const auto & column : column_names) { if (column == "_path") - need_path_column = true; + sources_info->need_path_column = true; if (column == "_file") - need_file_column = true; - } - for (const auto & res_path : res_paths) - { - result.push_back(std::make_shared(uri_without_path + res_path, need_path_column, need_file_column, format_name, getSampleBlock(), context_, - max_block_size, chooseCompressionMethod(res_path, compression_method))); + sources_info->need_file_column = true; } - return narrowBlockInputStreams(result, num_streams); + if (num_streams > sources_info->uris.size()) + num_streams = sources_info->uris.size(); + + Pipes pipes; + + for (size_t i = 0; i < num_streams; ++i) + pipes.emplace_back(std::make_shared( + sources_info, uri_without_path, format_name, getSampleBlock(), context_, max_block_size)); + + return pipes; } BlockOutputStreamPtr StorageHDFS::write(const ASTPtr & /*query*/, const Context & /*context*/) diff --git a/dbms/src/Storages/StorageHDFS.h b/dbms/src/Storages/StorageHDFS.h index 42f1891597e..c31589b5edb 100644 --- a/dbms/src/Storages/StorageHDFS.h +++ b/dbms/src/Storages/StorageHDFS.h @@ -19,13 +19,15 @@ class StorageHDFS : public ext::shared_ptr_helper, public IStorage public: String getName() const override { return "HDFS"; } - BlockInputStreams read(const Names & column_names, + Pipes readWithProcessors(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum processed_stage, size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; protected: From f9db37ebf77536c5e95b37a38c057741b61b449b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 17:26:51 +0300 Subject: [PATCH 0155/2007] Processors support for StorageInput reading. --- dbms/src/Storages/StorageInput.cpp | 34 ++++++++++++++++++------------ dbms/src/Storages/StorageInput.h | 4 +++- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/dbms/src/Storages/StorageInput.cpp b/dbms/src/Storages/StorageInput.cpp index 68b9dfe8810..aaab0513c0f 100644 --- a/dbms/src/Storages/StorageInput.cpp +++ b/dbms/src/Storages/StorageInput.cpp @@ -5,6 +5,9 @@ #include #include +#include +#include +#include namespace DB @@ -22,26 +25,28 @@ StorageInput::StorageInput(const String & table_name_, const ColumnsDescription } -class StorageInputBlockInputStream : public IBlockInputStream +class StorageInputSource : public SourceWithProgress { public: - StorageInputBlockInputStream(Context & context_, const Block sample_block_) - : context(context_), - sample_block(sample_block_) + StorageInputSource(Context & context_, Block sample_block) + : SourceWithProgress(std::move(sample_block)), context(context_) { } - Block readImpl() override { return context.getInputBlocksReaderCallback()(context); } - void readPrefix() override {} - void readSuffix() override {} + Chunk generate() override + { + auto block = context.getInputBlocksReaderCallback()(context); + if (!block) + return {}; + + UInt64 num_rows = block.rows(); + return Chunk(block.getColumns(), num_rows); + } String getName() const override { return "Input"; } - Block getHeader() const override { return sample_block; } - private: Context & context; - const Block sample_block; }; @@ -51,26 +56,29 @@ void StorageInput::setInputStream(BlockInputStreamPtr input_stream_) } -BlockInputStreams StorageInput::read(const Names & /*column_names*/, +Pipes StorageInput::readWithProcessors(const Names & /*column_names*/, const SelectQueryInfo & /*query_info*/, const Context & context, QueryProcessingStage::Enum /*processed_stage*/, size_t /*max_block_size*/, unsigned /*num_streams*/) { + Pipes pipes; Context & query_context = const_cast(context).getQueryContext(); /// It is TCP request if we have callbacks for input(). if (query_context.getInputBlocksReaderCallback()) { /// Send structure to the client. query_context.initializeInput(shared_from_this()); - input_stream = std::make_shared(query_context, getSampleBlock()); + pipes.emplace_back(std::make_shared(query_context, getSampleBlock())); + return pipes; } if (!input_stream) throw Exception("Input stream is not initialized, input() must be used only in INSERT SELECT query", ErrorCodes::INVALID_USAGE_OF_INPUT); - return {input_stream}; + pipes.emplace_back(std::make_shared(input_stream)); + return pipes; } } diff --git a/dbms/src/Storages/StorageInput.h b/dbms/src/Storages/StorageInput.h index 706bb5bbfdf..608271bf92c 100644 --- a/dbms/src/Storages/StorageInput.h +++ b/dbms/src/Storages/StorageInput.h @@ -17,7 +17,7 @@ public: /// A table will read from this stream. void setInputStream(BlockInputStreamPtr input_stream_); - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -25,6 +25,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + private: BlockInputStreamPtr input_stream; From 384e68d745b81ebb9c29d0c4572aa951b9baa16f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 17:51:09 +0300 Subject: [PATCH 0156/2007] Processors support for StorageJoin reading. --- dbms/src/Interpreters/Join.h | 2 +- dbms/src/Storages/StorageJoin.cpp | 52 ++++++++++++++++++------------- dbms/src/Storages/StorageJoin.h | 2 +- 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/dbms/src/Interpreters/Join.h b/dbms/src/Interpreters/Join.h index 337c18a5980..ca52c15d289 100644 --- a/dbms/src/Interpreters/Join.h +++ b/dbms/src/Interpreters/Join.h @@ -322,7 +322,7 @@ public: private: friend class NonJoinedBlockInputStream; - friend class JoinBlockInputStream; + friend class JoinSource; std::shared_ptr table_join; ASTTableJoin::Kind kind; diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index f18859ae90a..2f664416b93 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -14,6 +14,8 @@ #include /// toLower #include +#include +#include namespace DB @@ -235,11 +237,15 @@ size_t rawSize(const StringRef & t) return t.size; } -class JoinBlockInputStream : public IBlockInputStream +class JoinSource : public SourceWithProgress { public: - JoinBlockInputStream(const Join & parent_, UInt64 max_block_size_, Block && sample_block_) - : parent(parent_), lock(parent.data->rwlock), max_block_size(max_block_size_), sample_block(std::move(sample_block_)) + JoinSource(const Join & parent_, UInt64 max_block_size_, Block sample_block_) + : SourceWithProgress(sample_block_) + , parent(parent_) + , lock(parent.data->rwlock) + , max_block_size(max_block_size_) + , sample_block(std::move(sample_block_)) { columns.resize(sample_block.columns()); column_indices.resize(sample_block.columns()); @@ -263,20 +269,17 @@ public: String getName() const override { return "Join"; } - Block getHeader() const override { return sample_block; } - - protected: - Block readImpl() override + Chunk generate() override { if (parent.data->blocks.empty()) - return Block(); + return {}; - Block block; + Chunk chunk; if (!joinDispatch(parent.kind, parent.strictness, parent.data->maps, - [&](auto kind, auto strictness, auto & map) { block = createBlock(map); })) + [&](auto kind, auto strictness, auto & map) { chunk = createChunk(map); })) throw Exception("Logical error: unknown JOIN strictness", ErrorCodes::LOGICAL_ERROR); - return block; + return chunk; } private: @@ -294,7 +297,7 @@ private: template - Block createBlock(const Maps & maps) + Chunk createChunk(const Maps & maps) { for (size_t i = 0; i < sample_block.columns(); ++i) { @@ -305,7 +308,7 @@ private: if (key_pos == i) { // unwrap null key column - ColumnNullable & nullable_col = assert_cast(*columns[i]); + auto & nullable_col = assert_cast(*columns[i]); columns[i] = nullable_col.getNestedColumnPtr()->assumeMutable(); } else @@ -333,22 +336,25 @@ private: if (!rows_added) return {}; - Block res = sample_block.cloneEmpty(); + Columns res_columns; + res_columns.reserve(columns.size()); + for (size_t i = 0; i < columns.size(); ++i) if (column_with_null[i]) { if (key_pos == i) - res.getByPosition(i).column = makeNullable(std::move(columns[i])); + res_columns.emplace_back(makeNullable(std::move(columns[i]))); else { - const ColumnNullable & nullable_col = assert_cast(*columns[i]); - res.getByPosition(i).column = nullable_col.getNestedColumnPtr(); + const auto & nullable_col = assert_cast(*columns[i]); + res_columns.emplace_back(makeNullable(nullable_col.getNestedColumnPtr())); } } else - res.getByPosition(i).column = std::move(columns[i]); + res_columns.emplace_back(std::move(columns[i])); - return res; + UInt64 num_rows = res_columns.at(0)->size(); + return Chunk(std::move(columns), num_rows); } template @@ -438,7 +444,7 @@ private: // TODO: multiple stream read and index read -BlockInputStreams StorageJoin::read( +Pipes StorageJoin::readWithProcessors( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & /*context*/, @@ -447,7 +453,11 @@ BlockInputStreams StorageJoin::read( unsigned /*num_streams*/) { check(column_names); - return {std::make_shared(*join, max_block_size, getSampleBlockForColumns(column_names))}; + + Pipes pipes; + pipes.emplace_back(std::make_shared(*join, max_block_size, getSampleBlockForColumns(column_names))); + + return pipes; } } diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index 22500a23baa..4815f53c7e6 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -36,7 +36,7 @@ public: /// Verify that the data structure is suitable for implementing this type of JOIN. void assertCompatible(ASTTableJoin::Kind kind_, ASTTableJoin::Strictness strictness_) const; - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, From 046ff34525d3156a05ae4c655bf07f54a5a0aeb5 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 18:10:10 +0300 Subject: [PATCH 0157/2007] Processors support for StorageLog reading. --- dbms/src/Storages/StorageJoin.h | 2 ++ dbms/src/Storages/StorageLog.cpp | 54 ++++++++++++++++++-------------- dbms/src/Storages/StorageLog.h | 6 ++-- 3 files changed, 36 insertions(+), 26 deletions(-) diff --git a/dbms/src/Storages/StorageJoin.h b/dbms/src/Storages/StorageJoin.h index 4815f53c7e6..4c6bc3b9bf2 100644 --- a/dbms/src/Storages/StorageJoin.h +++ b/dbms/src/Storages/StorageJoin.h @@ -44,6 +44,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + private: Block sample_block; const Names key_names; diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 44c3af28267..24d798e955a 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -20,6 +20,8 @@ #include #include +#include +#include #define DBMS_STORAGE_LOG_DATA_FILE_EXTENSION ".bin" @@ -41,13 +43,25 @@ namespace ErrorCodes } -class LogBlockInputStream final : public IBlockInputStream +class LogSource final : public SourceWithProgress { public: - LogBlockInputStream( + + static Block getHeader(const NamesAndTypesList & columns) + { + Block res; + + for (const auto & name_type : columns) + res.insert({ name_type.type->createColumn(), name_type.type, name_type.name }); + + return Nested::flatten(res); + } + + LogSource( size_t block_size_, const NamesAndTypesList & columns_, StorageLog & storage_, size_t mark_number_, size_t rows_limit_, size_t max_read_buffer_size_) - : block_size(block_size_), + : SourceWithProgress(getHeader(columns_)), + block_size(block_size_), columns(columns_), storage(storage_), mark_number(mark_number_), @@ -58,18 +72,8 @@ public: String getName() const override { return "Log"; } - Block getHeader() const override - { - Block res; - - for (const auto & name_type : columns) - res.insert({ name_type.type->createColumn(), name_type.type, name_type.name }); - - return Nested::flatten(res); - } - protected: - Block readImpl() override; + Chunk generate() override; private: size_t block_size; @@ -181,15 +185,15 @@ private: }; -Block LogBlockInputStream::readImpl() +Chunk LogSource::generate() { Block res; if (rows_read == rows_limit) - return res; + return {}; if (storage.disk->isDirectoryEmpty(storage.table_path)) - return res; + return {}; /// How many rows to read for the next block. size_t max_rows_to_read = std::min(block_size, rows_limit - rows_read); @@ -208,7 +212,7 @@ Block LogBlockInputStream::readImpl() throw; } - if (column->size()) + if (!column->empty()) res.insert(ColumnWithTypeAndName(std::move(column), name_type.type, name_type.name)); } @@ -224,11 +228,13 @@ Block LogBlockInputStream::readImpl() streams.clear(); } - return Nested::flatten(res); + res = Nested::flatten(res); + UInt64 num_rows = res.rows(); + return Chunk(res.getColumns(), num_rows); } -void LogBlockInputStream::readData(const String & name, const IDataType & type, IColumn & column, size_t max_rows_to_read) +void LogSource::readData(const String & name, const IDataType & type, IColumn & column, size_t max_rows_to_read) { IDataType::DeserializeBinaryBulkSettings settings; /// TODO Use avg_value_size_hint. @@ -564,7 +570,7 @@ const StorageLog::Marks & StorageLog::getMarksWithRealRowCount() const return it->second.marks; } -BlockInputStreams StorageLog::read( +Pipes StorageLog::readWithProcessors( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, @@ -579,7 +585,7 @@ BlockInputStreams StorageLog::read( std::shared_lock lock(rwlock); - BlockInputStreams res; + Pipes pipes; const Marks & marks = getMarksWithRealRowCount(); size_t marks_size = marks.size(); @@ -597,7 +603,7 @@ BlockInputStreams StorageLog::read( size_t rows_begin = mark_begin ? marks[mark_begin - 1].rows : 0; size_t rows_end = mark_end ? marks[mark_end - 1].rows : 0; - res.emplace_back(std::make_shared( + pipes.emplace_back(std::make_shared( max_block_size, all_columns, *this, @@ -606,7 +612,7 @@ BlockInputStreams StorageLog::read( max_read_buffer_size)); } - return res; + return pipes; } BlockOutputStreamPtr StorageLog::write( diff --git a/dbms/src/Storages/StorageLog.h b/dbms/src/Storages/StorageLog.h index 696196e4069..e19fb8df07b 100644 --- a/dbms/src/Storages/StorageLog.h +++ b/dbms/src/Storages/StorageLog.h @@ -17,14 +17,14 @@ namespace DB */ class StorageLog : public ext::shared_ptr_helper, public IStorage { - friend class LogBlockInputStream; + friend class LogSource; friend class LogBlockOutputStream; friend struct ext::shared_ptr_helper; public: String getName() const override { return "Log"; } - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -32,6 +32,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; void rename( From fc7839716f91a0af9991b68f7d72b3fb646f5ccc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 18:16:06 +0300 Subject: [PATCH 0158/2007] Processors support for StorageMaterializedView reading. --- dbms/src/Storages/StorageMaterializedView.cpp | 21 ++++++++++++++----- dbms/src/Storages/StorageMaterializedView.h | 4 +++- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 6725fd6458e..b7e514df51f 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -18,6 +18,7 @@ #include #include +#include namespace DB @@ -164,7 +165,7 @@ QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage(cons return getTargetTable()->getQueryProcessingStage(context); } -BlockInputStreams StorageMaterializedView::read( +Pipes StorageMaterializedView::readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -177,10 +178,20 @@ BlockInputStreams StorageMaterializedView::read( if (query_info.order_by_optimizer) query_info.input_sorting_info = query_info.order_by_optimizer->getInputOrder(storage); - auto streams = storage->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); - for (auto & stream : streams) - stream->addTableLock(lock); - return streams; + Pipes pipes; + if (storage->supportProcessorsPipeline()) + pipes = storage->readWithProcessors(column_names, query_info, context, processed_stage, max_block_size, num_streams); + else + { + auto streams = storage->read(column_names, query_info, context, processed_stage, max_block_size, num_streams); + for (auto & stream : streams) + pipes.emplace_back(std::make_shared(stream)); + } + + for (auto & pipe : pipes) + pipe.addTableLock(lock); + + return pipes; } BlockOutputStreamPtr StorageMaterializedView::write(const ASTPtr & query, const Context & context) diff --git a/dbms/src/Storages/StorageMaterializedView.h b/dbms/src/Storages/StorageMaterializedView.h index b811d5a6e2f..4746772ddc9 100644 --- a/dbms/src/Storages/StorageMaterializedView.h +++ b/dbms/src/Storages/StorageMaterializedView.h @@ -55,7 +55,7 @@ public: ActionLock getActionLock(StorageActionBlockType type) override; - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -63,6 +63,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + Strings getDataPaths() const override; private: From dc96e6d1cfda91167712d0b92aad6e97dd0ea640 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 18:26:10 +0300 Subject: [PATCH 0159/2007] Processors support for StorageMemory reading. --- dbms/src/Storages/StorageMemory.cpp | 33 +++++++++++++++-------------- dbms/src/Storages/StorageMemory.h | 4 +++- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index b2cb5682f6d..597657187d1 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include namespace DB @@ -17,34 +19,34 @@ namespace ErrorCodes } -class MemoryBlockInputStream : public IBlockInputStream +class MemorySource : public SourceWithProgress { public: - MemoryBlockInputStream(const Names & column_names_, BlocksList::iterator begin_, BlocksList::iterator end_, const StorageMemory & storage_) - : column_names(column_names_), begin(begin_), end(end_), it(begin), storage(storage_) {} + MemorySource(Names column_names_, BlocksList::iterator begin_, BlocksList::iterator end_, const StorageMemory & storage) + : SourceWithProgress(storage.getSampleBlockForColumns(column_names)) + , column_names(std::move(column_names_)), begin(begin_), end(end_), it(begin) {} String getName() const override { return "Memory"; } - Block getHeader() const override { return storage.getSampleBlockForColumns(column_names); } - protected: - Block readImpl() override + Chunk generate() override { if (it == end) { - return Block(); + return {}; } else { Block src = *it; - Block res; + Columns columns; + columns.reserve(column_names.size()); /// Add only required columns to `res`. - for (size_t i = 0, size = column_names.size(); i < size; ++i) - res.insert(src.getByName(column_names[i])); + for (const auto & name : column_names) + columns.emplace_back(src.getByName(name).column); ++it; - return res; + return Chunk(std::move(columns), src.rows()); } } private: @@ -52,7 +54,6 @@ private: BlocksList::iterator begin; BlocksList::iterator end; BlocksList::iterator it; - const StorageMemory & storage; }; @@ -82,7 +83,7 @@ StorageMemory::StorageMemory(const StorageID & table_id_, ColumnsDescription col } -BlockInputStreams StorageMemory::read( +Pipes StorageMemory::readWithProcessors( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & /*context*/, @@ -99,7 +100,7 @@ BlockInputStreams StorageMemory::read( if (num_streams > size) num_streams = size; - BlockInputStreams res; + Pipes pipes; for (size_t stream = 0; stream < num_streams; ++stream) { @@ -109,10 +110,10 @@ BlockInputStreams StorageMemory::read( std::advance(begin, stream * size / num_streams); std::advance(end, (stream + 1) * size / num_streams); - res.push_back(std::make_shared(column_names, begin, end, *this)); + pipes.emplace_back(std::make_shared(column_names, begin, end, *this)); } - return res; + return pipes; } diff --git a/dbms/src/Storages/StorageMemory.h b/dbms/src/Storages/StorageMemory.h index 1e66b17606b..d6a83cdc694 100644 --- a/dbms/src/Storages/StorageMemory.h +++ b/dbms/src/Storages/StorageMemory.h @@ -28,7 +28,7 @@ public: size_t getSize() const { return data.size(); } - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -36,6 +36,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; void drop(TableStructureWriteLockHolder &) override; From 7bedce40ea43bb868764550d24b34c4271e9053f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 19:29:40 +0300 Subject: [PATCH 0160/2007] Processors support for StorageMerge reading. --- .../DataStreams/narrowBlockInputStreams.cpp | 49 ++++++-- .../src/DataStreams/narrowBlockInputStreams.h | 4 + dbms/src/Storages/StorageMerge.cpp | 112 +++++++++--------- dbms/src/Storages/StorageMerge.h | 17 +-- 4 files changed, 110 insertions(+), 72 deletions(-) diff --git a/dbms/src/DataStreams/narrowBlockInputStreams.cpp b/dbms/src/DataStreams/narrowBlockInputStreams.cpp index 35b60253fab..cbfeb71dfc0 100644 --- a/dbms/src/DataStreams/narrowBlockInputStreams.cpp +++ b/dbms/src/DataStreams/narrowBlockInputStreams.cpp @@ -1,12 +1,28 @@ #include #include #include +#include +#include #include "narrowBlockInputStreams.h" namespace DB { +namespace +{ + using Distribution = std::vector; + Distribution getDistribution(size_t from, size_t to) + { + Distribution distribution(from); + + for (size_t i = 0; i < from; ++i) + distribution[i] = i % to; + + std::shuffle(distribution.begin(), distribution.end(), thread_local_rng); + } +} + BlockInputStreams narrowBlockInputStreams(BlockInputStreams & inputs, size_t width) { size_t size = inputs.size(); @@ -15,13 +31,7 @@ BlockInputStreams narrowBlockInputStreams(BlockInputStreams & inputs, size_t wid std::vector partitions(width); - using Distribution = std::vector; - Distribution distribution(size); - - for (size_t i = 0; i < size; ++i) - distribution[i] = i % width; - - std::shuffle(distribution.begin(), distribution.end(), thread_local_rng); + auto distribution = getDistribution(size, width); for (size_t i = 0; i < size; ++i) partitions[distribution[i]].push_back(inputs[i]); @@ -33,4 +43,29 @@ BlockInputStreams narrowBlockInputStreams(BlockInputStreams & inputs, size_t wid return res; } +Pipes narrowPipes(Pipes pipes, size_t width) +{ + size_t size = pipes.size(); + if (size <= width) + return pipes; + + std::vector partitions(width); + + auto distribution = getDistribution(size, width); + + for (size_t i = 0; i < size; ++i) + partitions[distribution[i]].emplace_back(std::move(pipes[i])); + + Pipes res; + res.reserve(width); + + for (size_t i = 0; i < width; ++i) + { + auto processor = std::make_shared(partitions[i].at(0).getHeader(), partitions[i].size()); + res.emplace_back(std::move(partitions[i]), std::move(processor)); + } + + return res; +} + } diff --git a/dbms/src/DataStreams/narrowBlockInputStreams.h b/dbms/src/DataStreams/narrowBlockInputStreams.h index 305342185b7..3011f85b720 100644 --- a/dbms/src/DataStreams/narrowBlockInputStreams.h +++ b/dbms/src/DataStreams/narrowBlockInputStreams.h @@ -6,6 +6,9 @@ namespace DB { +class Pipe; +using Pipes = std::vector; + /** If the number of sources of `inputs` is greater than `width`, * then glues the sources to each other (using ConcatBlockInputStream), * so that the number of sources becomes no more than `width`. @@ -14,5 +17,6 @@ namespace DB * (to avoid overweighting if the distribution of the amount of data in different sources is subject to some pattern) */ BlockInputStreams narrowBlockInputStreams(BlockInputStreams & inputs, size_t width); +Pipes narrowPipes(Pipes pipes, size_t width); } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 1455556f366..be91c6ea283 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -25,6 +25,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace DB @@ -154,7 +159,7 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage(const Context & } -BlockInputStreams StorageMerge::read( +Pipes StorageMerge::readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -162,7 +167,7 @@ BlockInputStreams StorageMerge::read( const size_t max_block_size, unsigned num_streams) { - BlockInputStreams res; + Pipes res; bool has_table_virtual_column = false; Names real_column_names; @@ -193,7 +198,7 @@ BlockInputStreams StorageMerge::read( if (selected_tables.empty()) /// FIXME: do we support sampling in this case? - return createSourceStreams( + return createSources( query_info, processed_stage, max_block_size, header, {}, real_column_names, modified_context, 0, has_table_virtual_column); size_t tables_count = selected_tables.size(); @@ -219,58 +224,38 @@ BlockInputStreams StorageMerge::read( query_info.input_sorting_info = input_sorting_info; } - for (auto it = selected_tables.begin(); it != selected_tables.end(); ++it) + for (const auto & table : selected_tables) { size_t current_need_streams = tables_count >= num_streams ? 1 : (num_streams / tables_count); size_t current_streams = std::min(current_need_streams, remaining_streams); remaining_streams -= current_streams; current_streams = std::max(size_t(1), current_streams); - auto & storage = std::get<0>(*it); + auto & storage = std::get<0>(table); /// If sampling requested, then check that table supports it. if (query_info.query->as()->sample_size() && !storage->supportsSampling()) throw Exception("Illegal SAMPLE: table doesn't support sampling", ErrorCodes::SAMPLING_NOT_SUPPORTED); - BlockInputStreams source_streams; - - if (current_streams) - { - source_streams = createSourceStreams( + auto source_pipes = createSources( query_info, processed_stage, max_block_size, header, *it, real_column_names, modified_context, current_streams, has_table_virtual_column); - } - else - { - source_streams.emplace_back(std::make_shared( - header, [=, this]() mutable -> BlockInputStreamPtr - { - BlockInputStreams streams = createSourceStreams(query_info, processed_stage, max_block_size, - header, *it, real_column_names, - modified_context, current_streams, has_table_virtual_column, true); - if (!streams.empty() && streams.size() != 1) - throw Exception("LogicalError: the lazy stream size must to be one or empty.", ErrorCodes::LOGICAL_ERROR); - - return streams.empty() ? std::make_shared(header) : streams[0]; - })); - } - - res.insert(res.end(), source_streams.begin(), source_streams.end()); + for (auto & pipe : source_pipes) + res.emplace_back(std::move(pipe)); } if (res.empty()) return res; - res = narrowBlockInputStreams(res, num_streams); - return res; + return narrowPipes(std::move(res), num_streams); } -BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, - const UInt64 max_block_size, const Block & header, const StorageWithLockAndName & storage_with_lock, - Names & real_column_names, - Context & modified_context, size_t streams_num, bool has_table_virtual_column, - bool concat_streams) +Pipes StorageMerge::createSources(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, + const UInt64 max_block_size, const Block & header, const StorageWithLockAndName & storage_with_lock, + Names & real_column_names, + Context & modified_context, size_t streams_num, bool has_table_virtual_column, + bool concat_streams) { auto & [storage, struct_lock, table_name] = storage_with_lock; SelectQueryInfo modified_query_info = query_info; @@ -278,12 +263,17 @@ BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & quer VirtualColumnUtils::rewriteEntityInAst(modified_query_info.query, "_table", table_name); - if (!storage) - return BlockInputStreams{ - InterpreterSelectQuery(modified_query_info.query, modified_context, std::make_shared(header), - SelectQueryOptions(processed_stage).analyze()).execute().in}; + Pipes pipes; - BlockInputStreams source_streams; + if (!storage) + { + pipes.emplace_back( + InterpreterSelectQuery(modified_query_info.query, modified_context, + std::make_shared(header), + SelectQueryOptions(processed_stage).analyze()).executeWithProcessors().getPipe()); + + return pipes; + } if (processed_stage <= storage->getQueryProcessingStage(modified_context)) { @@ -291,8 +281,14 @@ BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & quer if (real_column_names.empty()) real_column_names.push_back(ExpressionActions::getSmallestColumn(storage->getColumns().getAllPhysical())); - source_streams = storage->read(real_column_names, modified_query_info, modified_context, processed_stage, max_block_size, - UInt32(streams_num)); + if (storage->supportProcessorsPipeline()) + pipes = storage->readWithProcessors(real_column_names, modified_query_info, modified_context, processed_stage, max_block_size, UInt32(streams_num)); + else + { + auto streams = storage->read(real_column_names, modified_query_info, modified_context, processed_stage, max_block_size, UInt32(streams_num)); + for (auto & stream : streams) + pipes.emplace_back(std::make_shared(std::move(stream))); + } } else if (processed_stage > storage->getQueryProcessingStage(modified_context)) { @@ -303,41 +299,41 @@ BlockInputStreams StorageMerge::createSourceStreams(const SelectQueryInfo & quer modified_context.getSettingsRef().max_streams_to_max_threads_ratio = 1; InterpreterSelectQuery interpreter{modified_query_info.query, modified_context, SelectQueryOptions(processed_stage)}; - BlockInputStreamPtr interpreter_stream = interpreter.execute().in; + Pipe pipe = interpreter.executeWithProcessors().getPipe(); /** Materialization is needed, since from distributed storage the constants come materialized. * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, * And this is not allowed, since all code is based on the assumption that in the block stream all types are the same. */ - source_streams.emplace_back(std::make_shared(interpreter_stream)); + pipe.addSimpleTransform(std::make_shared(pipe.getHeader())); } - if (!source_streams.empty()) + if (!pipes.empty()) { - if (concat_streams) + if (concat_streams && pipes.size() > 1) { - BlockInputStreamPtr stream = - source_streams.size() > 1 ? std::make_shared(source_streams) : source_streams[0]; + auto concat = std::make_shared(pipes.at(0).getHeader(), pipes.size()); + Pipe pipe(std::move(pipes), std::move(concat)); - source_streams.resize(1); - source_streams[0] = stream; + pipes.clear(); + pipes.emplace_back(std::move(pipe)); } - for (BlockInputStreamPtr & source_stream : source_streams) + for (auto & pipe : pipes) { if (has_table_virtual_column) - source_stream = std::make_shared>( - source_stream, std::make_shared(), table_name, "_table"); + pipe.addSimpleTransform(std::make_shared>( + pipe.getHeader(), std::make_shared(), table_name, "_table")); /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, modified_context, modified_query_info.query, source_stream, processed_stage); + convertingSourceStream(header, modified_context, modified_query_info.query, pipe, processed_stage); - source_stream->addTableLock(struct_lock); + pipe.addTableLock(struct_lock); } } - return source_streams; + return pipes; } @@ -456,10 +452,10 @@ Block StorageMerge::getQueryHeader( } void StorageMerge::convertingSourceStream(const Block & header, const Context & context, ASTPtr & query, - BlockInputStreamPtr & source_stream, QueryProcessingStage::Enum processed_stage) + Pipe & pipe, QueryProcessingStage::Enum processed_stage) { - Block before_block_header = source_stream->getHeader(); - source_stream = std::make_shared(context, source_stream, header, ConvertingBlockInputStream::MatchColumnsMode::Name); + Block before_block_header = pipe.getHeader(); + pipe.addSimpleTransform(std::make_shared(before_block_header, header, ConvertingTransform::MatchColumnsMode::Name, context)); auto where_expression = query->as()->where(); diff --git a/dbms/src/Storages/StorageMerge.h b/dbms/src/Storages/StorageMerge.h index c773ff3ae15..d319b2e316c 100644 --- a/dbms/src/Storages/StorageMerge.h +++ b/dbms/src/Storages/StorageMerge.h @@ -32,7 +32,7 @@ public: QueryProcessingStage::Enum getQueryProcessingStage(const Context &) const override; - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -40,6 +40,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + void checkAlterIsPossible(const AlterCommands & commands, const Settings & /* settings */) override; /// you need to add and remove columns in the sub-tables manually @@ -76,14 +78,15 @@ protected: Block getQueryHeader(const Names & column_names, const SelectQueryInfo & query_info, const Context & context, QueryProcessingStage::Enum processed_stage); - BlockInputStreams createSourceStreams(const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, - const UInt64 max_block_size, const Block & header, const StorageWithLockAndName & storage_with_lock, - Names & real_column_names, - Context & modified_context, size_t streams_num, bool has_table_virtual_column, - bool concat_streams = false); + Pipes createSources( + const SelectQueryInfo & query_info, const QueryProcessingStage::Enum & processed_stage, + const UInt64 max_block_size, const Block & header, const StorageWithLockAndName & storage_with_lock, + Names & real_column_names, + Context & modified_context, size_t streams_num, bool has_table_virtual_column, + bool concat_streams = false); void convertingSourceStream(const Block & header, const Context & context, ASTPtr & query, - BlockInputStreamPtr & source_stream, QueryProcessingStage::Enum processed_stage); + Pipe & pipe, QueryProcessingStage::Enum processed_stage); }; } From dbffabfd466b360f413a337e8f6586b45827493a Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 19:31:08 +0300 Subject: [PATCH 0161/2007] Processors support for StorageMerge reading. --- dbms/src/Storages/StorageMerge.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index be91c6ea283..f9e99d495f0 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -238,7 +238,7 @@ Pipes StorageMerge::readWithProcessors( throw Exception("Illegal SAMPLE: table doesn't support sampling", ErrorCodes::SAMPLING_NOT_SUPPORTED); auto source_pipes = createSources( - query_info, processed_stage, max_block_size, header, *it, real_column_names, modified_context, + query_info, processed_stage, max_block_size, header, table, real_column_names, modified_context, current_streams, has_table_virtual_column); for (auto & pipe : source_pipes) From edd2477ca79356450303c5d1cc11280c38a8bb8f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 19:32:00 +0300 Subject: [PATCH 0162/2007] Processors support for StorageMerge reading. --- dbms/src/DataStreams/narrowBlockInputStreams.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/DataStreams/narrowBlockInputStreams.cpp b/dbms/src/DataStreams/narrowBlockInputStreams.cpp index cbfeb71dfc0..370528d727c 100644 --- a/dbms/src/DataStreams/narrowBlockInputStreams.cpp +++ b/dbms/src/DataStreams/narrowBlockInputStreams.cpp @@ -20,6 +20,7 @@ namespace distribution[i] = i % to; std::shuffle(distribution.begin(), distribution.end(), thread_local_rng); + return distribution; } } From dd888f59a7f6a411e3e7604700cc20c3140c1754 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 19:36:57 +0300 Subject: [PATCH 0163/2007] Fix MemorySource. --- dbms/src/Storages/StorageMemory.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index 597657187d1..89fcac195ab 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -23,7 +23,7 @@ class MemorySource : public SourceWithProgress { public: MemorySource(Names column_names_, BlocksList::iterator begin_, BlocksList::iterator end_, const StorageMemory & storage) - : SourceWithProgress(storage.getSampleBlockForColumns(column_names)) + : SourceWithProgress(storage.getSampleBlockForColumns(column_names_)) , column_names(std::move(column_names_)), begin(begin_), end(end_), it(begin) {} String getName() const override { return "Memory"; } From cef9db809fd71ca9fbc134d373ff3b85af851a1b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 31 Jan 2020 19:46:27 +0300 Subject: [PATCH 0164/2007] Processors support for StorageMySQL reading. --- dbms/src/Storages/StorageMySQL.cpp | 11 +++++++++-- dbms/src/Storages/StorageMySQL.h | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/StorageMySQL.cpp b/dbms/src/Storages/StorageMySQL.cpp index 6f3c6d5b5ae..2eeaf719cf4 100644 --- a/dbms/src/Storages/StorageMySQL.cpp +++ b/dbms/src/Storages/StorageMySQL.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include namespace DB @@ -59,7 +61,7 @@ StorageMySQL::StorageMySQL( } -BlockInputStreams StorageMySQL::read( +Pipes StorageMySQL::readWithProcessors( const Names & column_names_, const SelectQueryInfo & query_info_, const Context & context_, @@ -78,7 +80,12 @@ BlockInputStreams StorageMySQL::read( sample_block.insert({ column_data.type, column_data.name }); } - return { std::make_shared(pool.Get(), query, sample_block, max_block_size_) }; + Pipes pipes; + /// TODO: rewrite MySQLBlockInputStream + pipes.emplace_back(std::make_shared( + std::make_shared(pool.Get(), query, sample_block, max_block_size_))); + + return pipes; } diff --git a/dbms/src/Storages/StorageMySQL.h b/dbms/src/Storages/StorageMySQL.h index 574221377dc..9fb831053e0 100644 --- a/dbms/src/Storages/StorageMySQL.h +++ b/dbms/src/Storages/StorageMySQL.h @@ -33,7 +33,7 @@ public: std::string getName() const override { return "MySQL"; } - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, From 75fe46523e4ad446dad2578ca912d401dad22269 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 31 Jan 2020 22:30:33 +0300 Subject: [PATCH 0165/2007] Working sequential order --- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 19 +-- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 1 + .../ReplicatedMergeTreeMutationEntry.cpp | 6 + .../ReplicatedMergeTreeMutationEntry.h | 2 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 36 ++++- .../MergeTree/ReplicatedMergeTreeQueue.h | 5 +- .../ReplicatedMergeTreeTableMetadata.cpp | 7 - .../Storages/StorageReplicatedMergeTree.cpp | 136 +++++++++--------- .../src/Storages/StorageReplicatedMergeTree.h | 5 +- 9 files changed, 123 insertions(+), 94 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 41fc42fc136..b1dc4856eb9 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -64,14 +64,16 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const << source_parts.at(0) << "\n" << "to\n" << new_part_name; + out << "\nalter_version\n"; + out << alter_version; break; case ALTER_METADATA: /// Just make local /metadata and /columns consistent with global out << "alter\n"; - out << "sync_mode\n"; - out << alter_sync_mode << "\n"; - out << "mutatation_commands\n"; - out << mutation_commands << "\n"; + out << "alter_version\n"; + out << alter_version<< "\n"; + out << "have_mutation\n"; + out << have_mutation << "\n"; out << "columns_str_size:\n"; out << columns_str.size() << "\n"; out << columns_str << "\n"; @@ -166,14 +168,15 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) >> "to\n" >> new_part_name; source_parts.push_back(source_part); + in >> "\nalter_version\n" >> alter_version; } else if (type_str == "alter") { type = ALTER_METADATA; - in >> "sync_mode\n"; - in >> alter_sync_mode; - in >> "\nmutatation_commands\n"; - in >> mutation_commands; + in >> "alter_version\n"; + in >> alter_version; + in >> "\nhave_mutation\n"; + in >> have_mutation; in >> "\ncolumns_str_size:\n"; size_t columns_size; in >> columns_size >> "\n"; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index 068fbc5c344..0284d3e99a6 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -109,6 +109,7 @@ struct ReplicatedMergeTreeLogEntryData //TODO(alesap) int alter_version; + bool have_mutation; String columns_str; String metadata_str; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp index 1695b7e823f..ea744b8f91e 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.cpp @@ -23,6 +23,11 @@ void ReplicatedMergeTreeMutationEntry::writeText(WriteBuffer & out) const out << "commands: "; commands.writeText(out); + out << "\n"; + + out << "alter version: "; + out << alter_version; + } void ReplicatedMergeTreeMutationEntry::readText(ReadBuffer & in) @@ -47,6 +52,7 @@ void ReplicatedMergeTreeMutationEntry::readText(ReadBuffer & in) in >> "commands: "; commands.readText(in); + in >> "\nalter version: " >> alter_version; } String ReplicatedMergeTreeMutationEntry::toString() const diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h index 8f253df3f10..eb5c1a5eab5 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeMutationEntry.h @@ -28,6 +28,8 @@ struct ReplicatedMergeTreeMutationEntry std::map block_numbers; MutationCommands commands; + + int alter_version = -1; }; using ReplicatedMergeTreeMutationEntryPtr = std::shared_ptr; diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 8903f525e94..d1cdf323b3f 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -151,11 +151,18 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } if (entry->type == LogEntry::ALTER_METADATA) { + std::cerr << "INSERT AlTER:" << entry->alter_version << std::endl; alter_sequence.addMetadataAlter(entry->alter_version); } + if (entry->type == LogEntry::MUTATE_PART) + { + + std::cerr << "MUTATIOn WITH VERSION:" << entry->alter_version << std::endl; + } if (entry->type == LogEntry::MUTATE_PART && entry->alter_version != -1) { + std::cerr << "INSERT MUTATE PART:" << entry->alter_version << std::endl; alter_sequence.addDataAlterIfEmpty(entry->alter_version); } } @@ -231,7 +238,8 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (entry->type == LogEntry::ALTER_METADATA) { - alter_sequence.finishMetadataAlter(entry->alter_version); + std::cerr << "Alter have mutation:" << entry->have_mutation << std::endl; + alter_sequence.finishMetadataAlter(entry->alter_version, entry->have_mutation); } } else @@ -408,6 +416,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C { std::lock_guard lock(pull_logs_to_queue_mutex); + std::cerr << "Pooling logs to queue\n"; String index_str = zookeeper->get(replica_path + "/log_pointer"); UInt64 index; @@ -582,6 +591,7 @@ Names ReplicatedMergeTreeQueue::getCurrentPartNamesToMutate(ReplicatedMergeTreeM void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) { std::lock_guard lock(update_mutations_mutex); + std::cerr << "UPdating mutations\n"; Strings entries_in_zk = zookeeper->getChildrenWatch(zookeeper_path + "/mutations", nullptr, watch_callback); StringSet entries_in_zk_set(entries_in_zk.begin(), entries_in_zk.end()); @@ -1025,11 +1035,16 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (entry.type == LogEntry::ALTER_METADATA) { + std::cerr << "Should we execute alter:"; + + std::cerr << alter_sequence.canExecuteMetadataAlter(entry.alter_version) << std::endl; return alter_sequence.canExecuteMetadataAlter(entry.alter_version); } if (entry.type == LogEntry::MUTATE_PART && entry.alter_version != -1) { + std::cerr << "Should we execute mutation:"; + std::cerr << alter_sequence.canExecuteDataAlter(entry.alter_version) << std::endl; return alter_sequence.canExecuteDataAlter(entry.alter_version); } @@ -1735,20 +1750,29 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir return {}; Int64 current_version = queue.getCurrentMutationVersionImpl(part->info.partition_id, part->info.getDataVersion(), lock); - Int64 max_version = in_partition->second.begin()->first; + Int64 max_version = in_partition->second.rbegin()->first; + + if (current_version >= max_version) + { + std::cerr << "But current version is:" << current_version << std::endl; + return {}; + } int alter_version = -1; + + std::cerr << "Looking for alter version for mutation\n"; + String version; for (auto [mutation_version, mutation_status] : in_partition->second) { max_version = mutation_version; - if (mutations_status->entry->alter_version != -1) + if (mutation_version > current_version && mutation_status->entry->alter_version != -1) { - alter_version = mutations_status->entry->alter_version; + alter_version = mutation_status->entry->alter_version; + version = mutation_status->entry->znode_name; break; } } - if (current_version >= max_version) - return {}; + std::cerr << "FOUND alter version:" << alter_version << " and mutation znode name:" << version << std::endl; return std::make_pair(max_version, alter_version); } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index 0506ab76c05..e6250612462 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -123,6 +123,7 @@ private: /// Mapping from znode path to Mutations Status std::map mutations_by_znode; + /// Partition -> (block_number -> MutationStatus) std::unordered_map> mutations_by_partition; /// Znode ID of the latest mutation that is done. String mutation_pointer; @@ -376,10 +377,6 @@ public: std::vector getMutationsStatus() const; - String getLatestMutation() const - { - return mutations_by_znode.rbegin()->first; - } }; class ReplicatedMergeTreeMergePredicate diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index b81108dcfe2..4ebb51f0b41 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -106,19 +106,12 @@ String ReplicatedMergeTreeTableMetadata::toString() const void ReplicatedMergeTreeTableMetadata::read(ReadBuffer & in) { in >> "metadata format version: 1\n"; - std::cerr << "READ METADATA1\n"; in >> "date column: " >> date_column >> "\n"; - std::cerr << "READ METADATA2\n"; in >> "sampling expression: " >> sampling_expression >> "\n"; - std::cerr << "READ METADATA3\n"; in >> "index granularity: " >> index_granularity >> "\n"; - std::cerr << "READ METADATA4\n"; in >> "mode: " >> merging_params_mode >> "\n"; - std::cerr << "READ METADATA5\n"; in >> "sign column: " >> sign_column >> "\n"; - std::cerr << "READ METADATA6\n"; in >> "primary key: " >> primary_key >> "\n"; - std::cerr << "READ METADATA\n"; if (in.eof()) data_format_version = 0; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 79da80cfa84..29c23641a5c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -405,7 +405,6 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() /// ALTERs of the metadata node. zookeeper->createIfNotExists(replica_path + "/metadata", String()); - zookeeper->createIfNotExists(zookeeper_path + "/alters", String()); } @@ -445,6 +444,10 @@ void StorageReplicatedMergeTree::createTableIfNotExists() ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/replicas", "", zkutil::CreateMode::Persistent)); + std::cerr << "Creating alters node\n"; + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/alters", "", + zkutil::CreateMode::Persistent)); + Coordination::Responses responses; auto code = zookeeper->tryMulti(ops, responses); if (code && code != Coordination::ZNODEEXISTS) @@ -3250,56 +3253,6 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer std::cerr << "Nodes in zk updated\n"; } - if (!entry.mutation_commands.empty()) - { - - MutationCommands commands; - ReadBufferFromString in(entry.mutation_commands); - commands.readText(in); - - String mutation_znode; - - while (true) - { - Coordination::Requests requests; - requests.emplace_back(zkutil::makeCreateRequest( - zookeeper_path + "/alters" + entry.znode_name, String(), zkutil::CreateMode::PersistentSequential)); - - prepareMutationEntry(zookeeper, commands, requests); - - std::cerr << replica_name << " - " << "REqeusts size:" << requests.size() << std::endl; - Coordination::Responses responses; - int32_t rc = zookeeper->tryMulti(requests, responses); - - if (rc == Coordination::ZOK) - { - const String & path_created = dynamic_cast(responses.back().get())->path_created; - mutation_znode = path_created.substr(path_created.find_last_of('/') + 1); - std::cerr << "Created mutation\n"; - LOG_TRACE(log, "Created mutation with ID " << mutation_znode); - break; - } - else if (rc == Coordination::ZNODEEXISTS) - { - queue.updateMutations(zookeeper); - - mutation_znode = queue.getLatestMutation(); - - std::cerr << replica_name << " - " << "Found mutation in queue:" << mutation_znode; - LOG_TRACE(log, "Already have mutation with ID " << mutation_znode); - break; - } - else if (rc != Coordination::ZBADVERSION) - { - throw Coordination::Exception("Cannot create mutation for alter", rc); - } - std::cerr << "LOOOPING\n"; - } - - waitMutation(mutation_znode, entry.alter_sync_mode); - } - - return true; } @@ -3344,9 +3297,11 @@ void StorageReplicatedMergeTree::alter( //std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; ReplicatedMergeTreeLogEntryData entry; - Coordination::Requests ops; + + std::optional mutation_znode; /// /columns and /metadata nodes { + Coordination::Requests ops; /// Just to read current structure. Alter will be done in separate thread. auto table_lock = lockStructureForShare(false, query_context.getCurrentQueryId()); @@ -3355,6 +3310,12 @@ void StorageReplicatedMergeTree::alter( StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); + String path_prefix = zookeeper_path + "/alters/alter-"; + String path = zookeeper->create(path_prefix, "", zkutil::CreateMode::EphemeralSequential); + std::cerr << "Path for alter:" << path << std::endl; + int alter_version = parse(path.c_str() + path_prefix.size(), path.size() - path_prefix.size()); + std::cerr << "Alter version:" << alter_version << std::endl; + String new_columns_str = metadata.columns.toString(); /// TODO(alesap) maybe check correct version @@ -3380,6 +3341,11 @@ void StorageReplicatedMergeTree::alter( if (new_metadata_str != ReplicatedMergeTreeTableMetadata(*this).toString()) ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, -1)); + if (ops.empty()) + { + std::cerr << "Alter doesn't change anything\n"; + return; + } /// Perform settings update locally auto old_metadata = getInMemoryMetadata(); old_metadata.settings_ast = metadata.settings_ast; @@ -3388,38 +3354,77 @@ void StorageReplicatedMergeTree::alter( entry.type = LogEntry::ALTER_METADATA; entry.source_replica = replica_name; - entry.alter_sync_mode = query_context.getSettingsRef().replication_alter_partitions_sync; entry.metadata_str = new_metadata_str; entry.columns_str = new_columns_str; - - WriteBufferFromString wb(entry.mutation_commands); - maybe_mutation_commands.writeText(wb); - wb.finalize(); + entry.alter_version = alter_version; entry.create_time = time(nullptr); + if (!maybe_mutation_commands.empty()) + entry.have_mutation = true; ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); - if (!entry.mutation_commands.empty()) + + bool have_mutation = false; + if (!maybe_mutation_commands.empty()) { - prepareMutationEntry(zookeeper, entry.mutation_commands, ops); + prepareMutationEntry(zookeeper, maybe_mutation_commands, ops, alter_version); + have_mutation = true; } - Coordination::Responses results = getZooKeeper()->multi(ops); + Coordination::Responses results; + int32_t rc = zookeeper->tryMulti(ops, results); + queue.pullLogsToQueue(zookeeper); + + std::cerr << "Results size:" << results.size() << std::endl; + std::cerr << "Have mutation:" << have_mutation << std::endl; + + if (rc == Coordination::ZOK) + { + if (have_mutation) + { + std::cerr << "In have mutation\n"; + std::cerr << "INDEX:" << results.size() - 2 << std::endl; + String alter_path = dynamic_cast(*results[results.size() - 3]).path_created; + std::cerr << "Alter path:" << alter_path << std::endl; + entry.znode_name = alter_path.substr(alter_path.find_last_of('/') + 1); + + String mutation_path = dynamic_cast(*results.back()).path_created; + std::cerr << "Mutations path:" << mutation_path << std::endl; + mutation_znode = mutation_path.substr(mutation_path.find_last_of('/') + 1); + } + else + { + std::cerr << "Results size:" << results.size() << std::endl; + String alter_path = dynamic_cast(*results.back()).path_created; + std::cerr << "Alters path:" << alter_path << std::endl; + entry.znode_name = alter_path.substr(alter_path.find_last_of('/') + 1); + } + } + else + { + throw Coordination::Exception("Cannot alter", rc); + } - String path_created = dynamic_cast(*results.back()).path_created; - entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); } - LOG_DEBUG(log, "Updated shared metadata nodes in ZooKeeper. Waiting for replicas to apply changes."); table_lock_holder.release(); + std::cerr << "Started wait for alter\n"; auto unwaited = waitForAllReplicasToProcessLogEntry(entry, false); + std::cerr << "FInished wait for alter\n"; if (!unwaited.empty()) { - throw Exception("Some replicas doesn't finish alter", ErrorCodes::UNFINISHED); + throw Exception("Some replicas doesn't finish metadata alter", ErrorCodes::UNFINISHED); + } + + if (mutation_znode) + { + std::cerr << "Started wait for mutation:" << *mutation_znode << std::endl; + waitMutation(*mutation_znode, query_context.getSettingsRef().replication_alter_partitions_sync); + std::cerr << "FInished wait for mutation\n"; } } @@ -4414,13 +4419,14 @@ void StorageReplicatedMergeTree::waitMutation(const String & znode_name, size_t ReplicatedMergeTreeMutationEntry StorageReplicatedMergeTree::prepareMutationEntry( - zkutil::ZooKeeperPtr zookeeper, const MutationCommands & commands, Coordination::Requests & requests) const + zkutil::ZooKeeperPtr zookeeper, const MutationCommands & commands, Coordination::Requests & requests, int alter_version) const { String mutations_path = zookeeper_path + "/mutations"; ReplicatedMergeTreeMutationEntry entry; entry.source_replica = replica_name; entry.commands = commands; + entry.alter_version = alter_version; Coordination::Stat mutations_stat; zookeeper->get(mutations_path, &mutations_stat); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 488c60ed7c1..67a8c6531cd 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -109,7 +109,7 @@ public: void alterPartition(const ASTPtr & query, const PartitionCommands & commands, const Context & query_context) override; void mutate(const MutationCommands & commands, const Context & context) override; - ReplicatedMergeTreeMutationEntry prepareMutationEntry(zkutil::ZooKeeperPtr zk, const MutationCommands & commands, Coordination::Requests & requests) const; + ReplicatedMergeTreeMutationEntry prepareMutationEntry(zkutil::ZooKeeperPtr zk, const MutationCommands & commands, Coordination::Requests & requests, int alter_version = -1) const; void mutateImpl(zkutil::ZooKeeperPtr zookeeper, const Coordination::Requests & requests, ReplicatedMergeTreeMutationEntry & entry); void waitMutation(const String & znode_name, size_t mutation_sync) const; std::vector getMutationsStatus() const override; @@ -424,9 +424,6 @@ private: /// Checks if some mutations are done and marks them as done. void mutationsFinalizingTask(); - /// finish alter after all heavy processes finished - void finishAlter(); - /** Write the selected parts to merge into the log, * Call when merge_selecting_mutex is locked. * Returns false if any part is not in ZK. From 85d94194c8eaebcc50205184131b682ec354cdb9 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 1 Feb 2020 14:47:09 +0300 Subject: [PATCH 0166/2007] Add additional check for metadata in zookeeper --- .../ReplicatedMergeTreeTableMetadata.cpp | 1 + .../ReplicatedMergeTreeTableMetadata.h | 2 + .../Storages/StorageReplicatedMergeTree.cpp | 45 ++++++++++++++++--- .../src/Storages/StorageReplicatedMergeTree.h | 2 + 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 4ebb51f0b41..cebbaf65723 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -276,4 +276,5 @@ ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTabl return diff; } + } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h index d8af3c2087a..f9697ccad5a 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -65,4 +66,5 @@ private: bool index_granularity_bytes_found_in_zk = false; }; + } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 29c23641a5c..15f15d1069c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3299,7 +3299,6 @@ void StorageReplicatedMergeTree::alter( ReplicatedMergeTreeLogEntryData entry; std::optional mutation_znode; - /// /columns and /metadata nodes { Coordination::Requests ops; /// Just to read current structure. Alter will be done in separate thread. @@ -3308,8 +3307,15 @@ void StorageReplicatedMergeTree::alter( if (is_readonly) throw Exception("Can't ALTER readonly table", ErrorCodes::TABLE_IS_READ_ONLY); - StorageInMemoryMetadata metadata = getInMemoryMetadata(); + Coordination::Stat metadata_stat; + String metadata_in_zk = zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); + Coordination::Stat columns_stat; + String columns_in_zk = zookeeper->get(zookeeper_path + "/columns", &columns_stat); + + StorageInMemoryMetadata metadata = getMetadataFromSharedZookeeper(metadata_in_zk, columns_in_zk); + params.validate(metadata, query_context); params.apply(metadata); + String path_prefix = zookeeper_path + "/alters/alter-"; String path = zookeeper->create(path_prefix, "", zkutil::CreateMode::EphemeralSequential); std::cerr << "Path for alter:" << path << std::endl; @@ -3318,9 +3324,8 @@ void StorageReplicatedMergeTree::alter( String new_columns_str = metadata.columns.toString(); - /// TODO(alesap) maybe check correct version if (new_columns_str != getColumns().toString()) - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, -1)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, columns_stat.version)); ReplicatedMergeTreeTableMetadata new_metadata(*this); if (ast_to_str(metadata.order_by_ast) != ast_to_str(order_by_ast)) @@ -3339,7 +3344,7 @@ void StorageReplicatedMergeTree::alter( String new_metadata_str = new_metadata.toString(); if (new_metadata_str != ReplicatedMergeTreeTableMetadata(*this).toString()) - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, -1)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, metadata_stat.version)); if (ops.empty()) { @@ -3361,6 +3366,8 @@ void StorageReplicatedMergeTree::alter( entry.create_time = time(nullptr); if (!maybe_mutation_commands.empty()) entry.have_mutation = true; + else + entry.have_mutation = false; ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); @@ -5461,4 +5468,32 @@ bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const } +StorageInMemoryMetadata +StorageReplicatedMergeTree::getMetadataFromSharedZookeeper(const String & metadata_str, const String & columns_str) const +{ + auto replicated_metadata = ReplicatedMergeTreeTableMetadata::parse(metadata_str); + StorageInMemoryMetadata result = getInMemoryMetadata(); + result.columns = ColumnsDescription::parse(columns_str); + result.constraints = ConstraintsDescription::parse(replicated_metadata.constraints); + result.indices = IndicesDescription::parse(replicated_metadata.skip_indices); + + ParserExpression expression_p; + + /// The only thing, that can be changed is ttl expression + if (replicated_metadata.primary_key.empty()) + throw Exception("Primary key cannot be empty" , ErrorCodes::LOGICAL_ERROR); + + if (!replicated_metadata.sorting_key.empty()) + { + result.order_by_ast = parseQuery(expression_p, "(" + replicated_metadata.sorting_key + ")", 0); + result.primary_key_ast = parseQuery(expression_p, "(" + replicated_metadata.primary_key + ")", 0); + } + else + { + result.order_by_ast = parseQuery(expression_p, "(" + replicated_metadata.primary_key + ")", 0); + } + return result; + +} + } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 67a8c6531cd..a5e1d1986e1 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -530,6 +530,8 @@ private: void waitMutationToFinishOnReplicas( const Strings & replicas, const String & mutation_id) const; + StorageInMemoryMetadata getMetadataFromSharedZookeeper(const String & metadata_str, const String & columns_str) const; + protected: /** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table. */ From bf883e5a91882c9be24842bfd6ed9d8a0f68e854 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 1 Feb 2020 15:46:22 +0300 Subject: [PATCH 0167/2007] Fix bug --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 3 +-- .../Storages/StorageReplicatedMergeTree.cpp | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index d1cdf323b3f..29ad4631ea6 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -1324,8 +1324,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep { LOG_TRACE(log, "Marking mutation " << znode << " done because it is <= mutation_pointer (" << mutation_pointer << ")"); mutation.is_done = true; - if (mutation.entry->alter_version != -1) - alter_sequence.finishDataAlter(mutation.entry->alter_version); + /// we don't remove alter from queue here, because this maybe called for loaded, but already finished mutation } else if (mutation.parts_to_do == 0) { diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 15f15d1069c..d5afad5b20c 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -3233,23 +3233,26 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer { LOG_INFO(log, "Version of metadata nodes in ZooKeeper changed. Waiting for structure write lock."); - auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); - - LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); - - setTableStructure(std::move(columns_from_entry), metadata_diff); - - LOG_INFO(log, "Applied changes to the metadata of the table."); - - ////std::cerr << "Recalculating columns sizes\n"; - recalculateColumnSizes(); - /// Update metadata ZK nodes for a specific replica. Coordination::Requests requests; requests.emplace_back(zkutil::makeSetRequest(replica_path + "/columns", entry.columns_str, -1)); requests.emplace_back(zkutil::makeSetRequest(replica_path + "/metadata", entry.metadata_str, -1)); zookeeper->multi(requests); + { + auto table_lock = lockExclusively(RWLockImpl::NO_QUERY); + + LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); + + setTableStructure(std::move(columns_from_entry), metadata_diff); + + LOG_INFO(log, "Applied changes to the metadata of the table."); + } + + ////std::cerr << "Recalculating columns sizes\n"; + recalculateColumnSizes(); + /// Update metadata ZK nodes for a specific replica. + std::cerr << "Nodes in zk updated\n"; } From 46e31cd643aea8855d319554ea5b2d94ffd23a40 Mon Sep 17 00:00:00 2001 From: alesapin Date: Sat, 1 Feb 2020 23:33:36 +0300 Subject: [PATCH 0168/2007] Some trash code --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 48 +++++++++---------- .../Storages/StorageReplicatedMergeTree.cpp | 42 ++++++++-------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 29ad4631ea6..403dd8bb966 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -125,7 +125,7 @@ void ReplicatedMergeTreeQueue::initialize( void ReplicatedMergeTreeQueue::insertUnlocked( const LogEntryPtr & entry, std::optional & min_unprocessed_insert_time_changed, - std::lock_guard & /* state_lock */) + std::lock_guard & state_lock) { for (const String & virtual_part_name : entry->getVirtualPartNames()) { @@ -151,19 +151,19 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } if (entry->type == LogEntry::ALTER_METADATA) { - std::cerr << "INSERT AlTER:" << entry->alter_version << std::endl; - alter_sequence.addMetadataAlter(entry->alter_version); + //std::cerr << "INSERT AlTER:" << entry->alter_version << std::endl; + alter_sequence.addMetadataAlter(entry->alter_version, state_lock); } if (entry->type == LogEntry::MUTATE_PART) { - std::cerr << "MUTATIOn WITH VERSION:" << entry->alter_version << std::endl; + //std::cerr << "MUTATIOn WITH VERSION:" << entry->alter_version << std::endl; } if (entry->type == LogEntry::MUTATE_PART && entry->alter_version != -1) { - std::cerr << "INSERT MUTATE PART:" << entry->alter_version << std::endl; - alter_sequence.addDataAlterIfEmpty(entry->alter_version); + //std::cerr << "INSERT MUTATE PART:" << entry->alter_version << std::endl; + alter_sequence.addDataAlterIfEmpty(entry->alter_version, state_lock); } } @@ -186,7 +186,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( bool is_successful, std::optional & min_unprocessed_insert_time_changed, std::optional & max_processed_insert_time_changed, - std::unique_lock & /* queue_lock */) + std::unique_lock & state_lock) { /// Update insert times. if (entry->type == LogEntry::GET_PART) @@ -238,8 +238,8 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (entry->type == LogEntry::ALTER_METADATA) { - std::cerr << "Alter have mutation:" << entry->have_mutation << std::endl; - alter_sequence.finishMetadataAlter(entry->alter_version, entry->have_mutation); + //std::cerr << "Alter have mutation:" << entry->have_mutation << std::endl; + alter_sequence.finishMetadataAlter(entry->alter_version, entry->have_mutation, state_lock); } } else @@ -416,7 +416,7 @@ void ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, C { std::lock_guard lock(pull_logs_to_queue_mutex); - std::cerr << "Pooling logs to queue\n"; + //std::cerr << "Pooling logs to queue\n"; String index_str = zookeeper->get(replica_path + "/log_pointer"); UInt64 index; @@ -591,7 +591,7 @@ Names ReplicatedMergeTreeQueue::getCurrentPartNamesToMutate(ReplicatedMergeTreeM void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) { std::lock_guard lock(update_mutations_mutex); - std::cerr << "UPdating mutations\n"; + //std::cerr << "UPdating mutations\n"; Strings entries_in_zk = zookeeper->getChildrenWatch(zookeeper_path + "/mutations", nullptr, watch_callback); StringSet entries_in_zk_set(entries_in_zk.begin(), entries_in_zk.end()); @@ -948,7 +948,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( String & out_postpone_reason, MergeTreeDataMergerMutator & merger_mutator, MergeTreeData & data, - std::lock_guard & queue_lock) const + std::lock_guard & state_lock) const { if (entry.type == LogEntry::MERGE_PARTS || entry.type == LogEntry::GET_PART @@ -956,7 +956,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { for (const String & new_part_name : entry.getBlockingPartNames()) { - if (!isNotCoveredByFuturePartsImpl(new_part_name, out_postpone_reason, queue_lock)) + if (!isNotCoveredByFuturePartsImpl(new_part_name, out_postpone_reason, state_lock)) { if (!out_postpone_reason.empty()) LOG_DEBUG(log, out_postpone_reason); @@ -1026,7 +1026,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( String range_name = (entry.type == LogEntry::REPLACE_RANGE) ? entry.replace_range_entry->drop_range_part_name : entry.new_part_name; auto range = MergeTreePartInfo::fromPartName(range_name, format_version); - if (0 != getConflictsCountForRange(range, entry, &conflicts_description, queue_lock)) + if (0 != getConflictsCountForRange(range, entry, &conflicts_description, state_lock)) { LOG_DEBUG(log, conflicts_description); return false; @@ -1035,17 +1035,17 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (entry.type == LogEntry::ALTER_METADATA) { - std::cerr << "Should we execute alter:"; + //std::cerr << "Should we execute alter:"; - std::cerr << alter_sequence.canExecuteMetadataAlter(entry.alter_version) << std::endl; - return alter_sequence.canExecuteMetadataAlter(entry.alter_version); + //std::cerr << alter_sequence.canExecuteMetadataAlter(entry.alter_version, state_lock) << std::endl; + return alter_sequence.canExecuteMetadataAlter(entry.alter_version, state_lock); } if (entry.type == LogEntry::MUTATE_PART && entry.alter_version != -1) { - std::cerr << "Should we execute mutation:"; - std::cerr << alter_sequence.canExecuteDataAlter(entry.alter_version) << std::endl; - return alter_sequence.canExecuteDataAlter(entry.alter_version); + //std::cerr << "Should we execute mutation:"; + //std::cerr << alter_sequence.canExecuteDataAlter(entry.alter_version, state_lock) << std::endl; + return alter_sequence.canExecuteDataAlter(entry.alter_version, state_lock); } return true; @@ -1362,7 +1362,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep LOG_TRACE(log, "Mutation " << entry->znode_name << " is done"); it->second.is_done = true; if (entry->alter_version != -1) - alter_sequence.finishDataAlter(entry->alter_version); + alter_sequence.finishDataAlter(entry->alter_version, lock); } } } @@ -1753,13 +1753,13 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir if (current_version >= max_version) { - std::cerr << "But current version is:" << current_version << std::endl; + //std::cerr << "But current version is:" << current_version << std::endl; return {}; } int alter_version = -1; - std::cerr << "Looking for alter version for mutation\n"; + //std::cerr << "Looking for alter version for mutation\n"; String version; for (auto [mutation_version, mutation_status] : in_partition->second) { @@ -1771,7 +1771,7 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir break; } } - std::cerr << "FOUND alter version:" << alter_version << " and mutation znode name:" << version << std::endl; + //std::cerr << "FOUND alter version:" << alter_version << " and mutation znode name:" << version << std::endl; return std::make_pair(max_version, alter_version); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index d5afad5b20c..30d82184e5f 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -293,7 +293,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( checkTableStructure(skip_sanity_checks, true); checkParts(skip_sanity_checks); - /// Temporary directories contain unfinalized results of Merges or Fetches (after forced restart) + /// Temporary directories contain untinalized results of Merges or Fetches (after forced restart) /// and don't allow to reinitialize them, so delete each of them immediately clearOldTemporaryDirectories(0); } @@ -444,7 +444,7 @@ void StorageReplicatedMergeTree::createTableIfNotExists() ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/replicas", "", zkutil::CreateMode::Persistent)); - std::cerr << "Creating alters node\n"; + ////std::cerr << "Creating alters node\n"; ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/alters", "", zkutil::CreateMode::Persistent)); @@ -3225,7 +3225,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer auto metadata_from_entry = ReplicatedMergeTreeTableMetadata::parse(entry.metadata_str); auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this).checkAndFindDiff(metadata_from_entry, /* allow_alter = */ true); - ////std::cerr << "Metadata received\n"; + //////std::cerr << "Metadata received\n"; MergeTreeData::DataParts parts; @@ -3249,11 +3249,11 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Applied changes to the metadata of the table."); } - ////std::cerr << "Recalculating columns sizes\n"; + //////std::cerr << "Recalculating columns sizes\n"; recalculateColumnSizes(); /// Update metadata ZK nodes for a specific replica. - std::cerr << "Nodes in zk updated\n"; + ////std::cerr << "Nodes in zk updated\n"; } return true; @@ -3297,7 +3297,7 @@ void StorageReplicatedMergeTree::alter( auto zookeeper = getZooKeeper(); - //std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; + ////std::cerr << " Columns preparation to alter:" << getColumns().getAllPhysical().toString() << std::endl; ReplicatedMergeTreeLogEntryData entry; @@ -3321,9 +3321,9 @@ void StorageReplicatedMergeTree::alter( String path_prefix = zookeeper_path + "/alters/alter-"; String path = zookeeper->create(path_prefix, "", zkutil::CreateMode::EphemeralSequential); - std::cerr << "Path for alter:" << path << std::endl; + ////std::cerr << "Path for alter:" << path << std::endl; int alter_version = parse(path.c_str() + path_prefix.size(), path.size() - path_prefix.size()); - std::cerr << "Alter version:" << alter_version << std::endl; + //std::cerr << "Alter version:" << alter_version << std::endl; String new_columns_str = metadata.columns.toString(); @@ -3351,7 +3351,7 @@ void StorageReplicatedMergeTree::alter( if (ops.empty()) { - std::cerr << "Alter doesn't change anything\n"; + //std::cerr << "Alter doesn't change anything\n"; return; } /// Perform settings update locally @@ -3386,28 +3386,28 @@ void StorageReplicatedMergeTree::alter( int32_t rc = zookeeper->tryMulti(ops, results); queue.pullLogsToQueue(zookeeper); - std::cerr << "Results size:" << results.size() << std::endl; - std::cerr << "Have mutation:" << have_mutation << std::endl; + //std::cerr << "Results size:" << results.size() << std::endl; + //std::cerr << "Have mutation:" << have_mutation << std::endl; if (rc == Coordination::ZOK) { if (have_mutation) { - std::cerr << "In have mutation\n"; - std::cerr << "INDEX:" << results.size() - 2 << std::endl; + //std::cerr << "In have mutation\n"; + //std::cerr << "INDEX:" << results.size() - 2 << std::endl; String alter_path = dynamic_cast(*results[results.size() - 3]).path_created; - std::cerr << "Alter path:" << alter_path << std::endl; + //std::cerr << "Alter path:" << alter_path << std::endl; entry.znode_name = alter_path.substr(alter_path.find_last_of('/') + 1); String mutation_path = dynamic_cast(*results.back()).path_created; - std::cerr << "Mutations path:" << mutation_path << std::endl; + //std::cerr << "Mutations path:" << mutation_path << std::endl; mutation_znode = mutation_path.substr(mutation_path.find_last_of('/') + 1); } else { - std::cerr << "Results size:" << results.size() << std::endl; + //std::cerr << "Results size:" << results.size() << std::endl; String alter_path = dynamic_cast(*results.back()).path_created; - std::cerr << "Alters path:" << alter_path << std::endl; + //std::cerr << "Alters path:" << alter_path << std::endl; entry.znode_name = alter_path.substr(alter_path.find_last_of('/') + 1); } } @@ -3421,9 +3421,9 @@ void StorageReplicatedMergeTree::alter( table_lock_holder.release(); - std::cerr << "Started wait for alter\n"; + //std::cerr << "Started wait for alter\n"; auto unwaited = waitForAllReplicasToProcessLogEntry(entry, false); - std::cerr << "FInished wait for alter\n"; + //std::cerr << "FInished wait for alter\n"; if (!unwaited.empty()) { @@ -3432,9 +3432,9 @@ void StorageReplicatedMergeTree::alter( if (mutation_znode) { - std::cerr << "Started wait for mutation:" << *mutation_znode << std::endl; + //std::cerr << "Started wait for mutation:" << *mutation_znode << std::endl; waitMutation(*mutation_znode, query_context.getSettingsRef().replication_alter_partitions_sync); - std::cerr << "FInished wait for mutation\n"; + //std::cerr << "FInished wait for mutation\n"; } } From 568893958b6abbfd5d414109025b6aa797c124ad Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Sun, 2 Feb 2020 00:22:00 +0300 Subject: [PATCH 0169/2007] implemented couple of generators --- .../DataTypes/DataTypeAggregateFunction.cpp | 4 - .../src/DataTypes/DataTypeAggregateFunction.h | 1 - dbms/src/DataTypes/DataTypeArray.cpp | 7 - dbms/src/DataTypes/DataTypeArray.h | 1 - dbms/src/DataTypes/DataTypeDecimalBase.cpp | 7 - dbms/src/DataTypes/DataTypeDecimalBase.h | 1 - dbms/src/DataTypes/DataTypeEnum.cpp | 8 - dbms/src/DataTypes/DataTypeEnum.h | 1 - dbms/src/DataTypes/DataTypeFixedString.cpp | 6 - dbms/src/DataTypes/DataTypeFixedString.h | 1 - dbms/src/DataTypes/DataTypeInterval.h | 2 +- dbms/src/DataTypes/DataTypeLowCardinality.cpp | 6 - dbms/src/DataTypes/DataTypeLowCardinality.h | 1 - dbms/src/DataTypes/DataTypeNothing.cpp | 8 - dbms/src/DataTypes/DataTypeNothing.h | 1 - dbms/src/DataTypes/DataTypeNullable.cpp | 5 - dbms/src/DataTypes/DataTypeNullable.h | 1 - dbms/src/DataTypes/DataTypeNumberBase.cpp | 7 - dbms/src/DataTypes/DataTypeNumberBase.h | 1 - dbms/src/DataTypes/DataTypeSet.h | 4 - dbms/src/DataTypes/DataTypeString.cpp | 5 - dbms/src/DataTypes/DataTypeString.h | 1 - dbms/src/DataTypes/DataTypeTuple.cpp | 8 - dbms/src/DataTypes/DataTypeTuple.h | 1 - dbms/src/DataTypes/IDataType.h | 4 - dbms/src/DataTypes/IDataTypeDummy.h | 5 - .../TableFunctions/TableFunctionRandom.cpp | 228 +++++++++++++++++- .../01072_random_table_function.sql | 46 +++- 28 files changed, 273 insertions(+), 98 deletions(-) diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp index f3b26497912..8111b1de2fe 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp @@ -304,10 +304,6 @@ MutableColumnPtr DataTypeAggregateFunction::createColumn() const return ColumnAggregateFunction::create(function); } -MutableColumnPtr DataTypeAggregateFunction::createColumnWithRandomData(size_t) const -{ - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} /// Create empty state Field DataTypeAggregateFunction::getDefault() const diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.h b/dbms/src/DataTypes/DataTypeAggregateFunction.h index e4c226b2917..9ae7c67a803 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.h +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.h @@ -63,7 +63,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index 0500182c61a..e2c03805ea8 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -487,13 +487,6 @@ MutableColumnPtr DataTypeArray::createColumn() const } -MutableColumnPtr DataTypeArray::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - Field DataTypeArray::getDefault() const { return Array(); diff --git a/dbms/src/DataTypes/DataTypeArray.h b/dbms/src/DataTypes/DataTypeArray.h index ccf269bd357..1451f27dfbe 100644 --- a/dbms/src/DataTypes/DataTypeArray.h +++ b/dbms/src/DataTypes/DataTypeArray.h @@ -94,7 +94,6 @@ public: bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.cpp b/dbms/src/DataTypes/DataTypeDecimalBase.cpp index a0f2bd7bd82..7b9a391427c 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.cpp +++ b/dbms/src/DataTypes/DataTypeDecimalBase.cpp @@ -41,13 +41,6 @@ MutableColumnPtr DataTypeDecimalBase::createColumn() const return ColumnType::create(0, scale); } -template -MutableColumnPtr DataTypeDecimalBase::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - template void DataTypeDecimalBase::serializeBinary(const Field & field, WriteBuffer & ostr) const { diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.h b/dbms/src/DataTypes/DataTypeDecimalBase.h index d579b965412..11f7490e80a 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.h +++ b/dbms/src/DataTypes/DataTypeDecimalBase.h @@ -83,7 +83,6 @@ public: Field getDefault() const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return true; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index fcaf9f2c8a3..5ca6296f43d 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -347,14 +347,6 @@ Field DataTypeEnum::castToValue(const Field & value_or_name) const } -template -MutableColumnPtr DataTypeEnum::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - /// Explicit instantiations. template class DataTypeEnum; template class DataTypeEnum; diff --git a/dbms/src/DataTypes/DataTypeEnum.h b/dbms/src/DataTypes/DataTypeEnum.h index a0408df0279..2cb677984df 100644 --- a/dbms/src/DataTypes/DataTypeEnum.h +++ b/dbms/src/DataTypes/DataTypeEnum.h @@ -111,7 +111,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override { return ColumnType::create(); } - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index a148d0b2d22..d30f1003ca0 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -268,12 +268,6 @@ MutableColumnPtr DataTypeFixedString::createColumn() const return ColumnFixedString::create(n); } -MutableColumnPtr DataTypeFixedString::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - Field DataTypeFixedString::getDefault() const { return String(); diff --git a/dbms/src/DataTypes/DataTypeFixedString.h b/dbms/src/DataTypes/DataTypeFixedString.h index 4f264d3ac86..6d1f1c4db83 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.h +++ b/dbms/src/DataTypes/DataTypeFixedString.h @@ -70,7 +70,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeInterval.h b/dbms/src/DataTypes/DataTypeInterval.h index 111a2489d65..d66b329185d 100644 --- a/dbms/src/DataTypes/DataTypeInterval.h +++ b/dbms/src/DataTypes/DataTypeInterval.h @@ -12,7 +12,7 @@ namespace DB * Mostly the same as Int64. * But also tagged with interval kind. * - * Intended isage is for temporary elements in expressions, + * Intended usage is for temporary elements in expressions, * not for storing values in tables. */ class DataTypeInterval final : public DataTypeNumberBase diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.cpp b/dbms/src/DataTypes/DataTypeLowCardinality.cpp index 24dc3af48c9..5db32bd5380 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.cpp +++ b/dbms/src/DataTypes/DataTypeLowCardinality.cpp @@ -934,12 +934,6 @@ MutableColumnPtr DataTypeLowCardinality::createColumn() const return ColumnLowCardinality::create(std::move(dictionary), std::move(indexes)); } -MutableColumnPtr DataTypeLowCardinality::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - Field DataTypeLowCardinality::getDefault() const { return dictionary_type->getDefault(); diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.h b/dbms/src/DataTypes/DataTypeLowCardinality.h index 9b22acea7e3..f8c314909b8 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.h +++ b/dbms/src/DataTypes/DataTypeLowCardinality.h @@ -68,7 +68,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNothing.cpp b/dbms/src/DataTypes/DataTypeNothing.cpp index ce4990748f9..79fbb002bff 100644 --- a/dbms/src/DataTypes/DataTypeNothing.cpp +++ b/dbms/src/DataTypes/DataTypeNothing.cpp @@ -14,14 +14,6 @@ MutableColumnPtr DataTypeNothing::createColumn() const return ColumnNothing::create(0); } - -MutableColumnPtr DataTypeNothing::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - void DataTypeNothing::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { size_t size = column.size(); diff --git a/dbms/src/DataTypes/DataTypeNothing.h b/dbms/src/DataTypes/DataTypeNothing.h index 5fbe0acc0a9..e9421fb15e8 100644 --- a/dbms/src/DataTypes/DataTypeNothing.h +++ b/dbms/src/DataTypes/DataTypeNothing.h @@ -19,7 +19,6 @@ public: TypeIndex getTypeId() const override { return TypeIndex::Nothing; } MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; /// These methods read and write zero bytes just to allow to figure out size of column. void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index 6f31e66a1e5..397d5ba0a65 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -488,11 +488,6 @@ MutableColumnPtr DataTypeNullable::createColumn() const return ColumnNullable::create(nested_data_type->createColumn(), ColumnUInt8::create()); } -MutableColumnPtr DataTypeNullable::createColumnWithRandomData(size_t limit) const -{ - return ColumnNullable::create(nested_data_type->createColumnWithRandomData(limit), DataTypeUInt8().createColumnWithRandomData(limit)); -} - Field DataTypeNullable::getDefault() const { return Null(); diff --git a/dbms/src/DataTypes/DataTypeNullable.h b/dbms/src/DataTypes/DataTypeNullable.h index 83a76ae0410..1766b399c2a 100644 --- a/dbms/src/DataTypes/DataTypeNullable.h +++ b/dbms/src/DataTypes/DataTypeNullable.h @@ -76,7 +76,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNumberBase.cpp b/dbms/src/DataTypes/DataTypeNumberBase.cpp index 937967d431a..90356817730 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.cpp +++ b/dbms/src/DataTypes/DataTypeNumberBase.cpp @@ -239,13 +239,6 @@ MutableColumnPtr DataTypeNumberBase::createColumn() const return ColumnVector::create(); } -template -MutableColumnPtr DataTypeNumberBase::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - template bool DataTypeNumberBase::isValueRepresentedByInteger() const { diff --git a/dbms/src/DataTypes/DataTypeNumberBase.h b/dbms/src/DataTypes/DataTypeNumberBase.h index 5a3dda5fe15..fb752ad5329 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.h +++ b/dbms/src/DataTypes/DataTypeNumberBase.h @@ -45,7 +45,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return false; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index f468881cfe9..7ef0d931279 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -21,10 +21,6 @@ public: // Used for expressions analysis. MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } - MutableColumnPtr createColumnWithRandomData(size_t) const override - { - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } // Used only for debugging, making it DUMPABLE Field getDefault() const override { return Tuple(); } diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index 46478396a68..ef32fe33690 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -360,11 +360,6 @@ MutableColumnPtr DataTypeString::createColumn() const return ColumnString::create(); } -MutableColumnPtr DataTypeString::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} bool DataTypeString::equals(const IDataType & rhs) const { diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h index 4a2c6be42e1..28968eef3f1 100644 --- a/dbms/src/DataTypes/DataTypeString.h +++ b/dbms/src/DataTypes/DataTypeString.h @@ -54,7 +54,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 5c912b89f2d..4d60177aa4d 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -454,14 +454,6 @@ MutableColumnPtr DataTypeTuple::createColumn() const return ColumnTuple::create(std::move(tuple_columns)); } - -MutableColumnPtr DataTypeTuple::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - Field DataTypeTuple::getDefault() const { return Tuple(ext::map(elems, [] (const DataTypePtr & elem) { return elem->getDefault(); })); diff --git a/dbms/src/DataTypes/DataTypeTuple.h b/dbms/src/DataTypes/DataTypeTuple.h index a3a8fb2847e..06f0f62026e 100644 --- a/dbms/src/DataTypes/DataTypeTuple.h +++ b/dbms/src/DataTypes/DataTypeTuple.h @@ -81,7 +81,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & reader, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/IDataType.h b/dbms/src/DataTypes/IDataType.h index 04ad5896154..92d0c1057c5 100644 --- a/dbms/src/DataTypes/IDataType.h +++ b/dbms/src/DataTypes/IDataType.h @@ -287,10 +287,6 @@ public: */ virtual MutableColumnPtr createColumn() const = 0; - /** Create column for corresponding type and fill with random values. - */ - virtual MutableColumnPtr createColumnWithRandomData(size_t size) const = 0; - /** Create ColumnConst for corresponding type, with specified size and value. */ ColumnPtr createColumnConst(size_t size, const Field & field) const; diff --git a/dbms/src/DataTypes/IDataTypeDummy.h b/dbms/src/DataTypes/IDataTypeDummy.h index e346689274f..f27359e5f74 100644 --- a/dbms/src/DataTypes/IDataTypeDummy.h +++ b/dbms/src/DataTypes/IDataTypeDummy.h @@ -42,11 +42,6 @@ public: throw Exception("Method createColumn() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); } - MutableColumnPtr createColumnWithRandomData(size_t) const override - { - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } - Field getDefault() const override { throw Exception("Method getDefault() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index 3d4bb1d3247..1391ecdc74b 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -4,11 +4,17 @@ #include #include #include +#include +#include + #include #include #include +#include +#include + #include #include #include @@ -24,6 +30,226 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; + extern const int BAD_TYPE_OF_FIELD; + extern const int LOGICAL_ERROR; +} + +MutableColumnPtr createColumnWithRandomData(DataTypePtr type, UInt64 limit) +{ + TypeIndex idx = type->getTypeId(); + MutableColumnPtr column = type->createColumn(); + + switch (idx) + { + case TypeIndex::Nothing: + for (UInt64 i = 0; i < limit; ++i) + { + column->insertDefault(); + } + throw Exception("Random Generator not implemented for type 'Nothing'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::UInt8: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt16: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt32: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt64: + { + pcg64 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt128: + throw Exception("Random Generator not implemented for type 'UInt128'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Int8: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int16: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int32: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int64: + { + pcg64 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int128: + throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Float32: + { + pcg32 generator(randomSeed()); + double d; + for (UInt64 i = 0; i < limit; ++i) + { + d = std::numeric_limits::max(); + column->insert( (d / pcg32::max()) * generator() ); + } + } + break; + case TypeIndex::Float64: + { + pcg64 generator(randomSeed()); + double d; + for (UInt64 i = 0; i < limit; ++i) + { + d = std::numeric_limits::max(); + column->insert( (d / pcg64::max()) * generator() ); + } + } + break; + case TypeIndex::Date: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::DateTime: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::DateTime64: + { + UInt32 scale; + if (auto * ptype = typeid_cast(type.get())) + scale = ptype->getScale(); + else + throw Exception("Static cast to DataTypeDateTime64 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + UInt32 fractional = static_cast(generator()) % intExp10(scale); + UInt32 whole = static_cast(generator()); + DateTime64 dt = DecimalUtils::decimalFromComponents(whole, fractional, scale); + column->insert(DecimalField(dt, scale)); + } + } + break; + case TypeIndex::String: + throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::FixedString: + throw Exception("Random Generator not implemented for type 'FixedString'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Enum8: + throw Exception("Random Generator not implemented for type 'Enum8'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Enum16: + throw Exception("Random Generator not implemented for type 'Enum16'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Decimal32: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Decimal64: + { + pcg64 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Decimal128: + throw Exception("Random Generator not implemented for type 'Decimal128'.", ErrorCodes::NOT_IMPLEMENTED); +/* + { + UInt32 scale = 0; + if (auto * ptype = typeid_cast *>(type.get())) + scale = ptype->getScale(); + else + throw Exception("Static cast to Decimal128 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + + pcg128_once_insecure generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(DecimalField(static_cast(generator()), scale)); + } + } + break; +*/ + case TypeIndex::UUID: + { + pcg128_once_insecure generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Array: + throw Exception("Random Generator not implemented for type 'Array'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Tuple: + throw Exception("Random Generator not implemented for type 'Tuple'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Set: + throw Exception("Random Generator not implemented for type 'Set'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Interval: + throw Exception("Type 'Interval' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); + case TypeIndex::Nullable: + throw Exception("Random Generator not implemented for type 'Nullable'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Function: + throw Exception("Random Generator not implemented for type 'Function'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::AggregateFunction: + throw Exception("Random Generator not implemented for type 'AggregateFunction'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::LowCardinality: + throw Exception("Random Generator not implemented for type 'LowCardinality'.", ErrorCodes::NOT_IMPLEMENTED); + } + return column; } StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const @@ -55,7 +281,7 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C Block res_block; for (const auto & name_type : columns.getOrdinary()) { - MutableColumnPtr column = name_type.type->createColumnWithRandomData(limit); + MutableColumnPtr column = createColumnWithRandomData(name_type.type, limit); res_block.insert({std::move(column), name_type.type, name_type.name}); } diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.sql b/dbms/tests/queries/0_stateless/01072_random_table_function.sql index 21f0925439d..c81d630d9b5 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.sql +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.sql @@ -1 +1,45 @@ -SELECT * FROM generate('id int', 3) +SELECT +toTypeName(ui64), toTypeName(i64), +toTypeName(ui32), toTypeName(i32), +toTypeName(ui16), toTypeName(i16), +toTypeName(ui8), toTypeName(i8) +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 1); +SELECT +ui64, i64, +ui32, i32, +ui16, i16, +ui8, i8 +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10); +SELECT +toTypeName(d), toTypeName(dt), toTypeName(dtm) +FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 1); +SELECT +d, dt, dtm +FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10) FORMAT JSONEachRow;; +SELECT +toTypeName(dt64), toTypeName(dts64), toTypeName(dtms64) +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 1); +SELECT +dt64, dts64, dtms64 +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10) FORMAT JSONEachRow; +SELECT +dt64, dts64, dtms64 +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10); +SELECT + toTypeName(f32), toTypeName(f64) +FROM generate('f32 Float32, f64 Float64', 1); +SELECT + f32, f64 +FROM generate('f32 Float32, f64 Float64', 10) FORMAT JSONEachRow; +SELECT + toTypeName(d32), toTypeName(d64) +FROM generate('d32 Decimal32(4), d64 Decimal64(8)', 1); +SELECT + d32, d64 +FROM generate('d32 Decimal32(4), d64 Decimal64(8)', 10) FORMAT JSONEachRow; +SELECT + toTypeName(i) +FROM generate('i Interval', 10); +SELECT + i +FROM generate('i Interval', 10) FORMAT JSONEachRow; From 55c5bdbace92cfbbc67da0e9aa272615bd05dc6a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 2 Feb 2020 05:49:36 +0300 Subject: [PATCH 0170/2007] Trigger CI --- dbms/src/Storages/MergeTree/DataPartsExchange.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index 22e04e3628e..cd6b62dd5b2 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -2,8 +2,8 @@ #include #include #include -#include #include +#include #include #include From 0c37458b2075e944ecec2b61f4b2ab0c00b152eb Mon Sep 17 00:00:00 2001 From: alesapin Date: Sun, 2 Feb 2020 11:45:09 +0300 Subject: [PATCH 0171/2007] Add missed file --- .../MergeTree/ReplicatedQueueAlterState.h | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h diff --git a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h new file mode 100644 index 00000000000..6980a117b15 --- /dev/null +++ b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h @@ -0,0 +1,122 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + +class AlterSequence +{ +private: + enum AlterState + { + APPLY_METADATA_CHANGES, + APPLY_DATA_CHANGES, + }; + + struct AlterInQueue + { + int alter_version; + AlterState state; + + AlterInQueue(int alter_version_, AlterState state_) + : alter_version(alter_version_) + , state(state_) + { + } + }; + + + std::deque queue; +public: + + bool empty() const { + return queue.empty(); + } + + void addMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) + { + queue.emplace_back(alter_version, AlterState::APPLY_METADATA_CHANGES); + } + + void addDataAlterIfEmpty(int alter_version, std::lock_guard & /*state_lock*/) + { + if (queue.empty()) + queue.emplace_back(alter_version, AlterState::APPLY_DATA_CHANGES); + + if (queue.front().alter_version != alter_version) + { + throw Exception( + "Alter head has another version number " + + std::to_string(queue.front().alter_version) + " than ours " + std::to_string(alter_version), + ErrorCodes::LOGICAL_ERROR); + } + } + + void finishMetadataAlter(int alter_version, bool have_data_alter, std::unique_lock & /*state_lock*/) + { + if (queue.empty()) + { + throw Exception("Queue shouldn't be empty on metadata alter", ErrorCodes::LOGICAL_ERROR); + } + if (queue.front().alter_version != alter_version) + throw Exception("Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue.front().alter_version), ErrorCodes::LOGICAL_ERROR); + + if (have_data_alter) + { + //std::cerr << "Switching head state:" << AlterState::APPLY_DATA_CHANGES << std::endl; + queue.front().state = AlterState::APPLY_DATA_CHANGES; + } + else + { + //std::cerr << "JUST POP FRONT\n"; + queue.pop_front(); + } + } + + void finishDataAlter(int alter_version, std::lock_guard & /*state_lock*/) + { + /// queue can be empty after load of finished mutation without move of mutation pointer + if (queue.empty()) + return; + //std::cerr << "Finishing data alter:" << alter_version << std::endl; + if (queue.front().alter_version != alter_version) + throw Exception( + "Finished data alter with version " + std::to_string(alter_version) + " but current alter in queue is " + + std::to_string(queue.front().alter_version), + ErrorCodes::LOGICAL_ERROR); + + if (queue.front().state != AlterState::APPLY_DATA_CHANGES) + { + throw Exception( + "Finished data alter but current alter should perform metadata alter", + ErrorCodes::LOGICAL_ERROR); + } + + queue.pop_front(); + } + + bool canExecuteMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) const + { + //std::cerr << "Alter queue front:" << queue.front().alter_version << " state:" << queue.front().state << std::endl; + return queue.front().alter_version == alter_version; + } + + bool canExecuteDataAlter(int alter_version, std::lock_guard & /*state_lock*/) const + { + //std::cerr << "CAn execute:" << alter_version << std::endl; + //std::cerr << "FRont version:" << queue.front().alter_version << std::endl; + //std::cerr << "State:" << queue.front().state << std::endl; + return queue.front().alter_version == alter_version && queue.front().state == AlterState::APPLY_DATA_CHANGES; + } + +}; + +} From 0eca2dd5caa7d0600a2f8ec511cd2deff59cc51b Mon Sep 17 00:00:00 2001 From: Aleksei Levushkin Date: Sun, 2 Feb 2020 16:29:02 +0300 Subject: [PATCH 0172/2007] added branch to grpc contrib --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 02b1146871b..b4bc3dc33a5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -107,6 +107,7 @@ [submodule "contrib/grpc"] path = contrib/grpc url = https://github.com/grpc/grpc.git + branch = v1.25.0 [submodule "contrib/aws"] path = contrib/aws url = https://github.com/aws/aws-sdk-cpp.git From 439ef6af9a7e455565e7247db8adeb2b10a70521 Mon Sep 17 00:00:00 2001 From: Aleksei Levushkin Date: Sun, 2 Feb 2020 16:36:59 +0300 Subject: [PATCH 0173/2007] turn off protobuf tests --- contrib/grpc-cmake/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index cc1c2b52b51..b1941ae508f 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -54,9 +54,10 @@ endif() # protobuf.cmake set(PROTOBUF_ROOT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/../protobuf) -if(NOT protobuf_BUILD_TESTS) - set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests") -endif() +# if(NOT protobuf_BUILD_TESTS) + # set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests") +# endif() +set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests") if(NOT protobuf_WITH_ZLIB) set(protobuf_WITH_ZLIB OFF CACHE BOOL "Build protobuf with zlib.") endif() From 9664476479817a183ab0a5385cb0b89dde400006 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 3 Feb 2020 02:52:27 +0300 Subject: [PATCH 0174/2007] Update OpenSSL --- contrib/openssl | 2 +- contrib/openssl-cmake/CMakeLists.txt | 17 +++++++++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/contrib/openssl b/contrib/openssl index c74e7895eb0..943e83620d1 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit c74e7895eb0d219d4007775eec134dd2bcd9d1ae +Subproject commit 943e83620d178025df0695e707557bd702922b49 diff --git a/contrib/openssl-cmake/CMakeLists.txt b/contrib/openssl-cmake/CMakeLists.txt index 8dc0c6ae6c5..4dd76665006 100644 --- a/contrib/openssl-cmake/CMakeLists.txt +++ b/contrib/openssl-cmake/CMakeLists.txt @@ -188,6 +188,7 @@ ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_buff.c ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_lbuf.c ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_nbio.c ${OPENSSL_SOURCE_DIR}/crypto/bio/bf_null.c +${OPENSSL_SOURCE_DIR}/crypto/bio/bf_prefix.c ${OPENSSL_SOURCE_DIR}/crypto/bio/bio_cb.c ${OPENSSL_SOURCE_DIR}/crypto/bio/bio_err.c ${OPENSSL_SOURCE_DIR}/crypto/bio/bio_lib.c @@ -320,6 +321,7 @@ ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_check.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_depr.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_err.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_gen.c +${OPENSSL_SOURCE_DIR}/crypto/dh/dh_group_params.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_kdf.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_key.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_lib.c @@ -327,7 +329,7 @@ ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_meth.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_pmeth.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_prn.c ${OPENSSL_SOURCE_DIR}/crypto/dh/dh_rfc5114.c -${OPENSSL_SOURCE_DIR}/crypto/dh/dh_rfc7919.c +${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_aid.c ${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_ameth.c ${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_asn1.c ${OPENSSL_SOURCE_DIR}/crypto/dsa/dsa_depr.c @@ -464,10 +466,10 @@ ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_md5.c ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_md5_sha1.c ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_mdc2.c ${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_sha.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_ripemd.c +${OPENSSL_SOURCE_DIR}/crypto/evp/legacy_wp.c ${OPENSSL_SOURCE_DIR}/crypto/evp/m_null.c -${OPENSSL_SOURCE_DIR}/crypto/evp/m_ripemd.c ${OPENSSL_SOURCE_DIR}/crypto/evp/m_sigver.c -${OPENSSL_SOURCE_DIR}/crypto/evp/m_wp.c ${OPENSSL_SOURCE_DIR}/crypto/evp/mac_lib.c ${OPENSSL_SOURCE_DIR}/crypto/evp/mac_meth.c ${OPENSSL_SOURCE_DIR}/crypto/evp/names.c @@ -486,6 +488,8 @@ ${OPENSSL_SOURCE_DIR}/crypto/evp/pkey_mac.c ${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_fn.c ${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_gn.c ${OPENSSL_SOURCE_DIR}/crypto/evp/pmeth_lib.c +${OPENSSL_SOURCE_DIR}/crypto/evp/signature.c +${OPENSSL_SOURCE_DIR}/crypto/ffc/ffc_params.c ${OPENSSL_SOURCE_DIR}/crypto/hmac/hm_ameth.c ${OPENSSL_SOURCE_DIR}/crypto/hmac/hmac.c ${OPENSSL_SOURCE_DIR}/crypto/idea/i_cbc.c @@ -529,6 +533,7 @@ ${OPENSSL_SOURCE_DIR}/crypto/provider_conf.c ${OPENSSL_SOURCE_DIR}/crypto/provider_core.c ${OPENSSL_SOURCE_DIR}/crypto/provider_predefined.c ${OPENSSL_SOURCE_DIR}/crypto/sparse_array.c +${OPENSSL_SOURCE_DIR}/crypto/self_test_core.c ${OPENSSL_SOURCE_DIR}/crypto/threads_none.c ${OPENSSL_SOURCE_DIR}/crypto/threads_pthread.c ${OPENSSL_SOURCE_DIR}/crypto/threads_win.c @@ -673,8 +678,8 @@ ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_crypt.c ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_err.c ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_pmeth.c ${OPENSSL_SOURCE_DIR}/crypto/sm2/sm2_sign.c -${OPENSSL_SOURCE_DIR}/crypto/sm3/m_sm3.c ${OPENSSL_SOURCE_DIR}/crypto/sm3/sm3.c +${OPENSSL_SOURCE_DIR}/crypto/sm3/legacy_sm3.c ${OPENSSL_SOURCE_DIR}/crypto/sm4/sm4.c ${OPENSSL_SOURCE_DIR}/crypto/srp/srp_lib.c ${OPENSSL_SOURCE_DIR}/crypto/srp/srp_vfy.c @@ -779,7 +784,11 @@ ${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509.c ${OPENSSL_SOURCE_DIR}/crypto/x509/x_x509a.c ${OPENSSL_SOURCE_DIR}/providers/implementations/asymciphers/rsa_enc.c ${OPENSSL_SOURCE_DIR}/providers/defltprov.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_null.c ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha1_hw.c +${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_cbc_hmac_sha256_hw.c ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ccm.c ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_ccm_hw.c ${OPENSSL_SOURCE_DIR}/providers/implementations/ciphers/cipher_aes_gcm.c From b470ff7aaedd10c24752d68fa7899d52f258980a Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Mon, 3 Feb 2020 06:11:12 +0300 Subject: [PATCH 0175/2007] Update array_functions.md --- docs/en/query_language/functions/array_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 8c9b2a7c151..35aa9c98dfb 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -898,7 +898,7 @@ Result: ``` ## arrayAUC(arr_scores, arr_labels) -Returns AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc); +Returns AUC (Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc). `arr_scores` represents scores prediction model gives, while `arr_labels` represents labels of samples, usually 1 for positive sample and 0 for negtive sample. From fde86203a7c20884d684c9066277fb2bd53a65e9 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 13:42:20 +0300 Subject: [PATCH 0176/2007] Fix StorageJoin --- dbms/src/Storages/StorageJoin.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index 2f664416b93..85ce13f15ed 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -354,7 +354,7 @@ private: res_columns.emplace_back(std::move(columns[i])); UInt64 num_rows = res_columns.at(0)->size(); - return Chunk(std::move(columns), num_rows); + return Chunk(std::move(res_columns), num_rows); } template From eae2ce194dc46ca19eee41ed3b4db16369a6ed90 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 14:22:21 +0300 Subject: [PATCH 0177/2007] Processors support for StorageNull reading. --- dbms/src/Storages/StorageNull.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index 347f1501ee1..58308e4a3e6 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include namespace DB @@ -20,7 +22,7 @@ class StorageNull : public ext::shared_ptr_helper, public IStorage public: std::string getName() const override { return "Null"; } - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo &, const Context & /*context*/, @@ -28,7 +30,9 @@ public: size_t, unsigned) override { - return { std::make_shared(getSampleBlockForColumns(column_names)) }; + Pipes pipes; + pipes.emplace_back(std::make_shared(getSampleBlockForColumns(column_names))); + return pipes; } BlockOutputStreamPtr write(const ASTPtr &, const Context &) override From a029e494382c82c5e8b7ce9c70289edcc6275e1c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 14:23:52 +0300 Subject: [PATCH 0178/2007] Processors support for StorageNull reading. --- dbms/src/Storages/StorageNull.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/src/Storages/StorageNull.h b/dbms/src/Storages/StorageNull.h index 58308e4a3e6..b0af1d547aa 100644 --- a/dbms/src/Storages/StorageNull.h +++ b/dbms/src/Storages/StorageNull.h @@ -35,6 +35,8 @@ public: return pipes; } + bool supportProcessorsPipeline() const override { return true; } + BlockOutputStreamPtr write(const ASTPtr &, const Context &) override { return std::make_shared(getSampleBlock()); From 257bb3b5994a86deb6734c35c7e0927161fcaa29 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 3 Feb 2020 15:08:40 +0300 Subject: [PATCH 0179/2007] add comments near DataPart code --- dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h | 3 +++ dbms/src/Storages/MergeTree/MergeTreeDataPartType.h | 8 ++++++++ .../Storages/MergeTree/MergeTreeDataPartWriterCompact.h | 1 + dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h | 1 + dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h | 2 ++ .../Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp | 8 +++++++- .../Storages/MergeTree/MergeTreeIndexGranularityInfo.h | 1 + 7 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index 1c751b04da1..cad49f34bcf 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -12,6 +12,9 @@ namespace DB { + +/// Writes data part to disk in different formats. +/// Calculates and serializes primary and skip indices if needed. class IMergeTreeDataPartWriter : private boost::noncopyable { public: diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h index 0f531351e0d..ed229414e3b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -2,11 +2,19 @@ namespace DB { + /// Types of data part format. enum class MergeTreeDataPartType { + /// Data of each is stored in one or several (for complex types) files. + /// Every data file is followed by marks file. WIDE, + + /// Data of all columns is stored in one file. Marks are also stored in single file. COMPACT, + + /// Format with buffering data in RAM. Not implemented yet. IN_MEMORY, + UNKNOWN, }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 8b39b522c17..089bb914c6d 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -3,6 +3,7 @@ namespace DB { +/// Writes data part in compact format. class MergeTreeDataPartWriterCompact : public IMergeTreeDataPartWriter { public: diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 9a49a6402e1..07002a0a80c 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -3,6 +3,7 @@ namespace DB { +/// Writes data part in wide format. class MergeTreeDataPartWriterWide : public IMergeTreeDataPartWriter { public: diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h index 2a9cbcf654c..5aefd0f102b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularity.h @@ -95,8 +95,10 @@ public: /// Add new mark with rows_count void appendMark(size_t rows_count); + /// Extends last mark by rows_count. void addRowsToLastMark(size_t rows_count); + /// Drops last mark if any exists. void popMark(); /// Add `size` of marks with `fixed_granularity` rows diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp index 967139638b1..165b80d23ab 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.cpp @@ -74,11 +74,17 @@ size_t MergeTreeIndexGranularityInfo::getMarkSizeInBytes(size_t columns_num) con if (type == MergeTreeDataPartType::WIDE) return is_adaptive ? getAdaptiveMrkSizeWide() : getNonAdaptiveMrkSizeWide(); else if (type == MergeTreeDataPartType::COMPACT) - return sizeof(UInt64) * (columns_num * 2 + 1); + return getAdaptiveMrkSizeCompact(columns_num); else throw Exception("Unknown part type", ErrorCodes::UNKNOWN_PART_TYPE); } +size_t getAdaptiveMrkSizeCompact(size_t columns_num) +{ + /// Each mark contains number of rows in granule and two offsets for every column. + return sizeof(UInt64) * (columns_num * 2 + 1); +} + std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type) { if (part_type == MergeTreeDataPartType::WIDE) diff --git a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h index 2c596401228..8b991140c0a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIndexGranularityInfo.h @@ -50,6 +50,7 @@ private: constexpr inline auto getNonAdaptiveMrkExtension() { return ".mrk"; } constexpr inline auto getNonAdaptiveMrkSizeWide() { return sizeof(UInt64) * 2; } constexpr inline auto getAdaptiveMrkSizeWide() { return sizeof(UInt64) * 3; } +inline size_t getAdaptiveMrkSizeCompact(size_t columns_num); std::string getAdaptiveMrkExtension(MergeTreeDataPartType part_type); } From e0eb2f542842b3fce5b9d86b6f4a29ed64ed7f1e Mon Sep 17 00:00:00 2001 From: Mikhail Korotov <55493615+millb@users.noreply.github.com> Date: Mon, 3 Feb 2020 16:18:00 +0300 Subject: [PATCH 0180/2007] Update Cluster.cpp --- dbms/src/Interpreters/Cluster.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index a972fe294d5..665ee5af1ed 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -158,7 +158,7 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string) const char * user_pw_end = strchr(full_string.data(), '@'); /// parsing with format [shard{shard_index}[_replica{replica_index}]] - if (!user_pw_end && startsWith("shard", full_string)) + if (!user_pw_end && startsWith(full_string, "shard")) { const char * underscore = strchr(full_string.data(), '_'); From 32aa1009387d4a6345cfa9c4771bb766b0ee3943 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 17:28:30 +0300 Subject: [PATCH 0181/2007] fix totals port for pipe. --- dbms/src/Processors/QueryPipeline.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dbms/src/Processors/QueryPipeline.cpp b/dbms/src/Processors/QueryPipeline.cpp index 187631aa552..9f9fc51b0ca 100644 --- a/dbms/src/Processors/QueryPipeline.cpp +++ b/dbms/src/Processors/QueryPipeline.cpp @@ -655,6 +655,9 @@ Pipe QueryPipeline::getPipe() && for (auto & storage : storage_holders) pipe.addStorageHolder(storage); + if (totals_having_port) + pipe.setTotalsPort(totals_having_port); + return pipe; } From 30586111a68e70ca81b209afe351792409b87fa7 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 17:50:19 +0300 Subject: [PATCH 0182/2007] fix totals port for pipe. --- .../src/Interpreters/ClusterProxy/SelectStreamFactory.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 2ee516852dc..4a1256f0746 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -272,7 +272,13 @@ void SelectStreamFactory::createForShard( } }; - res.emplace_back(std::make_shared("LazyShardWithLocalReplica", header, lazily_create_stream)); + auto add_totals = processed_stage == QueryProcessingStage::Complete; + auto source = std::make_shared("LazyShardWithLocalReplica", header, lazily_create_stream); + + if (add_totals) + source->addTotalsPort(); + + res.emplace_back(std::move(source)); } else emplace_remote_stream(); From a832a630d880620bcabccd1dc994140e1164d443 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 18:35:35 +0300 Subject: [PATCH 0183/2007] Update TreeExecutor. --- .../TreeExecutorBlockInputStream.cpp | 40 ++++++++++++++----- .../Executors/TreeExecutorBlockInputStream.h | 5 ++- 2 files changed, 35 insertions(+), 10 deletions(-) diff --git a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp index ee482d62f27..44967440957 100644 --- a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp +++ b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.cpp @@ -17,7 +17,7 @@ static void checkProcessorHasSingleOutput(IProcessor * processor) /// Check tree invariants (described in TreeExecutor.h). /// Collect sources with progress. -static void validateTree(const Processors & processors, IProcessor * root, std::vector & sources) +static void validateTree(const Processors & processors, IProcessor * root, IProcessor * totals_root, std::vector & sources) { std::unordered_map index; @@ -34,6 +34,8 @@ static void validateTree(const Processors & processors, IProcessor * root, std:: std::stack stack; stack.push(root); + if (totals_root) + stack.push(totals_root); while (!stack.empty()) { @@ -81,18 +83,33 @@ void TreeExecutorBlockInputStream::init() throw Exception("No processors were passed to TreeExecutorBlockInputStream.", ErrorCodes::LOGICAL_ERROR); root = &output_port.getProcessor(); + IProcessor * totals_root = nullptr; - validateTree(processors, root, sources_with_progress); + if (totals_port) + totals_root = &totals_port->getProcessor(); + + validateTree(processors, root, totals_root, sources_with_progress); input_port = std::make_unique(getHeader(), root); connect(output_port, *input_port); input_port->setNeeded(); + + if (totals_port) + { + input_totals_port = std::make_unique(totals_port->getHeader(), root); + connect(*totals_port, *input_totals_port); + input_totals_port->setNeeded(); + } } -void TreeExecutorBlockInputStream::execute() +void TreeExecutorBlockInputStream::execute(bool on_totals) { std::stack stack; - stack.push(root); + + if (on_totals) + stack.push(&totals_port->getProcessor()); + else + stack.push(root); auto prepare_processor = [](IProcessor * processor) { @@ -141,10 +158,6 @@ void TreeExecutorBlockInputStream::execute() break; } case IProcessor::Status::PortFull: - { - stack.pop(); - break; - } case IProcessor::Status::Finished: { stack.pop(); @@ -178,12 +191,21 @@ Block TreeExecutorBlockInputStream::readImpl() while (true) { if (input_port->isFinished()) + { + if (totals_port && !input_totals_port->isFinished()) + { + execute(true); + if (input_totals_port->hasData()) + totals = getHeader().cloneWithColumns(input_totals_port->pull().detachColumns()); + } + return {}; + } if (input_port->hasData()) return getHeader().cloneWithColumns(input_port->pull().detachColumns()); - execute(); + execute(false); } } diff --git a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h index 8787d3090c1..36f065bae45 100644 --- a/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h +++ b/dbms/src/Processors/Executors/TreeExecutorBlockInputStream.h @@ -29,6 +29,7 @@ public: for (auto & context : pipe.getContexts()) interpreter_context.emplace_back(context); + totals_port = pipe.getTotalsPort(); processors = std::move(pipe).detachProcessors(); init(); } @@ -49,9 +50,11 @@ protected: private: OutputPort & output_port; + OutputPort * totals_port = nullptr; Processors processors; IProcessor * root = nullptr; std::unique_ptr input_port; + std::unique_ptr input_totals_port; /// Remember sources that support progress. std::vector sources_with_progress; @@ -60,7 +63,7 @@ private: void init(); /// Execute tree step-by-step until root returns next chunk or execution is finished. - void execute(); + void execute(bool on_totals); /// Moved from pipe. std::vector> interpreter_context; From a0635ed390a267fd3ec274821f33f78fd05ee767 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 3 Feb 2020 15:46:25 +0300 Subject: [PATCH 0184/2007] better marks reading --- dbms/src/DataStreams/MarkInCompressedFile.h | 11 ++- .../MergeTree/MergeTreeMarksLoader.cpp | 80 ++++++++++++++++--- .../Storages/MergeTree/MergeTreeMarksLoader.h | 23 +++--- .../MergeTree/MergeTreeReaderCompact.cpp | 56 ++----------- .../MergeTree/MergeTreeReaderCompact.h | 1 - .../MergeTree/MergeTreeReaderStream.cpp | 60 +------------- .../MergeTree/MergeTreeReaderStream.h | 2 - 7 files changed, 101 insertions(+), 132 deletions(-) diff --git a/dbms/src/DataStreams/MarkInCompressedFile.h b/dbms/src/DataStreams/MarkInCompressedFile.h index 46d078f2b76..62886ffad57 100644 --- a/dbms/src/DataStreams/MarkInCompressedFile.h +++ b/dbms/src/DataStreams/MarkInCompressedFile.h @@ -40,6 +40,15 @@ struct MarkInCompressedFile }; -using MarksInCompressedFile = PODArray; +class MarksInCompressedFile : public PODArray +{ +public: + MarksInCompressedFile(size_t n) : PODArray(n) {} + + void read(ReadBuffer & buffer, size_t from, size_t count) + { + buffer.readStrict(reinterpret_cast(data() + from), count * sizeof(MarkInCompressedFile)); + } +}; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp index 8bdc8fc8307..452d46a4751 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.cpp @@ -1,50 +1,106 @@ #include +#include +#include +#include namespace DB { +namespace ErrorCodes +{ + extern const int CORRUPTED_DATA; + extern const int LOGICAL_ERROR; +} + MergeTreeMarksLoader::MergeTreeMarksLoader( MarkCache * mark_cache_, const String & mrk_path_, - const LoadFunc & load_func_, + size_t marks_count_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, bool save_marks_in_cache_, - size_t columns_num_) + size_t columns_in_mark_) : mark_cache(mark_cache_) , mrk_path(mrk_path_) - , load_func(load_func_) + , marks_count(marks_count_) + , index_granularity_info(index_granularity_info_) , save_marks_in_cache(save_marks_in_cache_) - , columns_num(columns_num_) {} + , columns_in_mark(columns_in_mark_) {} const MarkInCompressedFile & MergeTreeMarksLoader::getMark(size_t row_index, size_t column_index) { if (!marks) loadMarks(); - if (column_index >= columns_num) - throw Exception("Column index: " + toString(column_index) - + " is out of range [0, " + toString(columns_num) + ")", ErrorCodes::LOGICAL_ERROR); - return (*marks)[row_index * columns_num + column_index]; +#ifndef NDEBUG + if (column_index >= columns_in_mark) + throw Exception("Column index: " + toString(column_index) + + " is out of range [0, " + toString(columns_in_mark) + ")", ErrorCodes::LOGICAL_ERROR); +#endif + + return (*marks)[row_index * columns_in_mark + column_index]; +} + +MarkCache::MappedPtr MergeTreeMarksLoader::loadMarksImpl() +{ + /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. + auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); + + size_t file_size = Poco::File(mrk_path).getSize(); + size_t mark_size = index_granularity_info.getMarkSizeInBytes(columns_in_mark); + size_t expected_file_size = mark_size * marks_count; + + if (expected_file_size != file_size) + throw Exception( + "Bad size of marks file '" + mrk_path + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), + ErrorCodes::CORRUPTED_DATA); + + auto res = std::make_shared(marks_count * columns_in_mark); + + if (!index_granularity_info.is_adaptive) + { + /// Read directly to marks. + ReadBufferFromFile buffer(mrk_path, file_size, -1, reinterpret_cast(res->data())); + + if (buffer.eof() || buffer.buffer().size() != file_size) + throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); + } + else + { + ReadBufferFromFile buffer(mrk_path, file_size, -1); + size_t i = 0; + while (!buffer.eof()) + { + res->read(buffer, i * columns_in_mark, columns_in_mark); + buffer.seek(sizeof(size_t), SEEK_CUR); + ++i; + } + + if (i * mark_size != file_size) + throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); + } + res->protect(); + return res; } void MergeTreeMarksLoader::loadMarks() { - auto load = std::bind(load_func, mrk_path); if (mark_cache) { auto key = mark_cache->hash(mrk_path); if (save_marks_in_cache) { - marks = mark_cache->getOrSet(key, load); + auto callback = std::bind(&MergeTreeMarksLoader::loadMarksImpl, this); + marks = mark_cache->getOrSet(key, callback); } else { marks = mark_cache->get(key); if (!marks) - marks = load(); + marks = loadMarksImpl(); } } else - marks = load(); + marks = loadMarksImpl(); if (!marks) throw Exception("Failed to load marks: " + mrk_path, ErrorCodes::LOGICAL_ERROR); diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h index 316be9d051d..7e5b75ed407 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -3,19 +3,20 @@ namespace DB { +struct MergeTreeIndexGranularityInfo; + class MergeTreeMarksLoader { public: using MarksPtr = MarkCache::MappedPtr; - using LoadFunc = std::function; - MergeTreeMarksLoader() {} - - MergeTreeMarksLoader(MarkCache * mark_cache_, - const String & mrk_path_, - const LoadFunc & load_func_, + MergeTreeMarksLoader( + MarkCache * mark_cache_, + const String & mrk_path, + size_t marks_count_, + const MergeTreeIndexGranularityInfo & index_granularity_info_, bool save_marks_in_cache_, - size_t columns_num_ = 1); + size_t columns_num_in_mark_ = 1); const MarkInCompressedFile & getMark(size_t row_index, size_t column_index = 0); @@ -24,12 +25,14 @@ public: private: MarkCache * mark_cache = nullptr; String mrk_path; - LoadFunc load_func; + size_t marks_count; + const MergeTreeIndexGranularityInfo & index_granularity_info; bool save_marks_in_cache = false; - size_t columns_num; - MarksPtr marks; + size_t columns_num_in_mark_; + MarkCache::MappedPtr marks; void loadMarks(); + MarkCache::MappedPtr loadMarksImpl(); }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index e7027ec52d5..1d884bcab0e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -19,11 +19,14 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) - : IMergeTreeReader(data_part_, columns_ - , uncompressed_cache_, mark_cache_, mark_ranges_ - , settings_, avg_value_size_hints_) + : IMergeTreeReader(data_part_, columns_, + uncompressed_cache_, mark_cache_, mark_ranges_, + settings_, avg_value_size_hints_) + , marks_loader(mark_cache, + data_part->index_granularity_info.getMarksFilePath(path + MergeTreeDataPartCompact::DATA_FILE_NAME), + data_part->getMarksCount(), data_part->index_granularity_info, + settings.save_marks_in_cache, data_part->getColumns().size()) { - initMarksLoader(); size_t buffer_size = settings.max_read_buffer_size; const String full_data_path = path + MergeTreeDataPartCompact::DATA_FILE_NAME + MergeTreeDataPartCompact::DATA_FILE_EXTENSION; @@ -194,51 +197,6 @@ void MergeTreeReaderCompact::readData( } -void MergeTreeReaderCompact::initMarksLoader() -{ - if (marks_loader.initialized()) - return; - - size_t columns_num = data_part->getColumns().size(); - - auto load = [this, columns_num](const String & mrk_path) -> MarkCache::MappedPtr - { - size_t file_size = Poco::File(mrk_path).getSize(); - size_t marks_count = data_part->getMarksCount(); - size_t mark_size_in_bytes = data_part->index_granularity_info.getMarkSizeInBytes(columns_num); - - size_t expected_file_size = mark_size_in_bytes * marks_count; - if (expected_file_size != file_size) - throw Exception( - "Bad size of marks file '" + mrk_path + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), - ErrorCodes::CORRUPTED_DATA); - - /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. - auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - - auto res = std::make_shared(marks_count * columns_num); - - ReadBufferFromFile buffer(mrk_path, file_size); - size_t i = 0; - - while (!buffer.eof()) - { - buffer.readStrict(reinterpret_cast(res->data() + i * columns_num), sizeof(MarkInCompressedFile) * columns_num); - buffer.seek(sizeof(size_t), SEEK_CUR); - ++i; - } - - if (i * mark_size_in_bytes != file_size) - throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); - - res->protect(); - return res; - }; - - auto mrk_path = data_part->index_granularity_info.getMarksFilePath(path + MergeTreeDataPartCompact::DATA_FILE_NAME); - marks_loader = MergeTreeMarksLoader{mark_cache, std::move(mrk_path), load, settings.save_marks_in_cache, columns_num}; -} - void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) { MarkInCompressedFile mark = marks_loader.getMark(row_index, column_index); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 55a3ab15bac..29a97fe7c5a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -46,7 +46,6 @@ private: size_t next_mark = 0; std::optional> last_read_granule; - void initMarksLoader(); void seekToMark(size_t row_index, size_t column_index); void readData(const String & name, IColumn & column, const IDataType & type, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp index b70c3c8b573..b132bda0f9e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -16,7 +16,7 @@ namespace ErrorCodes MergeTreeReaderStream::MergeTreeReaderStream( - const String & path_prefix_,const String & data_file_extension_, size_t marks_count_, + const String & path_prefix_, const String & data_file_extension_, size_t marks_count_, const MarkRanges & all_mark_ranges, const MergeTreeReaderSettings & settings, MarkCache * mark_cache_, @@ -26,15 +26,13 @@ MergeTreeReaderStream::MergeTreeReaderStream( : path_prefix(path_prefix_), data_file_extension(data_file_extension_), marks_count(marks_count_) , mark_cache(mark_cache_), save_marks_in_cache(settings.save_marks_in_cache) , index_granularity_info(index_granularity_info_) + , marks_loader(mark_cache, index_granularity_info->getMarksFilePath(path_prefix), + marks_count, *index_granularity_info, save_marks_in_cache) { /// Compute the size of the buffer. size_t max_mark_range_bytes = 0; size_t sum_mark_range_bytes = 0; - /// Care should be taken to not load marks when the part is empty (marks_count == 0). - - initMarksLoader(); - for (const auto & mark_range : all_mark_ranges) { size_t left_mark = mark_range.begin; @@ -106,58 +104,6 @@ MergeTreeReaderStream::MergeTreeReaderStream( } -void MergeTreeReaderStream::initMarksLoader() -{ - if (marks_loader.initialized()) - return; - - auto load = [this](const String & mrk_path) -> MarkCache::MappedPtr - { - /// Memory for marks must not be accounted as memory usage for query, because they are stored in shared cache. - auto temporarily_disable_memory_tracker = getCurrentMemoryTrackerActionLock(); - - size_t file_size = Poco::File(mrk_path).getSize(); - size_t mark_size = index_granularity_info->getMarkSizeInBytes(); - - size_t expected_file_size = mark_size * marks_count; - if (expected_file_size != file_size) - throw Exception( - "Bad size of marks file '" + mrk_path + "': " + std::to_string(file_size) + ", must be: " + std::to_string(expected_file_size), - ErrorCodes::CORRUPTED_DATA); - - auto res = std::make_shared(marks_count); - - if (!index_granularity_info->is_adaptive) - { - /// Read directly to marks. - ReadBufferFromFile buffer(mrk_path, file_size, -1, reinterpret_cast(res->data())); - - if (buffer.eof() || buffer.buffer().size() != file_size) - throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); - } - else - { - ReadBufferFromFile buffer(mrk_path, file_size, -1); - size_t i = 0; - while (!buffer.eof()) - { - readIntBinary((*res)[i].offset_in_compressed_file, buffer); - readIntBinary((*res)[i].offset_in_decompressed_block, buffer); - buffer.seek(sizeof(size_t), SEEK_CUR); - ++i; - } - if (i * mark_size != file_size) - throw Exception("Cannot read all marks from file " + mrk_path, ErrorCodes::CANNOT_READ_ALL_DATA); - } - res->protect(); - return res; - }; - - auto mrk_path = index_granularity_info->getMarksFilePath(path_prefix); - marks_loader = MergeTreeMarksLoader{mark_cache, std::move(mrk_path), load, save_marks_in_cache}; -} - - void MergeTreeReaderStream::seekToMark(size_t index) { MarkInCompressedFile mark = marks_loader.getMark(index); diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h index 8356fed8382..ccadbcd8e4a 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -31,8 +31,6 @@ public: ReadBuffer * data_buffer; private: - void initMarksLoader(); - std::string path_prefix; std::string data_file_extension; From 663e944b33c0a1dbc8002a53fa6011d0a6dfd2f0 Mon Sep 17 00:00:00 2001 From: millb Date: Mon, 3 Feb 2020 19:03:42 +0300 Subject: [PATCH 0185/2007] Attempt --- dbms/src/Interpreters/Cluster.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index a972fe294d5..61b28252f8b 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -139,7 +139,8 @@ String Cluster::Address::toFullString() const { return ((shard_index == 0) ? "" : "shard" + std::to_string(shard_index)) + - ((replica_index == 0) ? "" : "_replica" + std::to_string(replica_index)); + ((replica_index == 0) ? "" : "_replica" + std::to_string(replica_index)) + + ((secure == Protocol::Secure::Enable) ? "+secure" : ""); } Cluster::Address Cluster::Address::fromFullString(const String & full_string) From 1785b27ae5f098bb78de82c0825491c8f8051f01 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 3 Feb 2020 19:37:00 +0300 Subject: [PATCH 0186/2007] fix build --- dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h index 7e5b75ed407..927e78ed8b9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h +++ b/dbms/src/Storages/MergeTree/MergeTreeMarksLoader.h @@ -28,7 +28,7 @@ private: size_t marks_count; const MergeTreeIndexGranularityInfo & index_granularity_info; bool save_marks_in_cache = false; - size_t columns_num_in_mark_; + size_t columns_in_mark; MarkCache::MappedPtr marks; void loadMarks(); From d8be70d709b2757006f913461d7c98250ee3faca Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 3 Feb 2020 19:39:36 +0300 Subject: [PATCH 0187/2007] Fix bug --- .../MergeTree/ReplicatedQueueAlterState.h | 16 ++++++++-------- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 3 ++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h index 6980a117b15..225581298e6 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h +++ b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h @@ -34,8 +34,8 @@ private: }; - std::deque queue; public: + std::deque queue; bool empty() const { return queue.empty(); @@ -51,13 +51,13 @@ public: if (queue.empty()) queue.emplace_back(alter_version, AlterState::APPLY_DATA_CHANGES); - if (queue.front().alter_version != alter_version) - { - throw Exception( - "Alter head has another version number " - + std::to_string(queue.front().alter_version) + " than ours " + std::to_string(alter_version), - ErrorCodes::LOGICAL_ERROR); - } + //if (queue.front().alter_version != alter_version) + //{ + // throw Exception( + // "Alter head has another version number " + // + std::to_string(queue.front().alter_version) + " than ours " + std::to_string(alter_version), + // ErrorCodes::LOGICAL_ERROR); + //} } void finishMetadataAlter(int alter_version, bool have_data_alter, std::unique_lock & /*state_lock*/) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 30d82184e5f..db4ee81e2a4 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2333,6 +2333,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() void StorageReplicatedMergeTree::mutationsFinalizingTask() { + LOG_DEBUG(log, "Trying to finalize mutations"); bool needs_reschedule = false; try @@ -3384,13 +3385,13 @@ void StorageReplicatedMergeTree::alter( Coordination::Responses results; int32_t rc = zookeeper->tryMulti(ops, results); - queue.pullLogsToQueue(zookeeper); //std::cerr << "Results size:" << results.size() << std::endl; //std::cerr << "Have mutation:" << have_mutation << std::endl; if (rc == Coordination::ZOK) { + queue.pullLogsToQueue(zookeeper); if (have_mutation) { //std::cerr << "In have mutation\n"; From bce3ec2a296cb32abac6b3f10e6e8127f36e9acd Mon Sep 17 00:00:00 2001 From: proller Date: Mon, 3 Feb 2020 19:46:17 +0300 Subject: [PATCH 0188/2007] Revert "Revert "Build fixes"" This reverts commit 9e0b40bf09e373135758737f4ba31e1f270d8429. --- cmake/find/h3.cmake | 3 +-- cmake/find/zlib.cmake | 2 +- cmake/tools.cmake | 8 +++++--- contrib/CMakeLists.txt | 13 ++++++++----- contrib/replxx-cmake/CMakeLists.txt | 5 ++++- dbms/CMakeLists.txt | 8 ++++++-- dbms/src/Functions/geoToH3.cpp | 6 +++++- dbms/src/Functions/h3EdgeAngle.cpp | 6 +++++- dbms/src/Functions/h3EdgeLengthM.cpp | 9 +++++++-- dbms/src/Functions/h3GetResolution.cpp | 6 +++++- dbms/src/Functions/h3IsValid.cpp | 6 +++++- dbms/src/Functions/h3kRing.cpp | 6 +++++- debian/pbuilder-hooks/B90test-server | 7 ------- 13 files changed, 57 insertions(+), 28 deletions(-) diff --git a/cmake/find/h3.cmake b/cmake/find/h3.cmake index 6abe1ee657c..e01f0269507 100644 --- a/cmake/find/h3.cmake +++ b/cmake/find/h3.cmake @@ -15,9 +15,8 @@ if (USE_INTERNAL_H3_LIBRARY) set (H3_LIBRARY h3) set (H3_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib/include) elseif (NOT MISSING_INTERNAL_H3_LIBRARY) - set (H3_INCLUDE_PATHS /usr/local/include/h3) find_library (H3_LIBRARY h3) - find_path (H3_INCLUDE_DIR NAMES h3api.h PATHS ${H3_INCLUDE_PATHS}) + find_path (H3_INCLUDE_DIR NAMES h3/h3api.h PATHS ${H3_INCLUDE_PATHS}) endif () if (H3_LIBRARY AND H3_INCLUDE_DIR) diff --git a/cmake/find/zlib.cmake b/cmake/find/zlib.cmake index 42cfce871d7..7fc30fbfe42 100644 --- a/cmake/find/zlib.cmake +++ b/cmake/find/zlib.cmake @@ -29,7 +29,7 @@ if (NOT ZLIB_FOUND AND NOT MISSING_INTERNAL_ZLIB_LIBRARY) set (ZLIB_INCLUDE_DIRS ${ZLIB_INCLUDE_DIR}) # for poco set (ZLIB_INCLUDE_DIRECTORIES ${ZLIB_INCLUDE_DIR}) # for protobuf set (ZLIB_FOUND 1) # for poco - if (USE_STATIC_LIBRARIES) + if (USE_STATIC_LIBRARIES AND DEFINED BUILD_SHARED_LIBS) set (ZLIB_LIBRARIES zlibstatic CACHE INTERNAL "") else () set (ZLIB_LIBRARIES zlib CACHE INTERNAL "") diff --git a/cmake/tools.cmake b/cmake/tools.cmake index 025aac79b53..6aaeb5b8a16 100644 --- a/cmake/tools.cmake +++ b/cmake/tools.cmake @@ -32,8 +32,9 @@ else () find_program (GOLD_PATH NAMES "ld.gold" "gold") endif () +if (NOT OS_FREEBSD) # We prefer LLD linker over Gold or BFD. -if (NOT LINKER_NAME) + if (NOT LINKER_NAME) if (LLD_PATH) if (COMPILER_GCC) # GCC driver requires one of supported linker names like "lld". @@ -43,9 +44,9 @@ if (NOT LINKER_NAME) set (LINKER_NAME ${LLD_PATH}) endif () endif () -endif () + endif () -if (NOT LINKER_NAME) + if (NOT LINKER_NAME) if (GOLD_PATH) if (COMPILER_GCC) set (LINKER_NAME "gold") @@ -53,6 +54,7 @@ if (NOT LINKER_NAME) set (LINKER_NAME ${GOLD_PATH}) endif () endif () + endif () endif () if (LINKER_NAME) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 999d1e96c9a..8fb4c612ecb 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -54,7 +54,6 @@ if (USE_INTERNAL_BTRIE_LIBRARY) endif () if (USE_INTERNAL_ZLIB_LIBRARY) - unset (BUILD_SHARED_LIBS CACHE) set (ZLIB_ENABLE_TESTS 0 CACHE INTERNAL "") set (SKIP_INSTALL_ALL 1 CACHE INTERNAL "") set (ZLIB_COMPAT 1 CACHE INTERNAL "") # also enables WITH_GZFILEOP @@ -70,10 +69,14 @@ if (USE_INTERNAL_ZLIB_LIBRARY) add_subdirectory (${INTERNAL_ZLIB_NAME}) # We should use same defines when including zlib.h as used when zlib compiled target_compile_definitions (zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP) - target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP) + if (TARGET zlibstatic) + target_compile_definitions (zlibstatic PUBLIC ZLIB_COMPAT WITH_GZFILEOP) + endif () if (ARCH_AMD64 OR ARCH_AARCH64) - target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK) - target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK) + target_compile_definitions (zlib PUBLIC X86_64 UNALIGNED_OK) + if (TARGET zlibstatic) + target_compile_definitions (zlibstatic PUBLIC X86_64 UNALIGNED_OK) + endif () endif () endif () @@ -116,7 +119,7 @@ function(mysql_support) endif() if (USE_INTERNAL_ZLIB_LIBRARY) set(ZLIB_FOUND ON) - set(ZLIB_LIBRARY zlibstatic) + set(ZLIB_LIBRARY ${ZLIB_LIBRARIES}) set(WITH_EXTERNAL_ZLIB ON) endif() add_subdirectory (mariadb-connector-c) diff --git a/contrib/replxx-cmake/CMakeLists.txt b/contrib/replxx-cmake/CMakeLists.txt index 1240eb56b39..cc428d957cf 100644 --- a/contrib/replxx-cmake/CMakeLists.txt +++ b/contrib/replxx-cmake/CMakeLists.txt @@ -45,7 +45,10 @@ if (ENABLE_REPLXX) endif () endif () - target_compile_options(replxx PUBLIC -Wno-documentation) + if (NOT (COMPILER_GCC AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9)) + target_compile_options(replxx PUBLIC -Wno-documentation) + endif () + target_compile_definitions(replxx PUBLIC USE_REPLXX=1) message (STATUS "Using replxx") diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index eeda7aa6a1f..16c38400ba2 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -176,8 +176,12 @@ elseif (COMPILER_GCC) add_cxx_compile_options(-Wsizeof-array-argument) # Warn for suspicious length parameters to certain string and memory built-in functions if the argument uses sizeof add_cxx_compile_options(-Wsizeof-pointer-memaccess) - # Warn about overriding virtual functions that are not marked with the override keyword - add_cxx_compile_options(-Wsuggest-override) + + if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 9) + # Warn about overriding virtual functions that are not marked with the override keyword + add_cxx_compile_options(-Wsuggest-override) + endif () + # Warn whenever a switch statement has an index of boolean type and the case values are outside the range of a boolean type add_cxx_compile_options(-Wswitch-bool) # Warn if a self-comparison always evaluates to true or false diff --git a/dbms/src/Functions/geoToH3.cpp b/dbms/src/Functions/geoToH3.cpp index 40c267d8656..7fc00f2ccbe 100644 --- a/dbms/src/Functions/geoToH3.cpp +++ b/dbms/src/Functions/geoToH3.cpp @@ -10,7 +10,11 @@ #include #include -#include +#if __has_include(

) +# include

+#else +# include +#endif namespace DB diff --git a/dbms/src/Functions/h3EdgeAngle.cpp b/dbms/src/Functions/h3EdgeAngle.cpp index 18abc864c3e..130081f50ed 100644 --- a/dbms/src/Functions/h3EdgeAngle.cpp +++ b/dbms/src/Functions/h3EdgeAngle.cpp @@ -7,7 +7,11 @@ # include # include -# include +# if __has_include(

) +# include

+# else +# include +# endif namespace DB diff --git a/dbms/src/Functions/h3EdgeLengthM.cpp b/dbms/src/Functions/h3EdgeLengthM.cpp index d3c0db4b4b5..72cbc98bc65 100644 --- a/dbms/src/Functions/h3EdgeLengthM.cpp +++ b/dbms/src/Functions/h3EdgeLengthM.cpp @@ -8,8 +8,13 @@ # include # include -# include -# include +# if __has_include(

) +# include

+# include

+# else +# include +# include +# endif namespace DB diff --git a/dbms/src/Functions/h3GetResolution.cpp b/dbms/src/Functions/h3GetResolution.cpp index dfa84e8e076..1334c7dab5a 100644 --- a/dbms/src/Functions/h3GetResolution.cpp +++ b/dbms/src/Functions/h3GetResolution.cpp @@ -7,7 +7,11 @@ # include # include -# include +# if __has_include(

) +# include

+# else +# include +# endif namespace DB diff --git a/dbms/src/Functions/h3IsValid.cpp b/dbms/src/Functions/h3IsValid.cpp index 0c4123caf00..26ec272a6ae 100644 --- a/dbms/src/Functions/h3IsValid.cpp +++ b/dbms/src/Functions/h3IsValid.cpp @@ -7,7 +7,11 @@ # include # include -# include +# if __has_include(

) +# include

+# else +# include +# endif namespace DB diff --git a/dbms/src/Functions/h3kRing.cpp b/dbms/src/Functions/h3kRing.cpp index 55571472c16..1b9ae132897 100644 --- a/dbms/src/Functions/h3kRing.cpp +++ b/dbms/src/Functions/h3kRing.cpp @@ -11,7 +11,11 @@ # include # include -# include +# if __has_include(

) +# include

+# else +# include +# endif namespace DB diff --git a/debian/pbuilder-hooks/B90test-server b/debian/pbuilder-hooks/B90test-server index 670b0c15fcc..e36c255f9fc 100755 --- a/debian/pbuilder-hooks/B90test-server +++ b/debian/pbuilder-hooks/B90test-server @@ -8,11 +8,6 @@ PACKAGE_INSTALL=${PACKAGE_INSTALL=1} TEST_PORT_RANDOM=${TEST_PORT_RANDOM=1} if [ "${PACKAGE_INSTALL}" ]; then - #for PKG in $(ls /tmp/buildd/*.deb | sed -e's,.*/,,;s,_.*,,' ); do - # apt-get install -y --force-yes "$PKG" ||: - # apt-get remove -y "$PKG" ||: - #done - dpkg --auto-deconfigure -i /tmp/buildd/*.deb ||: apt install -y -f --allow-downgrades ||: dpkg -l | grep clickhouse ||: @@ -45,8 +40,6 @@ export CLICKHOUSE_PORT_TCP_SECURE=${CLICKHOUSE_PORT_TCP_SECURE:=9440} export CLICKHOUSE_PORT_HTTPS=${CLICKHOUSE_PORT_HTTPS:=8443} if [ "${TEST_CONNECT}" ]; then - sed -i 's/ssl_conf = ssl_sect/#ssl_conf = ssl_sect/g' /etc/ssl/openssl.cnf ||: - [ "${TEST_PORT_RANDOM}" ] && echo "${CLICKHOUSE_PORT_HTTP}${CLICKHOUSE_PORT_TCP}${CLICKHOUSE_PORT_INTERSERVER}" > /etc/clickhouse-server/config.d/port.xml if [ "${TEST_SSL}" ]; then From c83387fbbc2f566bc9154c4591018728c85cacea Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 3 Feb 2020 19:52:53 +0300 Subject: [PATCH 0189/2007] Add angry test --- ...allel_alter_replicated_zookeeper.reference | 12 ++ ...076_parallel_alter_replicated_zookeeper.sh | 109 ++++++++++++++++++ 2 files changed, 121 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference create mode 100755 dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh diff --git a/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference b/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference new file mode 100644 index 00000000000..be716831212 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference @@ -0,0 +1,12 @@ +1725 +1725 +1725 +1725 +1725 +Starting alters +Finishing alters +1 +1 +1 +1 +1 diff --git a/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh new file mode 100755 index 00000000000..839ff932cc3 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + + +for i in {1..2}; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" +done + +for i in {1..2}; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/concurrent_alter_mt', '$i') ORDER BY key" +done + +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, toString(number) from numbers(10)" +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, toString(number) from numbers(10, 40)" + +for i in {1..2}; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_mt_$i" +done + +for i in {1..2}; do + $CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_mt_$i" +done + +INITIAL_SUM=`$CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_mt_1"` + +# This is just garbage thread with conflictings alter +# it additionally loads alters "queue". +function garbage_alter_thread() +{ + while true; do + REPLICA=$(($RANDOM % 2 + 1)) + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE concurrent_alter_mt_$REPLICA ADD COLUMN h String DEFAULT '0'; ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN h UInt64; ALTER TABLE concurrent_alter_mt_$REPLICA DROP COLUMN h;"; + done +} + + +# This alters mostly requires not only metadata change +# but also conversion of data. Also they are all compatible +# between each other, so can be executed concurrently. +function correct_alter_thread() +{ + TYPES=(Float64 String UInt8 UInt32) + while true; do + REPLICA=$(($RANDOM % 2 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done +} + +# This thread add some data to table. After we finish we can check, that +# all our data have same types. +# insert queries will fail sometime because of wrong types. +function insert_thread() +{ + + VALUES=(7.0 7 '7') + while true; do + REPLICA=$(($RANDOM % 2 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))" + sleep 0.$RANDOM + done +} + +echo "Starting alters" + +export -f garbage_alter_thread; +export -f correct_alter_thread; +export -f insert_thread; + +TIMEOUT=10 + +timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & +#timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & +#timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & + +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +#timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +#timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & + +# We don't want too many parts, just several alters per second +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +#timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +#timeout $TIMEOUT bash -c insert_thread 2> /dev/null & + +wait + +echo "Finishing alters" + +# This alter will finish all previous +$CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String" + +for i in {1..2}; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_mt_$i" +done + +for i in {1..2}; do + $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_mt_$i" + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" +done From 71f746e01ac4854c89f00562662bac796f42c69f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 20:55:06 +0300 Subject: [PATCH 0190/2007] Update SelectStreamFactory. --- .../Interpreters/ClusterProxy/SelectStreamFactory.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index 4a1256f0746..ac14aeeab0e 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -131,7 +131,10 @@ void SelectStreamFactory::createForShard( if (!table_func_ptr) stream->setMainTable(main_table); - res.emplace_back(std::make_shared(std::move(stream))); + auto source = std::make_shared(std::move(stream)); + source->addTotalsPort(); + + res.emplace_back(std::move(source)); }; const auto & settings = context.getSettingsRef(); @@ -272,11 +275,8 @@ void SelectStreamFactory::createForShard( } }; - auto add_totals = processed_stage == QueryProcessingStage::Complete; auto source = std::make_shared("LazyShardWithLocalReplica", header, lazily_create_stream); - - if (add_totals) - source->addTotalsPort(); + source->addTotalsPort(); res.emplace_back(std::move(source)); } From bc757f6b249713d6dffdc268992adc8cb0d80f9c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 21:01:41 +0300 Subject: [PATCH 0191/2007] Processors support for StorageS3 reading. --- dbms/src/Storages/StorageS3.cpp | 56 +++++++++++++++++++-------------- dbms/src/Storages/StorageS3.h | 4 ++- 2 files changed, 36 insertions(+), 24 deletions(-) diff --git a/dbms/src/Storages/StorageS3.cpp b/dbms/src/Storages/StorageS3.cpp index 0798820104b..2e4dd21b8a3 100644 --- a/dbms/src/Storages/StorageS3.cpp +++ b/dbms/src/Storages/StorageS3.cpp @@ -22,6 +22,8 @@ #include #include +#include +#include namespace DB @@ -34,23 +36,27 @@ namespace ErrorCodes namespace { - class StorageS3BlockInputStream : public IBlockInputStream + class StorageS3Source : public SourceWithProgress { public: - StorageS3BlockInputStream( + StorageS3Source( const String & format, - const String & name_, + String name_, const Block & sample_block, const Context & context, + const ColumnDefaults & column_defaults, UInt64 max_block_size, const CompressionMethod compression_method, const std::shared_ptr & client, const String & bucket, const String & key) - : name(name_) + : SourceWithProgress(sample_block), name(std::move(name_)) { read_buf = wrapReadBufferWithCompressionMethod(std::make_unique(client, bucket, key), compression_method); reader = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size); + + if (!column_defaults.empty()) + reader = std::make_shared(reader, column_defaults, context); } String getName() const override @@ -58,30 +64,34 @@ namespace return name; } - Block readImpl() override + Chunk generate() override { - return reader->read(); - } + if (!reader) + return {}; - Block getHeader() const override - { - return reader->getHeader(); - } + if (!initialized) + { + reader->readSuffix(); + initialized = true; + } - void readPrefixImpl() override - { - reader->readPrefix(); - } + if (auto block = reader->read()) + { + UInt64 num_rows = block.rows(); + return Chunk(block.getColumns(), num_rows); + } - void readSuffixImpl() override - { reader->readSuffix(); + reader.reset(); + + return {}; } private: String name; std::unique_ptr read_buf; BlockInputStreamPtr reader; + bool initialized = false; }; class StorageS3BlockOutputStream : public IBlockOutputStream @@ -158,7 +168,7 @@ StorageS3::StorageS3( } -BlockInputStreams StorageS3::read( +Pipes StorageS3::readWithProcessors( const Names & column_names, const SelectQueryInfo & /*query_info*/, const Context & context, @@ -166,21 +176,21 @@ BlockInputStreams StorageS3::read( size_t max_block_size, unsigned /*num_streams*/) { - BlockInputStreamPtr block_input = std::make_shared( + auto block_input = std::make_shared( format_name, getName(), getHeaderBlock(column_names), context, + getColumns().getDefaults(), max_block_size, chooseCompressionMethod(uri.endpoint, compression_method), client, uri.bucket, uri.key); - auto column_defaults = getColumns().getDefaults(); - if (column_defaults.empty()) - return {block_input}; - return {std::make_shared(block_input, column_defaults, context)}; + Pipes pipes; + pipes.emplace_back(std::move(block_input)); + return pipes; } BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const Context & /*context*/) diff --git a/dbms/src/Storages/StorageS3.h b/dbms/src/Storages/StorageS3.h index 9854f70a733..177269dcb21 100644 --- a/dbms/src/Storages/StorageS3.h +++ b/dbms/src/Storages/StorageS3.h @@ -46,7 +46,7 @@ public: return getSampleBlock(); } - BlockInputStreams read( + Pipes readWithProcessors( const Names & column_names, const SelectQueryInfo & query_info, const Context & context, @@ -54,6 +54,8 @@ public: size_t max_block_size, unsigned num_streams) override; + bool supportProcessorsPipeline() const override { return true; } + BlockOutputStreamPtr write(const ASTPtr & query, const Context & context) override; private: From ed71fb347e762d5ac5d102eb507a91154e21433f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 3 Feb 2020 21:14:26 +0300 Subject: [PATCH 0192/2007] Update SelectStreamFactory. --- .../ClusterProxy/SelectStreamFactory.cpp | 21 +++++++++++-------- .../Sources/SourceFromInputStream.cpp | 4 +++- .../Sources/SourceFromInputStream.h | 2 +- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index ac14aeeab0e..c9ba336d8fd 100644 --- a/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/dbms/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -75,7 +75,13 @@ Pipe createLocalStream(const ASTPtr & query_ast, const Block & header, const Con checkStackSize(); InterpreterSelectQuery interpreter{query_ast, context, SelectQueryOptions(processed_stage)}; - Pipe pipe = interpreter.executeWithProcessors().getPipe(); + auto pipeline = interpreter.executeWithProcessors(); + + pipeline.addSimpleTransform([&](const Block & source_header) + { + return std::make_shared( + source_header, header, ConvertingTransform::MatchColumnsMode::Name, context); + }); /** Materialization is needed, since from remote servers the constants come materialized. * If you do not do this, different types (Const and non-Const) columns will be produced in different threads, @@ -87,12 +93,7 @@ Pipe createLocalStream(const ASTPtr & query_ast, const Block & header, const Con */ /// return std::make_shared(stream); - auto converting = std::make_shared( - pipe.getHeader(), header,ConvertingTransform::MatchColumnsMode::Name, context); - - pipe.addSimpleTransform(std::move(converting)); - - return pipe; + return interpreter.executeWithProcessors().getPipe();; } static String formattedAST(const ASTPtr & ast) @@ -112,6 +113,8 @@ void SelectStreamFactory::createForShard( const Context & context, const ThrottlerPtr & throttler, Pipes & res) { + bool force_add_agg_info = processed_stage == QueryProcessingStage::WithMergeableState; + auto modified_query_ast = query_ast->clone(); if (has_virtual_shard_num_column) VirtualColumnUtils::rewriteEntityInAst(modified_query_ast, "_shard_num", shard_info.shard_num, "toUInt32"); @@ -131,7 +134,7 @@ void SelectStreamFactory::createForShard( if (!table_func_ptr) stream->setMainTable(main_table); - auto source = std::make_shared(std::move(stream)); + auto source = std::make_shared(std::move(stream), force_add_agg_info); source->addTotalsPort(); res.emplace_back(std::move(source)); @@ -275,7 +278,7 @@ void SelectStreamFactory::createForShard( } }; - auto source = std::make_shared("LazyShardWithLocalReplica", header, lazily_create_stream); + auto source = std::make_shared("LazyShardWithLocalReplica", header, lazily_create_stream, force_add_agg_info); source->addTotalsPort(); res.emplace_back(std::move(source)); diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.cpp b/dbms/src/Processors/Sources/SourceFromInputStream.cpp index a9b4ca96c04..45b00c8ee6f 100644 --- a/dbms/src/Processors/Sources/SourceFromInputStream.cpp +++ b/dbms/src/Processors/Sources/SourceFromInputStream.cpp @@ -14,8 +14,10 @@ SourceFromInputStream::SourceFromInputStream(BlockInputStreamPtr stream_, bool f init(); } -SourceFromInputStream::SourceFromInputStream(String name, Block header, std::function stream_builder_) +SourceFromInputStream::SourceFromInputStream( + String name, Block header, std::function stream_builder_, bool force_add_aggregating_info_) : ISourceWithProgress(std::move(header)) + , force_add_aggregating_info(force_add_aggregating_info_) , stream_builder(std::move(stream_builder_)) , source_name(std::move(name)) { diff --git a/dbms/src/Processors/Sources/SourceFromInputStream.h b/dbms/src/Processors/Sources/SourceFromInputStream.h index bb043c03006..1a6cc0c5e46 100644 --- a/dbms/src/Processors/Sources/SourceFromInputStream.h +++ b/dbms/src/Processors/Sources/SourceFromInputStream.h @@ -13,7 +13,7 @@ class SourceFromInputStream : public ISourceWithProgress public: explicit SourceFromInputStream(BlockInputStreamPtr stream_, bool force_add_aggregating_info_ = false); /// Constructor which works like LazyBlockInputStream. First 'generate' method creates stream using callback. - SourceFromInputStream(String name, Block header, std::function stream_builder_); + SourceFromInputStream(String name, Block header, std::function stream_builder_, bool force_add_aggregating_info_ = false); String getName() const override { return source_name.empty() ? "SourceFromInputStream" : source_name; } Status prepare() override; From ad9cdd080cfcfe36c727ff77d34c5a46ad1a6e4b Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 4 Feb 2020 13:59:26 +0300 Subject: [PATCH 0193/2007] Fix everything, add docs. --- dbms/src/DataTypes/DataTypeNumberBase.cpp | 2 +- .../TableFunctions/TableFunctionRandom.cpp | 440 ++++++++++++------ .../01072_random_table_function.reference | 207 ++++++++ .../01072_random_table_function.sql | 133 +++++- .../table_functions/generate.md | 38 ++ .../table_functions/generate.md | 1 + .../table_functions/generate.md | 37 ++ .../table_functions/generate.md | 1 + 8 files changed, 705 insertions(+), 154 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01072_random_table_function.reference create mode 100644 docs/en/query_language/table_functions/generate.md create mode 120000 docs/ja/query_language/table_functions/generate.md create mode 100644 docs/ru/query_language/table_functions/generate.md create mode 120000 docs/zh/query_language/table_functions/generate.md diff --git a/dbms/src/DataTypes/DataTypeNumberBase.cpp b/dbms/src/DataTypes/DataTypeNumberBase.cpp index 90356817730..ce01269bc4d 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.cpp +++ b/dbms/src/DataTypes/DataTypeNumberBase.cpp @@ -257,7 +257,7 @@ template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; -template class DataTypeNumberBase; +template class DataTypeNumberBase; // used only in UUID template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index 1391ecdc74b..d70c8a73c63 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -4,8 +4,17 @@ #include #include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include #include @@ -34,222 +43,355 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -MutableColumnPtr createColumnWithRandomData(DataTypePtr type, UInt64 limit) +void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, + UInt64 max_array_length, UInt64 max_string_length, UInt64 random_seed) { TypeIndex idx = type->getTypeId(); - MutableColumnPtr column = type->createColumn(); + if (!random_seed) + random_seed = randomSeed(); + (void) max_string_length; switch (idx) { case TypeIndex::Nothing: - for (UInt64 i = 0; i < limit; ++i) - { - column->insertDefault(); - } throw Exception("Random Generator not implemented for type 'Nothing'.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::UInt8: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt16: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg64 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt128: - throw Exception("Random Generator not implemented for type 'UInt128'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("There is no DataType 'UInt128' support.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::Int8: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int16: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg64 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int128: - throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("There is no DataType 'Int128' support.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::Float32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + double d = 1.0; + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - double d; - for (UInt64 i = 0; i < limit; ++i) - { - d = std::numeric_limits::max(); - column->insert( (d / pcg32::max()) * generator() ); - } + d = std::numeric_limits::max(); + data[i] = (d / pcg32::max()) * generator(); } break; + } case TypeIndex::Float64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + double d = 1.0; + for (UInt64 i = 0; i < limit; ++i) { - pcg64 generator(randomSeed()); - double d; - for (UInt64 i = 0; i < limit; ++i) - { - d = std::numeric_limits::max(); - column->insert( (d / pcg64::max()) * generator() ); - } + d = std::numeric_limits::max(); + data[i] = (d / pcg64::max()) * generator(); } break; + } case TypeIndex::Date: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::DateTime: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::DateTime64: + { + UInt32 scale; + if (auto * ptype = typeid_cast(type.get())) + scale = ptype->getScale(); + else + throw Exception("Static cast to DataTypeDateTime64 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - UInt32 scale; - if (auto * ptype = typeid_cast(type.get())) - scale = ptype->getScale(); - else - throw Exception("Static cast to DataTypeDateTime64 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - UInt32 fractional = static_cast(generator()) % intExp10(scale); - UInt32 whole = static_cast(generator()); - DateTime64 dt = DecimalUtils::decimalFromComponents(whole, fractional, scale); - column->insert(DecimalField(dt, scale)); - } + UInt32 fractional = static_cast(generator()) % intExp10(scale); + UInt32 whole = static_cast(generator()); + DateTime64 dt = DecimalUtils::decimalFromComponents(whole, fractional, scale); + data[i] = dt; } break; + } case TypeIndex::String: - throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::FixedString: - throw Exception("Random Generator not implemented for type 'FixedString'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Enum8: - throw Exception("Random Generator not implemented for type 'Enum8'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Enum16: - throw Exception("Random Generator not implemented for type 'Enum16'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Decimal32: - { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } - } - break; - case TypeIndex::Decimal64: - { - pcg64 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } - } - break; - case TypeIndex::Decimal128: - throw Exception("Random Generator not implemented for type 'Decimal128'.", ErrorCodes::NOT_IMPLEMENTED); -/* - { - UInt32 scale = 0; - if (auto * ptype = typeid_cast *>(type.get())) - scale = ptype->getScale(); - else - throw Exception("Static cast to Decimal128 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + { + auto & column_string = typeid_cast(column); + auto & offsets = column_string.getOffsets(); + auto & chars = column_string.getChars(); - pcg128_once_insecure generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(DecimalField(static_cast(generator()), scale)); - } - } - break; -*/ - case TypeIndex::UUID: + UInt64 offset = 0; { - pcg128_once_insecure generator(randomSeed()); + pcg32 generator(random_seed); + offsets.resize(limit); for (UInt64 i = 0; i < limit; ++i) { - column->insert(static_cast(generator())); + offset += 1 + static_cast(generator()) % max_string_length; + offsets[i] = offset - 1; + } + chars.resize(offset); + for (UInt64 i = 0; i < offset; ++i) { + chars[i] = 32 + generator() % 95; + } + // add terminating zero char + for (auto & i : offsets) + { + chars[i] = 0; } } break; + } + case TypeIndex::FixedString: + { + auto & column_string = typeid_cast(column); + size_t len = column_string.sizeOfValueIfFixed(); + auto & chars = column_string.getChars(); + + UInt64 num_chars = static_cast(len) * limit; + { + pcg32 generator(random_seed); + chars.resize(num_chars); + for (UInt64 i = 0; i < num_chars; ++i) { + chars[i] = static_cast(generator()); + } + } + break; + } + case TypeIndex::Enum8: + { + auto values = typeid_cast *>(type.get())->getValues(); + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + + UInt8 size = values.size(); + UInt8 off; + for (UInt64 i = 0; i < limit; ++i) + { + off = static_cast(generator()) % size; + data[i] = values[off].second; + } + break; + } + case TypeIndex::Enum16: + { + auto values = typeid_cast *>(type.get())->getValues(); + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + + UInt16 size = values.size(); + UInt8 off; + for (UInt64 i = 0; i < limit; ++i) + { + off = static_cast(generator()) % size; + data[i] = values[off].second; + } + break; + } + case TypeIndex::Decimal32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) + { + data[i] = static_cast(generator()); + } + break; + } + case TypeIndex::Decimal64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) + { + data[i] = static_cast(generator()); + } + break; + } + case TypeIndex::Decimal128: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) + { + Int128 x = static_cast(generator()) << 64 | static_cast(generator()); + data[i] = x; + } + } + break; + case TypeIndex::UUID: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { + auto x = UInt128(generator(), generator()); + data[i] = x; + } + } + break; case TypeIndex::Array: - throw Exception("Random Generator not implemented for type 'Array'.", ErrorCodes::NOT_IMPLEMENTED); + { + auto & column_array = typeid_cast(column); + auto nested_type = typeid_cast(type.get())->getNestedType(); + + auto & offsets = column_array.getOffsets(); + IColumn & data = column_array.getData(); + + UInt64 offset = 0; + { + pcg32 generator(random_seed); + offsets.resize(limit); + for (UInt64 i = 0; i < limit; ++i) { + offset += static_cast(generator()) % max_array_length; + offsets[i] = offset; + } + } + fillColumnWithRandomData(data, nested_type, offset, max_array_length, max_string_length, random_seed); + break; + } case TypeIndex::Tuple: - throw Exception("Random Generator not implemented for type 'Tuple'.", ErrorCodes::NOT_IMPLEMENTED); + { + auto &column_tuple = typeid_cast(column); + auto elements = typeid_cast(type.get())->getElements(); + + for (size_t i = 0; i < column_tuple.tupleSize(); ++i) + { + fillColumnWithRandomData(column_tuple.getColumn(i), elements[i], limit, max_array_length, max_string_length, random_seed); + } + break; + } case TypeIndex::Set: - throw Exception("Random Generator not implemented for type 'Set'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Type 'Set' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); case TypeIndex::Interval: throw Exception("Type 'Interval' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); case TypeIndex::Nullable: - throw Exception("Random Generator not implemented for type 'Nullable'.", ErrorCodes::NOT_IMPLEMENTED); + { + auto & column_nullable = typeid_cast(column); + auto nested_type = typeid_cast(type.get())->getNestedType(); + + auto & null_map = column_nullable.getNullMapData(); + IColumn & nested_column = column_nullable.getNestedColumn(); + + fillColumnWithRandomData(nested_column, nested_type, limit, max_array_length, max_string_length, random_seed); + + pcg32 generator(random_seed); + null_map.resize(limit); + for (UInt64 i = 0; i < limit; ++i) { + null_map[i] = generator() < 1024; + } + break; + } case TypeIndex::Function: - throw Exception("Random Generator not implemented for type 'Function'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Type 'Funclion' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); case TypeIndex::AggregateFunction: throw Exception("Random Generator not implemented for type 'AggregateFunction'.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::LowCardinality: throw Exception("Random Generator not implemented for type 'LowCardinality'.", ErrorCodes::NOT_IMPLEMENTED); } - return column; } StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const @@ -261,30 +403,48 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C ASTs & args = args_func.at(0)->children; - if (args.size() > 2) - throw Exception("Table function '" + getName() + "' requires one or two arguments: structure (and limit).", + if (args.size() > 5) + throw Exception("Table function '" + getName() + "' requires at most five arguments: "\ + " structure, limit, max_array_length, max_string_length, random_seed.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); /// Parsing first argument as table structure and creating a sample block std::string structure = args[0]->as().value.safeGet(); UInt64 limit = 1; + UInt64 max_array_length = 10; + UInt64 max_string_length = 10; + UInt64 random_seed = 0; // zero for random + /// Parsing second argument if present - if (args.size() == 2) + if (args.size() >= 2) limit = args[1]->as().value.safeGet(); if (!limit) throw Exception("Table function '" + getName() + "' limit should not be 0.", ErrorCodes::BAD_ARGUMENTS); + if (args.size() >= 3) + max_array_length = args[1]->as().value.safeGet(); + + if (args.size() >= 4) + max_string_length = args[1]->as().value.safeGet(); + + if (args.size() == 5) + random_seed = args[1]->as().value.safeGet(); + ColumnsDescription columns = parseColumnsListFromString(structure, context); Block res_block; for (const auto & name_type : columns.getOrdinary()) { - MutableColumnPtr column = createColumnWithRandomData(name_type.type, limit); + MutableColumnPtr column = name_type.type->createColumn(); res_block.insert({std::move(column), name_type.type, name_type.name}); } + for (auto & ctn : res_block.getColumnsWithTypeAndName()) + { + fillColumnWithRandomData(ctn.column->assumeMutableRef(), ctn.type, limit, max_array_length, max_string_length, random_seed); + } auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); return res; diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.reference b/dbms/tests/queries/0_stateless/01072_random_table_function.reference new file mode 100644 index 00000000000..93ea1861756 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.reference @@ -0,0 +1,207 @@ +Enum8(\'hello\' = 1, \'world\' = 5) +world +hello +world +hello +hello +hello +world +hello +hello +hello +Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) +['world','hello','world','hello','hello','hello','world','hello','hello'] +['hello','world','world','hello','world','world'] +['hello','world','hello','hello','world','world','world'] +['world','world','world','world','world','world','hello','hello'] +['world','hello'] +['hello','hello'] +['world'] +['hello','hello'] +['hello','hello'] +['hello','world','hello','hello','world','world','world','world'] +Nullable(Enum16(\'o\' = -200, \'h\' = 1, \'w\' = 5)) +o +w +w +w +h +w +h +h +w +o +UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 +2254772619926532955 2254772619926532955 1234817989 1234817989 54213 -11323 197 -59 +9120028858397505560 9120028858397505560 1171957426 1171957426 42674 -22862 178 -78 +4555697903102013946 4555697903102013946 275100647 275100647 46055 -19481 231 -25 +5784362079052877875 5784362079052877875 1033685688 1033685688 51896 -13640 184 -72 +11035971995277520997 -7410772078432030619 180895192 180895192 15832 15832 216 -40 +7901646768096461004 7901646768096461004 135557292 135557292 28844 28844 172 -84 +6733841386518201279 6733841386518201279 716914271 716914271 15967 15967 95 95 +7736560050027905187 7736560050027905187 1012211222 1012211222 7702 7702 22 22 +2199287578947862030 2199287578947862030 2185722662 -2109244634 31526 31526 38 38 +3019483913099890467 3019483913099890467 2647224658 -1647742638 29010 29010 82 82 +Date DateTime DateTime(\'Europe/Moscow\') +2106-02-07 2009-02-16 23:59:49 2009-02-16 23:59:49 +2086-11-02 2007-02-20 10:43:46 2007-02-20 10:43:46 +2096-02-04 1978-09-20 03:50:47 1978-09-20 03:50:47 +2106-02-07 2002-10-04 02:54:48 2002-10-04 02:54:48 +2013-05-07 1975-09-25 19:39:52 1975-09-25 19:39:52 +2048-12-21 1974-04-19 01:48:12 1974-04-19 01:48:12 +2013-09-19 1992-09-19 18:51:11 1992-09-19 18:51:11 +1991-02-02 2002-01-28 12:47:02 2002-01-28 12:47:02 +2056-04-25 2039-04-06 20:11:02 2039-04-06 20:11:02 +2049-06-05 2053-11-20 07:10:58 2053-11-20 07:10:58 +DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') +2007-02-20 10:43:46.989 2007-02-20 10:43:46.817989 2007-02-20 10:43:46.817989 +2002-10-04 02:54:48.647 2002-10-04 02:54:48.100647 2002-10-04 02:54:48.100647 +1974-04-19 01:48:12.192 1974-04-19 01:48:12.895192 1974-04-19 01:48:12.895192 +2002-01-28 12:47:02.271 2002-01-28 12:47:02.914271 2002-01-28 12:47:02.914271 +2053-11-20 07:10:58.662 2053-11-20 07:10:58.722662 2053-11-20 07:10:58.722662 +1986-04-08 19:07:15.849 1986-04-08 19:07:15.510849 1986-04-08 19:07:15.510849 +2081-03-06 04:00:55.914 2081-03-06 04:00:55.448914 2081-03-06 04:00:55.448914 +1979-01-20 20:39:20.939 1979-01-20 20:39:20.162939 1979-01-20 20:39:20.162939 +2063-07-18 01:46:10.215 2063-07-18 01:46:10.908215 2063-07-18 01:46:10.908215 +1996-11-02 14:35:41.110 1996-11-02 14:35:41.183110 1996-11-02 14:35:41.183110 +Float32 Float64 +9.783235e37 2.1973467205491123e307 +9.285203e37 8.887754501811354e307 +2.1795718e37 4.4396706606805647e307 +8.1897013e37 5.637042481600483e307 +1.4331993e37 1.07549012514996e308 +1.0739954e37 7.700402896226395e307 +5.67998e37 6.562339881458101e307 +8.019563e37 7.539520705557441e307 +1.7317079e38 2.143274805821858e307 +2.0973474e38 2.9425818885529257e307 +Decimal32(4) Decimal64(8) Decimal64(8) +123481.7989 22547726199.26532955 4159321346419233104838.6879832895010840 +117195.7426 91200288583.97505560 8403779329565810688767.7049545291714611 +27510.0647 45556979031.02013946 -13670461591942827725055.0250490776469300 +103368.5688 57843620790.52877875 12421744869005473959544.2499747955622051 +18089.5192 -74107720784.32030619 4056969511333950153663.4915186231430947 +13555.7292 79016467680.96461004 -8819413736166121578589.4583420666183888 +71691.4271 67338413865.18201279 13058329479868658041313.8432372419860363 +101221.1222 77365600500.27905187 -4693380431928321782727.0243506636623202 +-210924.4634 21992875789.47862030 13765369952377767241248.9441272127848016 +-164774.2638 30194839130.99890467 -13890064946313418575619.0315227826809939 +UUID +1f4a8fc0-63ff-735b-7e90-d9ed3e183818 +3f39171b-1263-31fa-5046-2ea9fe2fd033 +9927a60f-01ac-f065-6da8-49def100c0cc +5d736910-493d-c3bf-6b5d-c8601d6440a3 +1e857066-961d-be0e-29e7-5c9efd534f23 +bda66d4f-737b-3622-b60f-aa27fe38ff30 +623d6d82-4422-2885-297f-7b2fec54178b +dcb0e0ca-3a43-5f2e-556e-7945df65729e +678f2360-36ac-d439-8d6d-f92295887e50 +9780b53e-dc0f-4a21-bdb3-9798af1913ad +Tuple(Int32, Int64) +(1234817989,2254772619926532955) +(1171957426,9120028858397505560) +(275100647,4555697903102013946) +(1033685688,5784362079052877875) +(180895192,-7410772078432030619) +(135557292,7901646768096461004) +(716914271,6733841386518201279) +(1012211222,7736560050027905187) +(-2109244634,2199287578947862030) +(-1647742638,3019483913099890467) +Array(Int8) +[-59,-78,-25,-72,-40,-84,95,22,38] +[82,65,35,-110,-57,-69] +[72,119,-78,-58,13,39,-71] +[81,107,-11,-63,-59,69,-80,-122] +[87,-76] +[22,-84] +[-45] +[-40,84] +[-104,-86] +[-36,123,44,60,5,25,-5,-127] +Array(Nullable(Int32)) +[1234817989,1171957426,275100647,1033685688,180895192,135557292,716914271,1012211222,-2109244634] +[-1647742638,319510849,513356835,-1966518382,-786518841,269162939] +[285701960,1943908215,-1343029326,1474183110,846934541,1007818023,-1664171079] +[195050577,371018347,734173429,2001591233,-1812297275,1172704837,-728923984,774864518] +[-462583209,-1520633676] +[-638906858,1986832300] +[378774483] +[-1399152424,-953863084] +[733724312,-23652950] +[371735004,462118779,148602156,-1055384004,-1041274619,247762201,522289659,822210177] +Array(Nullable(UUID)) +['1f4a8fc0-63ff-735b-7e90-d9ed3e183818','3f39171b-1263-31fa-5046-2ea9fe2fd033','9927a60f-01ac-f065-6da8-49def100c0cc','5d736910-493d-c3bf-6b5d-c8601d6440a3','1e857066-961d-be0e-29e7-5c9efd534f23','bda66d4f-737b-3622-b60f-aa27fe38ff30','623d6d82-4422-2885-297f-7b2fec54178b','dcb0e0ca-3a43-5f2e-556e-7945df65729e','678f2360-36ac-d439-8d6d-f92295887e50'] +['9780b53e-dc0f-4a21-bdb3-9798af1913ad','c79810de-3635-d333-5ca1-7a81ab302b25','1c756bca-4438-3f17-a766-c8bcbe3ba400','d9072738-ac93-7ed6-167b-3c3c66d35a18','b1e8dec2-de29-3c9f-aaf2-f78fd92df3ce','9cd25f9f-3c0d-f43d-5a46-0194f0be04dd'] +['10a4718d-ab8c-49c6-c785-66ccf112f7d5','02ac2bf5-5634-a5a8-9a18-05ce8d1fb583','8037a13d-2004-08f2-f831-fa2387f5c29a','a99c4373-1121-2691-ecbb-216adbd748c7','ef0986ff-5031-0353-2f21-1de3ea53af08','778064a7-653b-ef7b-c77b-4d769b12b917','a1607e6f-691a-0ff0-b0b3-e454dae7bef7'] +['71c1b47a-c0eb-42b5-eecd-18dc585284fd','72bbf272-9ec5-09ec-f339-b5dac55c037b','26e5bce5-43f7-59b0-84c6-ef509f4c45eb','305fcbff-c366-2033-a8c5-d648f236e754','3a0d329f-f897-84e9-9e87-9501a713e63d','54bda20c-d5cd-a08a-c078-3c4fd81f4f55','43f549d1-3e5b-d5bf-ed32-b4850648bdc8','7eb6ac4f-06e0-ff48-6330-3c7afa5f2644'] +['17b9a4a5-fef8-a3f9-5af4-3b6e67ca62c9','3f524d8e-320d-00dc-c210-e199206550db'] +['005c592e-5081-9f3d-1fcb-5a9e82f39f97','29cf228d-b325-4a34-3eff-e80494a79260'] +['6c08b54b-8cf8-b96d-f087-8b54f5e72d0e'] +['7122e162-ab8b-a84a-6b71-c0846cf0204d','51c1de1a-24c7-18d6-39ed-e9023205610c'] +['f09d6779-1106-d667-e7c9-9a0cad544afe','62060fec-ee13-7c66-5da4-02c8f4d50dc9'] +['df1d0d54-d639-9c9b-2070-622fc9d82203','f23ef5b9-3797-9b0e-b8ac-67ea31b99c3e','e48afe73-9e22-7439-afed-d53b6ea204f4','d7f1ab47-4928-7623-283e-fb3f16aebeba','ea270407-d32f-a407-add2-3ae2d1113ccb','c43e9fff-2980-a1d1-f1bb-ff94d3cffbc2','a0cd54e6-0a2d-07ec-88ad-4f5d29c15b06','5e93413f-2eb9-5363-17ab-e2215b8b19e0'] +Tuple(Int32, Array(Int64)) +(1234817989,[2254772619926532955,9120028858397505560,4555697903102013946,5784362079052877875,-7410772078432030619,7901646768096461004,6733841386518201279,7736560050027905187,2199287578947862030]) +(1171957426,[3019483913099890467,-4781013766399904222,-5327852745410412752,7078934595552553093,2990244123355912075,-2544286630298820818]) +(275100647,[6155991081669718686,7462222003717329977,-8255668614967296432,-7529819295378879967,-4777308097681484883,-4064480117123591373,6674750820081216293]) +(1033685688,[2050663721809231639,-6384194708780112896,-2808232718275215658,1619954721090656792,-5627002805867168609,-6128563945701772338,-7146544521171569603,6504888450989032669]) +(180895192,[1199208254069819846,-4069733657855461419]) +(135557292,[192577216783361448,-7343112807738526333]) +(716914271,[-9207713629233477390]) +(1012211222,[-562393447932771686,-6225026423445182831]) +(-2109244634,[-1388479317275096889,-1222297392734207149]) +(-1647742638,[3396028458740199176,8610993157653131131,-4072576266223306473,-6818310818869145616,-5713972449102020873,8197031236106666677,-1239306987803343619,8267468115072584172]) +FixedString(4) +Ų +ج_ +&RA# +ǻH +w\r +\'Qk +E +W + +T +String +String +String +String +String +String +String +String +String +String +)/VC)%f9 +\0ih|;B +\0J"Z,kd +\0m"m]$35 +\00 +\0( +\0 +\0g +\0> +\0XjbW:s< +Nullable(String) +)/VC)%f9 +\0ih|;B +\0J"Z,kd +\0m"m]$35 +\00 +\0( +\0 +\0g +\0> +\0XjbW:s< +Array(String) +['(|ZVAg2F','\0GXjbW','\0<^guT(','\0y M$lZ0','\03','\0p','\0','\0i','\0P'] +['\0"}YRG%B','\0T3(E^> p','\0JTaj','\0)*3','\0k%=p','\0Yub$81`X'] +['','\0\\p]|]','\05','\0k$C/pnA'] +['\0ryz{*p',''] +['\07`mjt*G',''] +['\0~g'] +['\0k','\0 '] +['\0F','\0&h diff --git a/docs/ja/query_language/table_functions/generate.md b/docs/ja/query_language/table_functions/generate.md new file mode 120000 index 00000000000..de0b0a41754 --- /dev/null +++ b/docs/ja/query_language/table_functions/generate.md @@ -0,0 +1 @@ +en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/ru/query_language/table_functions/generate.md b/docs/ru/query_language/table_functions/generate.md new file mode 100644 index 00000000000..11d7f7073a9 --- /dev/null +++ b/docs/ru/query_language/table_functions/generate.md @@ -0,0 +1,37 @@ +# generate + +Генерирует случайные данные с заданной схемой. +Позволяет заполнять тестовые таблицы данными. +Поддерживает все типы данных, которые могут храниться в таблице, за исключением LowCardinality, AggregateFunction. + +```sql +generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); +``` + +**Входные параметры** +- `name` — название соответствующего столбца. +- `TypeName` — тип соответствующего столбца. +- `limit` — количество строк для генерации. +- `max_array_length` — максимальная длина массива для всех сгенерированных массивов. По умолчанию `10`. +- `max_string_length` — максимальная длина строки для всех генерируемых строк. По умолчанию `10`. +- `random_seed` — укажите состояние генератора случайных чисел вручную, чтобы получить стабильные результаты. По умолчанию `0` - генератор инициализируется случайным состоянием. + +**Возвращаемое значение** + +Объект таблицы с запрошенной схемой. + +## Пример + + +```sql +SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); +``` +```text +┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ +│ [77] │ -124167.6723 │ ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') │ +│ [32,110] │ -141397.7312 │ ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') │ +│ [68] │ -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │ +└──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/generate/) diff --git a/docs/zh/query_language/table_functions/generate.md b/docs/zh/query_language/table_functions/generate.md new file mode 120000 index 00000000000..de0b0a41754 --- /dev/null +++ b/docs/zh/query_language/table_functions/generate.md @@ -0,0 +1 @@ +en/query_language/table_functions/generate.md \ No newline at end of file From 31c39c8137a7dd7ec3de2edf66666a5b5edc9828 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 4 Feb 2020 15:11:32 +0300 Subject: [PATCH 0194/2007] fix mutations --- dbms/src/Storages/MergeTree/IMergeTreeDataPart.h | 1 - dbms/src/Storages/MergeTree/MergeTreeData.h | 9 --------- .../Storages/MergeTree/MergeTreeDataMergerMutator.cpp | 8 +++++--- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index 54e918f5b49..e4a3621126b 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -71,7 +71,6 @@ public: virtual bool isStoredOnDisk() const = 0; - virtual bool supportsVerticalMerge() const { return false; } /// NOTE: Returns zeros if column files are not found in checksums. diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.h b/dbms/src/Storages/MergeTree/MergeTreeData.h index 8d87edb166a..f633e2ab1ea 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.h +++ b/dbms/src/Storages/MergeTree/MergeTreeData.h @@ -942,15 +942,6 @@ protected: void setStoragePolicy(const String & new_storage_policy_name, bool only_check = false); - /// out_rename_map maps column files for the out_expression onto new table files. - /// out_force_update_metadata denotes if metadata must be changed even if out_rename_map is empty (used - /// for transformation-free changing of Enum values list). - /// Files to be deleted are mapped to an empty string in out_rename_map. - /// If part == nullptr, just checks that all type conversions are possible. - // void createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns, - // const IndicesASTs & old_indices, const IndicesASTs & new_indices, - // ExpressionActionsPtr & out_expression, NameToNameMap & out_rename_map, bool & out_force_update_metadata) const; - /// Calculates column sizes in compressed form for the current state of data_parts. Call with data_parts mutex locked. void calculateColumnSizesImpl(); /// Adds or subtracts the contribution of the part to compressed column sizes. diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 62b546ff830..6ad04e4e0f2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -990,16 +990,18 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor auto new_data_part = data.createPart( future_part.name, + source_part->getType(), future_part.part_info, space_reservation->getDisk(), - std::move(new_columns), - source_part->bytes_on_disk, - source_part->rows_count, "tmp_mut_" + future_part.name); new_data_part->is_temp = true; new_data_part->ttl_infos = source_part->ttl_infos; + /// It shouldn't be changed by mutation. + new_data_part->index_granularity_info = source_part->index_granularity_info; + new_data_part->setColumns(new_columns); + String new_part_tmp_path = new_data_part->getFullPath(); /// Note: this is done before creating input streams, because otherwise data.data_parts_mutex From 683a639ce9280a42c1d15511bc62f2d7542625f9 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 4 Feb 2020 15:25:09 +0300 Subject: [PATCH 0195/2007] doc fixes --- docs/en/query_language/table_functions/generate.md | 2 +- docs/ru/query_language/table_functions/generate.md | 2 +- docs/toc_en.yml | 1 + docs/toc_fa.yml | 1 + docs/toc_ja.yml | 3 ++- docs/toc_ru.yml | 1 + docs/toc_zh.yml | 1 + 7 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/en/query_language/table_functions/generate.md b/docs/en/query_language/table_functions/generate.md index c04ebff8a1a..ed9e2150b03 100644 --- a/docs/en/query_language/table_functions/generate.md +++ b/docs/en/query_language/table_functions/generate.md @@ -2,7 +2,7 @@ Generates random data with given schema. Allows to populate test tables with data. -Supports all data types that can be stored in table except LowCardinality, AggregateFunction. +Supports all data types that can be stored in table except `LowCardinality` and `AggregateFunction`. ```sql generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); diff --git a/docs/ru/query_language/table_functions/generate.md b/docs/ru/query_language/table_functions/generate.md index 11d7f7073a9..53544d16e7d 100644 --- a/docs/ru/query_language/table_functions/generate.md +++ b/docs/ru/query_language/table_functions/generate.md @@ -2,7 +2,7 @@ Генерирует случайные данные с заданной схемой. Позволяет заполнять тестовые таблицы данными. -Поддерживает все типы данных, которые могут храниться в таблице, за исключением LowCardinality, AggregateFunction. +Поддерживает все типы данных, которые могут храниться в таблице, за исключением `LowCardinality` и `AggregateFunction`. ```sql generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 8558216b15b..76d115045e7 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -142,6 +142,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index bd1e84d590e..280d5a6f53a 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -168,6 +168,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index f47bc065890..6661300a97e 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -140,7 +140,8 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - - 'Dictionaries': + - 'generate': 'query_language/table_functions/generate.md' +- 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': - 'General Description': 'query_language/dicts/external_dicts.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 5999ac74b56..06f196fd2f5 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -141,6 +141,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Словари': - 'Введение': 'query_language/dicts/index.md' - 'Внешние словари': diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index e85c6b50f27..c7ec13a1943 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -167,6 +167,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - '字典': - '介绍': 'query_language/dicts/index.md' - '外部字典': From f8e66e88a43f3f8b0bab4f807b46105ea0e32913 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 4 Feb 2020 16:29:08 +0300 Subject: [PATCH 0196/2007] style fixes --- .../src/TableFunctions/TableFunctionRandom.cpp | 18 ++++++++++++------ .../query_language/table_functions/generate.md | 1 + .../query_language/table_functions/generate.md | 2 +- .../query_language/table_functions/generate.md | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) create mode 120000 docs/fa/query_language/table_functions/generate.md diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index d70c8a73c63..fff1ed83539 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -224,12 +224,14 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { pcg32 generator(random_seed); offsets.resize(limit); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { offset += 1 + static_cast(generator()) % max_string_length; offsets[i] = offset - 1; } chars.resize(offset); - for (UInt64 i = 0; i < offset; ++i) { + for (UInt64 i = 0; i < offset; ++i) + { chars[i] = 32 + generator() % 95; } // add terminating zero char @@ -250,7 +252,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { pcg32 generator(random_seed); chars.resize(num_chars); - for (UInt64 i = 0; i < num_chars; ++i) { + for (UInt64 i = 0; i < num_chars; ++i) + { chars[i] = static_cast(generator()); } } @@ -327,7 +330,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, auto & data = typeid_cast &>(column).getData(); data.resize(limit); pcg64 generator(random_seed); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { auto x = UInt128(generator(), generator()); data[i] = x; } @@ -345,7 +349,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { pcg32 generator(random_seed); offsets.resize(limit); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { offset += static_cast(generator()) % max_array_length; offsets[i] = offset; } @@ -380,7 +385,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, pcg32 generator(random_seed); null_map.resize(limit); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { null_map[i] = generator() < 1024; } break; diff --git a/docs/fa/query_language/table_functions/generate.md b/docs/fa/query_language/table_functions/generate.md new file mode 120000 index 00000000000..141c05da1e3 --- /dev/null +++ b/docs/fa/query_language/table_functions/generate.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/generate.md b/docs/ja/query_language/table_functions/generate.md index de0b0a41754..141c05da1e3 120000 --- a/docs/ja/query_language/table_functions/generate.md +++ b/docs/ja/query_language/table_functions/generate.md @@ -1 +1 @@ -en/query_language/table_functions/generate.md \ No newline at end of file +../../../en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/zh/query_language/table_functions/generate.md b/docs/zh/query_language/table_functions/generate.md index de0b0a41754..141c05da1e3 120000 --- a/docs/zh/query_language/table_functions/generate.md +++ b/docs/zh/query_language/table_functions/generate.md @@ -1 +1 @@ -en/query_language/table_functions/generate.md \ No newline at end of file +../../../en/query_language/table_functions/generate.md \ No newline at end of file From 2f9f5dfc7a10f7233d26a45ec5d4d6639e2dde94 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 4 Feb 2020 16:34:57 +0300 Subject: [PATCH 0197/2007] better initialization of MergedBlockOutputStream --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../Storages/MergeTree/MergeTreeIOSettings.h | 5 ++-- .../MergeTree/MergedBlockOutputStream.cpp | 30 +++++++------------ .../MergeTree/MergedBlockOutputStream.h | 18 +++++------ .../MergedColumnOnlyOutputStream.cpp | 7 +++-- 5 files changed, 27 insertions(+), 35 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 5be89489138..aea6a392fa5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -3927,7 +3927,7 @@ void MergeTreeData::checkCanUsePolymorphicParts(bool no_throw) if (no_throw) { - message << " Settings 'min_bytes_for_wide_part' and 'min_bytes_for_wide_part' will be ignored further."; + message << " Settings 'min_rows_for_wide_part' and 'min_bytes_for_wide_part' will be ignored further."; LOG_WARNING(log, message.str()); } else diff --git a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h index f79a948dd33..8119c076b73 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -17,10 +17,11 @@ struct MergeTreeReaderSettings struct MergeTreeWriterSettings { - MergeTreeWriterSettings(const Settings & global_settings, bool can_use_adaptive_granularity_, bool blocks_are_granules_size_ = false) + MergeTreeWriterSettings(const Settings & global_settings, bool can_use_adaptive_granularity_, + size_t aio_threshold_, bool blocks_are_granules_size_ = false) : min_compress_block_size(global_settings.min_compress_block_size) , max_compress_block_size(global_settings.min_compress_block_size) - , aio_threshold(global_settings.min_bytes_to_use_direct_io) + , aio_threshold(aio_threshold_) , can_use_adaptive_granularity(can_use_adaptive_granularity_) , blocks_are_granules_size(blocks_are_granules_size_) {} diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 86c396b4955..59fe231ee80 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -16,14 +16,11 @@ MergedBlockOutputStream::MergedBlockOutputStream( const NamesAndTypesList & columns_list_, CompressionCodecPtr default_codec, bool blocks_are_granules_size) - : IMergedBlockOutputStream(data_part) - , columns_list(columns_list_) + : MergedBlockOutputStream( + data_part, columns_list_, default_codec, {}, + data_part->storage.global_context.getSettings().min_bytes_to_use_direct_io, + blocks_are_granules_size) { - MergeTreeWriterSettings writer_settings(data_part->storage.global_context.getSettings(), - data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); - - writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, std::move(writer_settings)); - init(); } MergedBlockOutputStream::MergedBlockOutputStream( @@ -37,8 +34,7 @@ MergedBlockOutputStream::MergedBlockOutputStream( , columns_list(columns_list_) { MergeTreeWriterSettings writer_settings(data_part->storage.global_context.getSettings(), - data_part->storage.canUseAdaptiveGranularity(), blocks_are_granules_size); - writer_settings.aio_threshold = aio_threshold; + data_part->storage.canUseAdaptiveGranularity(), aio_threshold, blocks_are_granules_size); if (aio_threshold > 0 && !merged_column_to_size.empty()) { @@ -50,9 +46,11 @@ MergedBlockOutputStream::MergedBlockOutputStream( } } - writer = data_part->getWriter(columns_list, - data_part->storage.getSkipIndices(), default_codec, writer_settings); - init(); + Poco::File(part_path).createDirectories(); + + writer = data_part->getWriter(columns_list, data_part->storage.getSkipIndices(), default_codec, writer_settings); + writer->initPrimaryIndex(); + writer->initSkipIndices(); } std::string MergedBlockOutputStream::getPartPath() const @@ -145,14 +143,6 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( new_part->index_granularity = writer->getIndexGranularity(); } -void MergedBlockOutputStream::init() -{ - Poco::File(part_path).createDirectories(); - writer->initPrimaryIndex(); - writer->initSkipIndices(); -} - - void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation) { block.checkNumberOfRows(); diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h index 1bacd06ab1c..d7514957bdd 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -14,18 +14,18 @@ class MergedBlockOutputStream final : public IMergedBlockOutputStream { public: MergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part_, + const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, - CompressionCodecPtr default_codec_, - bool blocks_are_granules_size_ = false); + CompressionCodecPtr default_codec, + bool blocks_are_granules_size = false); MergedBlockOutputStream( - const MergeTreeDataPartPtr & data_part_, + const MergeTreeDataPartPtr & data_part, const NamesAndTypesList & columns_list_, - CompressionCodecPtr default_codec_, - const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size_, - size_t aio_threshold_, - bool blocks_are_granules_size_ = false); + CompressionCodecPtr default_codec, + const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size, + size_t aio_threshold, + bool blocks_are_granules_size = false); std::string getPartPath() const; @@ -48,8 +48,6 @@ public: MergeTreeData::DataPart::Checksums * additional_column_checksums = nullptr); private: - void init(); - /** If `permutation` is given, it rearranges the values in the columns when writing. * This is necessary to not keep the whole block in the RAM to sort it. */ diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 0665a2c58de..93e3592d819 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -14,9 +14,12 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( : IMergedBlockOutputStream(data_part), header(header_), sync(sync_) { + const auto & global_settings = data_part->storage.global_context.getSettings(); MergeTreeWriterSettings writer_settings( - data_part->storage.global_context.getSettings(), - index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity()); + global_settings, + index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity(), + global_settings.min_bytes_to_use_direct_io); + writer_settings.filename_suffix = filename_suffix; writer_settings.skip_offsets = skip_offsets_; From b9443c98feab6e0e4b421ef2bb328932eee7e4ba Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Wed, 5 Feb 2020 16:00:06 +0800 Subject: [PATCH 0198/2007] ISSUES-8971 fix view with predicate optimizer --- dbms/src/Storages/StorageView.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 5f49cdb1263..3c0348eb30a 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -61,13 +61,13 @@ BlockInputStreams StorageView::read( auto new_outer_query = query_info.query->clone(); auto * new_outer_select = new_outer_query->as(); - replaceTableNameWithSubquery(new_outer_select, new_inner_query); - /// TODO: remove getTableExpressions and getTablesWithColumns { const auto & table_expressions = getTableExpressions(*new_outer_select); const auto & tables_with_columns = getDatabaseAndTablesWithColumnNames(table_expressions, context); + replaceTableNameWithSubquery(new_outer_select, new_inner_query); + auto & settings = context.getSettingsRef(); if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) { @@ -109,7 +109,7 @@ void StorageView::replaceTableNameWithSubquery(ASTSelectQuery * select_query, AS const auto alias = table_expression->database_and_table_name->tryGetAlias(); table_expression->database_and_table_name = {}; table_expression->subquery = std::make_shared(); - table_expression->subquery->children.push_back(subquery); + table_expression->children.push_back(subquery); if (!alias.empty()) table_expression->subquery->setAlias(alias); } From f8f615dfdb80faa3d3b8575d6610c677d334053e Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 5 Feb 2020 14:18:11 +0300 Subject: [PATCH 0199/2007] Debugging --- .../MergeTree/EphemeralLockInZooKeeper.h | 2 +- .../ReplicatedMergeTreeBlockOutputStream.cpp | 3 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 94 +++++++++++++++++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 3 - .../MergeTree/ReplicatedQueueAlterState.h | 72 ++++++++++++-- dbms/src/Storages/MergeTree/checkDataPart.cpp | 22 +++-- .../Storages/StorageReplicatedMergeTree.cpp | 84 ++++++++++++----- .../src/Storages/StorageReplicatedMergeTree.h | 5 + ...076_parallel_alter_replicated_zookeeper.sh | 39 ++++---- 9 files changed, 253 insertions(+), 71 deletions(-) diff --git a/dbms/src/Storages/MergeTree/EphemeralLockInZooKeeper.h b/dbms/src/Storages/MergeTree/EphemeralLockInZooKeeper.h index 3480b1e461e..b85761e0b15 100644 --- a/dbms/src/Storages/MergeTree/EphemeralLockInZooKeeper.h +++ b/dbms/src/Storages/MergeTree/EphemeralLockInZooKeeper.h @@ -87,7 +87,7 @@ private: /// Acquires block number locks in all partitions. -class EphemeralLocksInAllPartitions +class EphemeralLocksInAllPartitions : private boost::noncopyable { public: EphemeralLocksInAllPartitions( diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 3bb7e04fe6e..0442d964e5e 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -255,6 +255,8 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo log_entry.toString(), zkutil::CreateMode::PersistentSequential)); + ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/columns", storage.getMetadataVersion())); + /// Deletes the information that the block number is used for writing. block_number_lock->getUnlockOps(ops); @@ -308,6 +310,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo transaction.commit(); storage.merge_selecting_task->schedule(); + LOG_DEBUG(log, "COMMITED INSERT WITH VERSION:" << storage.getMetadataVersion() << " of part " << part_name); /// Lock nodes have been already deleted, do not delete them in destructor block_number_lock->assumeUnlocked(); } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 403dd8bb966..c92228c3325 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -24,6 +24,7 @@ ReplicatedMergeTreeQueue::ReplicatedMergeTreeQueue(StorageReplicatedMergeTree & , format_version(storage.format_version) , current_parts(format_version) , virtual_parts(format_version) + , alter_sequence(&Logger::get(storage_.getStorageID().table_name)) {} @@ -151,7 +152,7 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } if (entry->type == LogEntry::ALTER_METADATA) { - //std::cerr << "INSERT AlTER:" << entry->alter_version << std::endl; + LOG_DEBUG(log, "ADDING METADATA ENTRY WITH ALTER VERSION:" << entry->alter_version); alter_sequence.addMetadataAlter(entry->alter_version, state_lock); } @@ -162,6 +163,7 @@ void ReplicatedMergeTreeQueue::insertUnlocked( } if (entry->type == LogEntry::MUTATE_PART && entry->alter_version != -1) { + //LOG_DEBUG(log, "ADDING DATA ENTRY WITH ALTER VERSION:" << entry->alter_version); //std::cerr << "INSERT MUTATE PART:" << entry->alter_version << std::endl; alter_sequence.addDataAlterIfEmpty(entry->alter_version, state_lock); } @@ -217,11 +219,16 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( { Strings replaced_parts; current_parts.add(virtual_part_name, &replaced_parts); + virtual_parts.add(virtual_part_name, &replaced_parts); + LOG_DEBUG(log, "Replaced parts size for new part:" << virtual_part_name << " is " << replaced_parts.size()); /// Each part from `replaced_parts` should become Obsolete as a result of executing the entry. /// So it is one less part to mutate for each mutation with block number greater than part_info.getDataVersion() for (const String & replaced_part_name : replaced_parts) + { + LOG_DEBUG(log, "REMOVING REPLACED PART FROM MUTATIONS:" << replaced_part_name); updateMutationsPartsToDo(replaced_part_name, /* add = */ false); + } } String drop_range_part_name; @@ -240,6 +247,12 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( { //std::cerr << "Alter have mutation:" << entry->have_mutation << std::endl; alter_sequence.finishMetadataAlter(entry->alter_version, entry->have_mutation, state_lock); + + LOG_DEBUG(log, "FIN ALTER FOR PART with ALTER VERSION:" << entry->alter_version); + } + if (entry->type == LogEntry::MUTATE_PART) + { + LOG_DEBUG(log, "FIN MUTATION FOR PART:" << entry->source_parts[0] << " with ALTER VERSION:" << entry->alter_version); } } else @@ -248,6 +261,7 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( { /// Because execution of the entry is unsuccessful, `virtual_part_name` will never appear /// so we won't need to mutate it. + LOG_DEBUG(log, "REMOVING PART FROM MUTATIONS:" << virtual_part_name); updateMutationsPartsToDo(virtual_part_name, /* add = */ false); } } @@ -256,20 +270,41 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( void ReplicatedMergeTreeQueue::updateMutationsPartsToDo(const String & part_name, bool add) { + LOG_DEBUG(log, "Updating mutations parts to do with flag " << add << " and part " << part_name); auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); auto in_partition = mutations_by_partition.find(part_info.partition_id); if (in_partition == mutations_by_partition.end()) + { + LOG_DEBUG(log, "Not found partition in mutations for part:" << part_name); return; + } bool some_mutations_are_probably_done = false; auto from_it = in_partition->second.upper_bound(part_info.getDataVersion()); + if (from_it != in_partition->second.end()) + { + LOG_DEBUG(log, "FIRST MUTATION FOR PART "<second->entry->znode_name); + } + else + { + LOG_DEBUG(log, "NO MUTATIONS FOUND FOR PART:" << part_name << " maximum block number is " << in_partition->second.rbegin()->first); + } for (auto it = from_it; it != in_partition->second.end(); ++it) { MutationStatus & status = *it->second; + + if (!add) + LOG_DEBUG(log, "DECREMENTING parts to do for mutation:" << status.entry->znode_name << " because of part " << part_name); + else + LOG_DEBUG(log, "INCREMENTING parts to do for mutation:" << status.entry->znode_name << " because of part " << part_name); status.parts_to_do += (add ? +1 : -1); + LOG_DEBUG(log, "Parts left:" << status.parts_to_do << " for mutation " << status.entry->znode_name << " after " << part_name); if (status.parts_to_do <= 0) + { + LOG_DEBUG(log, "ALL PARTS ARE LEFT FOR MUTATION:" << status.entry->znode_name << " because of part " << part_name); some_mutations_are_probably_done = true; + } if (!add && !status.latest_failed_part.empty() && part_info.contains(status.latest_failed_part_info)) { @@ -672,7 +707,13 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C } /// Initialize `mutation.parts_to_do`. First we need to mutate all parts in `current_parts`. - mutation.parts_to_do += getPartNamesToMutate(*entry, current_parts).size(); + auto cur_parts = getPartNamesToMutate(*entry, current_parts); + mutation.parts_to_do += cur_parts.size(); + String parts; + for (auto & part : cur_parts) + { + parts += part + ","; + } /// And next we would need to mutate all parts with getDataVersion() greater than /// mutation block number that would appear as a result of executing the queue. @@ -683,10 +724,18 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C auto part_info = MergeTreePartInfo::fromPartName(produced_part_name, format_version); auto it = entry->block_numbers.find(part_info.partition_id); if (it != entry->block_numbers.end() && it->second > part_info.getDataVersion()) + { ++mutation.parts_to_do; + parts += produced_part_name + ","; + } } } + LOG_DEBUG( + log, + "MUTATION ENTRY " << entry->znode_name << " blocks " << entry->block_numbers.begin()->second << " alter version " + << entry->alter_version << " parts to do " << mutation.parts_to_do << " parts:" << parts); + if (mutation.parts_to_do == 0) some_mutations_are_probably_done = true; } @@ -1034,18 +1083,29 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( } if (entry.type == LogEntry::ALTER_METADATA) - { - //std::cerr << "Should we execute alter:"; + { //std::cerr << "Should we execute alter:"; //std::cerr << alter_sequence.canExecuteMetadataAlter(entry.alter_version, state_lock) << std::endl; - return alter_sequence.canExecuteMetadataAlter(entry.alter_version, state_lock); + if (!alter_sequence.canExecuteMetadataAlter(entry.alter_version, state_lock) || queue.front()->znode_name != entry.znode_name) + { + out_postpone_reason = "Cannot execute alter metadata with version: " + std::to_string(entry.alter_version) + + " because current head is " + std::to_string(alter_sequence.queue.front().alter_version) + + " with state: " + std::to_string(alter_sequence.queue.front().state); + return false; + } } if (entry.type == LogEntry::MUTATE_PART && entry.alter_version != -1) { //std::cerr << "Should we execute mutation:"; //std::cerr << alter_sequence.canExecuteDataAlter(entry.alter_version, state_lock) << std::endl; - return alter_sequence.canExecuteDataAlter(entry.alter_version, state_lock); + if (!alter_sequence.canExecuteDataAlter(entry.alter_version, state_lock)) + { + out_postpone_reason = "Cannot execute alter data with version: " + std::to_string(entry.alter_version) + + " because current head is " + std::to_string(alter_sequence.queue.front().alter_version) + + " with state: " + std::to_string(alter_sequence.queue.front().state); + return false; + } } return true; @@ -1362,7 +1422,10 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep LOG_TRACE(log, "Mutation " << entry->znode_name << " is done"); it->second.is_done = true; if (entry->alter_version != -1) + { + LOG_TRACE(log, "Finishing data alter with version " << entry->alter_version << " for entry " << entry->znode_name); alter_sequence.finishDataAlter(entry->alter_version, lock); + } } } } @@ -1467,7 +1530,7 @@ std::vector ReplicatedMergeTreeQueue::getMutationsStatu { const MutationStatus & status = pair.second; const ReplicatedMergeTreeMutationEntry & entry = *status.entry; - const Names parts_to_mutate = getPartNamesToMutate(entry, current_parts); + const Names parts_to_mutate = getPartNamesToMutate(entry, virtual_parts); for (const MutationCommand & command : entry.commands) { @@ -1735,24 +1798,35 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir /// the part (checked by querying queue.virtual_parts), we can confidently assign a mutation to /// version X for this part. + LOG_DEBUG(queue.log, "LOOKING for desired mutation version for part:" << part->name); if (last_quorum_parts.find(part->name) != last_quorum_parts.end() || part->name == inprogress_quorum_part) + { + LOG_DEBUG(queue.log, "PART " << part->name <<" NAME NOT IN QUORUM"); return {}; + } std::lock_guard lock(queue.state_mutex); if (queue.virtual_parts.getContainingPart(part->info) != part->name) + { + LOG_DEBUG(queue.log, "VIRTUAL PARTS HAVE CONTAINING PART " << part->name); return {}; + } auto in_partition = queue.mutations_by_partition.find(part->info.partition_id); if (in_partition == queue.mutations_by_partition.end()) + { + LOG_DEBUG(queue.log, "NO PARTITION FOR MUTATION FOR PART " << part->name); return {}; + } Int64 current_version = queue.getCurrentMutationVersionImpl(part->info.partition_id, part->info.getDataVersion(), lock); Int64 max_version = in_partition->second.rbegin()->first; if (current_version >= max_version) { + LOG_DEBUG(queue.log, "PART VERSION FOR " << part->name << " IS BIGGER THAN MAX"); //std::cerr << "But current version is:" << current_version << std::endl; return {}; } @@ -1772,6 +1846,12 @@ std::optional> ReplicatedMergeTreeMergePredicate::getDesir } } //std::cerr << "FOUND alter version:" << alter_version << " and mutation znode name:" << version << std::endl; + if (current_version >= max_version) + { + LOG_DEBUG(queue.log, "PART VERSION FOR " << part->name << " IS BIGGER THAN MAX AFTER ALTER"); + return {}; + } + return std::make_pair(max_version, alter_version); } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index e6250612462..6865c0b4dfb 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -78,9 +78,6 @@ private: time_t last_queue_update = 0; - /// This vector is used for sequential execution of alters - std::deque alter_znodes_in_queue; - /// parts that will appear as a result of actions performed right now by background threads (these actions are not in the queue). /// Used to block other actions on parts in the range covered by future_parts. using FuturePartsSet = std::map; diff --git a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h index 225581298e6..2a61284af05 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h +++ b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h @@ -3,6 +3,7 @@ #include #include #include +#include namespace DB { @@ -19,6 +20,7 @@ private: { APPLY_METADATA_CHANGES, APPLY_DATA_CHANGES, + DATA_CHANGES_NOT_NEEDED, }; struct AlterInQueue @@ -32,9 +34,14 @@ private: { } }; + Poco::Logger * log; public: + AlterSequence(Poco::Logger * log_) + : log(log_) + { + } std::deque queue; bool empty() const { @@ -43,6 +50,10 @@ public: void addMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) { + if (!queue.empty() && queue.front().alter_version > alter_version) + { + throw Exception("Alter not in order " + std::to_string(alter_version), ErrorCodes::LOGICAL_ERROR); + } queue.emplace_back(alter_version, AlterState::APPLY_METADATA_CHANGES); } @@ -62,20 +73,32 @@ public: void finishMetadataAlter(int alter_version, bool have_data_alter, std::unique_lock & /*state_lock*/) { + if (queue.empty()) { throw Exception("Queue shouldn't be empty on metadata alter", ErrorCodes::LOGICAL_ERROR); } + + LOG_DEBUG( + log, + "FINISHING METADATA ALTER WITH VERSION:" << alter_version << " AND HAVE DATA ALTER: " << have_data_alter + << " QUEUE HEAD:" << queue.front().alter_version << " state:" << queue.front().state); if (queue.front().alter_version != alter_version) throw Exception("Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue.front().alter_version), ErrorCodes::LOGICAL_ERROR); - if (have_data_alter) + if (have_data_alter && queue.front().state == AlterState::APPLY_METADATA_CHANGES) { + LOG_DEBUG( + log, + "FINISHING METADATA ALTER WITH VERSION:" << alter_version << " AND SWITCHING QUEUE STATE"); + //std::cerr << "Switching head state:" << AlterState::APPLY_DATA_CHANGES << std::endl; queue.front().state = AlterState::APPLY_DATA_CHANGES; } else { + LOG_DEBUG(log, "FINISHING METADATA ALTER WITH VERSION:" << alter_version << " AND DOING POP"); + //std::cerr << "JUST POP FRONT\n"; queue.pop_front(); } @@ -83,37 +106,70 @@ public: void finishDataAlter(int alter_version, std::lock_guard & /*state_lock*/) { + /// queue can be empty after load of finished mutation without move of mutation pointer if (queue.empty()) + { + LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE EMPTY"); + return; + } //std::cerr << "Finishing data alter:" << alter_version << std::endl; if (queue.front().alter_version != alter_version) - throw Exception( - "Finished data alter with version " + std::to_string(alter_version) + " but current alter in queue is " - + std::to_string(queue.front().alter_version), - ErrorCodes::LOGICAL_ERROR); + { + for (auto & state : queue) + { + if (state.alter_version == alter_version) + { + LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT HEAD IS NOT SAME SO MAKE DATA_CHANGED_NOT_NEEDED"); + state.state = AlterState::DATA_CHANGES_NOT_NEEDED; + return; + } + } + } + //if (queue.front().alter_version != alter_version) + //{ + // LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE VERSION IS " << queue.front().alter_version << " state " << queue.front().state); + // throw Exception( + // "Finished data alter with version " + std::to_string(alter_version) + " but current alter in queue is " + // + std::to_string(queue.front().alter_version), + // ErrorCodes::LOGICAL_ERROR); + //} if (queue.front().state != AlterState::APPLY_DATA_CHANGES) { - throw Exception( - "Finished data alter but current alter should perform metadata alter", - ErrorCodes::LOGICAL_ERROR); + LOG_DEBUG( + log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT STATE IS METADATA"); + queue.front().state = AlterState::DATA_CHANGES_NOT_NEEDED; + return; } + LOG_DEBUG( + log, + "FINISHING DATA ALTER WITH VERSION:" << alter_version << " QUEUE VERSION IS " << queue.front().alter_version << " STATE " + << queue.front().state); queue.pop_front(); } bool canExecuteMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) const { + if (queue.empty()) + throw Exception("QUEUE EMPTY ON METADATA", ErrorCodes::LOGICAL_ERROR); + LOG_DEBUG(log, "CHECK METADATADATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE HEAD IS " << queue.front().alter_version); + //std::cerr << "Alter queue front:" << queue.front().alter_version << " state:" << queue.front().state << std::endl; return queue.front().alter_version == alter_version; } bool canExecuteDataAlter(int alter_version, std::lock_guard & /*state_lock*/) const { + if (queue.empty()) + throw Exception("QUEUE EMPTY ON DATA", ErrorCodes::LOGICAL_ERROR); + //std::cerr << "Alter queue front:" << queue.front().alter_version << " state:" << queue.front().state << std::endl; //std::cerr << "CAn execute:" << alter_version << std::endl; //std::cerr << "FRont version:" << queue.front().alter_version << std::endl; //std::cerr << "State:" << queue.front().state << std::endl; + LOG_DEBUG(log, "CHECK DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE HEAD IS " << queue.front().alter_version << " state:" << queue.front().state); return queue.front().alter_version == alter_version && queue.front().state == AlterState::APPLY_DATA_CHANGES; } diff --git a/dbms/src/Storages/MergeTree/checkDataPart.cpp b/dbms/src/Storages/MergeTree/checkDataPart.cpp index a2d6a836d6f..c8793beeec9 100644 --- a/dbms/src/Storages/MergeTree/checkDataPart.cpp +++ b/dbms/src/Storages/MergeTree/checkDataPart.cpp @@ -444,14 +444,20 @@ MergeTreeData::DataPart::Checksums checkDataPart( const MergeTreeIndices & indices, std::function is_cancelled) { - return checkDataPart( - data_part->getFullPath(), - data_part->index_granularity, - data_part->index_granularity_info.marks_file_extension, - require_checksums, - primary_key_data_types, - indices, - is_cancelled); + try { + return checkDataPart( + data_part->getFullPath(), + data_part->index_granularity, + data_part->index_granularity_info.marks_file_extension, + require_checksums, + primary_key_data_types, + indices, + is_cancelled); + }catch (...) + { + tryLogCurrentException("PartChecker"); + std::terminate(); + } } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index db4ee81e2a4..59a1d9cfec6 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -300,6 +300,10 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( createNewZooKeeperNodes(); + Coordination::Stat metadata_stat; + current_zookeeper->get(zookeeper_path + "/columns", &metadata_stat); + metadata_version = metadata_stat.version; + other_replicas_fixed_granularity = checkFixedGranualrityInZookeeper(); } @@ -772,8 +776,16 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: if (part_name.empty()) part_name = part->name; - check(part->columns); - //int expected_columns_version = columns_version; + try + { + check(part->columns); + } + catch (...) + { + LOG_DEBUG(log, "EXCEPTION ADDING PART:" << part_name << " VERSION:" << getMetadataVersion()); + throw; + } + auto local_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( part->columns, part->checksums); @@ -3246,8 +3258,9 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); setTableStructure(std::move(columns_from_entry), metadata_diff); + metadata_version = entry.alter_version; - LOG_INFO(log, "Applied changes to the metadata of the table."); + LOG_INFO(log, "Applied changes to the metadata of the table. Setting metadata version:" << metadata_version); } //////std::cerr << "Recalculating columns sizes\n"; @@ -3313,23 +3326,17 @@ void StorageReplicatedMergeTree::alter( Coordination::Stat metadata_stat; String metadata_in_zk = zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); - Coordination::Stat columns_stat; - String columns_in_zk = zookeeper->get(zookeeper_path + "/columns", &columns_stat); - StorageInMemoryMetadata metadata = getMetadataFromSharedZookeeper(metadata_in_zk, columns_in_zk); - params.validate(metadata, query_context); + StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - String path_prefix = zookeeper_path + "/alters/alter-"; - String path = zookeeper->create(path_prefix, "", zkutil::CreateMode::EphemeralSequential); ////std::cerr << "Path for alter:" << path << std::endl; - int alter_version = parse(path.c_str() + path_prefix.size(), path.size() - path_prefix.size()); + //int alter_version = columns_stat.version; //std::cerr << "Alter version:" << alter_version << std::endl; String new_columns_str = metadata.columns.toString(); - if (new_columns_str != getColumns().toString()) - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, columns_stat.version)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/columns", new_columns_str, metadata_version)); ReplicatedMergeTreeTableMetadata new_metadata(*this); if (ast_to_str(metadata.order_by_ast) != ast_to_str(order_by_ast)) @@ -3347,14 +3354,8 @@ void StorageReplicatedMergeTree::alter( new_metadata.constraints = new_constraints_str; String new_metadata_str = new_metadata.toString(); - if (new_metadata_str != ReplicatedMergeTreeTableMetadata(*this).toString()) - ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, metadata_stat.version)); + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", new_metadata_str, metadata_stat.version)); - if (ops.empty()) - { - //std::cerr << "Alter doesn't change anything\n"; - return; - } /// Perform settings update locally auto old_metadata = getInMemoryMetadata(); old_metadata.settings_ast = metadata.settings_ast; @@ -3365,7 +3366,7 @@ void StorageReplicatedMergeTree::alter( entry.source_replica = replica_name; entry.metadata_str = new_metadata_str; entry.columns_str = new_columns_str; - entry.alter_version = alter_version; + entry.alter_version = metadata_version + 1; entry.create_time = time(nullptr); if (!maybe_mutation_commands.empty()) @@ -3377,9 +3378,42 @@ void StorageReplicatedMergeTree::alter( bool have_mutation = false; + std::optional lock_holder; + size_t partitions_count = 0; if (!maybe_mutation_commands.empty()) { - prepareMutationEntry(zookeeper, maybe_mutation_commands, ops, alter_version); + String mutations_path = zookeeper_path + "/mutations"; + + ReplicatedMergeTreeMutationEntry mutation_entry; + mutation_entry.source_replica = replica_name; + mutation_entry.commands = maybe_mutation_commands; + mutation_entry.alter_version = metadata_version + 1; + Coordination::Stat mutations_stat; + zookeeper->get(mutations_path, &mutations_stat); + + lock_holder.emplace( + zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + + Coordination::Stat block_numbers_version; + zookeeper->get(zookeeper_path + "/block_numbers", &block_numbers_version); + + for (const auto & lock : lock_holder->getLocks()) + { + Coordination::Stat partition_stat; + mutation_entry.block_numbers[lock.partition_id] = lock.number; + zookeeper->get(zookeeper_path + "/block_numbers/" + lock.partition_id, &partition_stat); + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/block_numbers/" + lock.partition_id, partition_stat.version)); + partitions_count++; + } + + mutation_entry.create_time = time(nullptr); + /// We have to be sure, that no inserts happened + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/block_numbers", block_numbers_version.version)); + + ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); + ops.emplace_back( + zkutil::makeCreateRequest(mutations_path + "/", mutation_entry.toString(), zkutil::CreateMode::PersistentSequential)); + have_mutation = true; } @@ -3396,7 +3430,7 @@ void StorageReplicatedMergeTree::alter( { //std::cerr << "In have mutation\n"; //std::cerr << "INDEX:" << results.size() - 2 << std::endl; - String alter_path = dynamic_cast(*results[results.size() - 3]).path_created; + String alter_path = dynamic_cast(*results[results.size() - 4 - partitions_count]).path_created; //std::cerr << "Alter path:" << alter_path << std::endl; entry.znode_name = alter_path.substr(alter_path.find_last_of('/') + 1); @@ -3422,8 +3456,12 @@ void StorageReplicatedMergeTree::alter( table_lock_holder.release(); + std::vector unwaited; //std::cerr << "Started wait for alter\n"; - auto unwaited = waitForAllReplicasToProcessLogEntry(entry, false); + if (query_context.getSettingsRef().replication_alter_partitions_sync == 2) + unwaited = waitForAllReplicasToProcessLogEntry(entry, false); + else if (query_context.getSettingsRef().replication_alter_partitions_sync == 1) + waitForReplicaToProcessLogEntry(replica_name, entry); //std::cerr << "FInished wait for alter\n"; if (!unwaited.empty()) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index a5e1d1986e1..1945924e84d 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -180,6 +180,10 @@ public: /// Checks ability to use granularity bool canUseAdaptiveGranularity() const override; + int getMetadataVersion() const { + return metadata_version; + } + private: /// Get a sequential consistent view of current parts. @@ -255,6 +259,7 @@ private: /// Limiting parallel fetches per one table std::atomic_uint current_table_fetches {0}; + int metadata_version = 0; /// Threads. /// A task that keeps track of the updates in the logs of all replicas and loads them into the queue. diff --git a/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index 839ff932cc3..a9498ff8e54 100755 --- a/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/dbms/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -3,23 +3,24 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh +REPLICAS=5 -for i in {1..2}; do +for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" done -for i in {1..2}; do +for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/concurrent_alter_mt', '$i') ORDER BY key" done $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, toString(number) from numbers(10)" $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, toString(number) from numbers(10, 40)" -for i in {1..2}; do +for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_mt_$i" done -for i in {1..2}; do +for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_mt_$i" done @@ -30,7 +31,7 @@ INITIAL_SUM=`$CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alte function garbage_alter_thread() { while true; do - REPLICA=$(($RANDOM % 2 + 1)) + REPLICA=$(($RANDOM % 5 + 1)) $CLICKHOUSE_CLIENT -n --query "ALTER TABLE concurrent_alter_mt_$REPLICA ADD COLUMN h String DEFAULT '0'; ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN h UInt64; ALTER TABLE concurrent_alter_mt_$REPLICA DROP COLUMN h;"; done } @@ -43,7 +44,7 @@ function correct_alter_thread() { TYPES=(Float64 String UInt8 UInt32) while true; do - REPLICA=$(($RANDOM % 2 + 1)) + REPLICA=$(($RANDOM % 5 + 1)) TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency sleep 0.$RANDOM @@ -58,7 +59,7 @@ function insert_thread() VALUES=(7.0 7 '7') while true; do - REPLICA=$(($RANDOM % 2 + 1)) + REPLICA=$(($RANDOM % 5 + 1)) VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))" sleep 0.$RANDOM @@ -71,26 +72,21 @@ export -f garbage_alter_thread; export -f correct_alter_thread; export -f insert_thread; -TIMEOUT=10 +TIMEOUT=30 -timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & -#timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & #timeout $TIMEOUT bash -c garbage_alter_thread 2> /dev/null & timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -#timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -#timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & -# We don't want too many parts, just several alters per second timeout $TIMEOUT bash -c insert_thread 2> /dev/null & timeout $TIMEOUT bash -c insert_thread 2> /dev/null & timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -#timeout $TIMEOUT bash -c insert_thread 2> /dev/null & -#timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & wait @@ -99,11 +95,12 @@ echo "Finishing alters" # This alter will finish all previous $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String" -for i in {1..2}; do +for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_mt_$i" done -for i in {1..2}; do + +for i in `seq $REPLICAS`; do $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_mt_$i" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" done From 16bb4e975d230d2dd8fb04bff94d3dbac8b4fcc4 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Wed, 5 Feb 2020 16:22:25 +0800 Subject: [PATCH 0200/2007] ISSUES-8971 support explain & view storage --- .../Interpreters/InterpreterExplainQuery.cpp | 78 ++++++++++++++++++- dbms/src/Storages/StorageView.cpp | 65 +++++++++------- dbms/src/Storages/StorageView.h | 4 + 3 files changed, 117 insertions(+), 30 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterExplainQuery.cpp b/dbms/src/Interpreters/InterpreterExplainQuery.cpp index 0a158a2b9bc..31c812a9248 100644 --- a/dbms/src/Interpreters/InterpreterExplainQuery.cpp +++ b/dbms/src/Interpreters/InterpreterExplainQuery.cpp @@ -3,19 +3,88 @@ #include #include #include +#include #include -#include #include +#include #include +#include +#include +#include #include #include +#include #include +#include +#include +#include namespace DB { +namespace +{ + struct ExplainAnalyzedSyntaxMatcher + { + struct Data + { + bool analyzed = false; + const Context & context; + }; + + static bool needChildVisit(ASTPtr &, ASTPtr &) { return true; } + + static void visit(ASTPtr & ast, Data & data) + { + if (auto * select_query = ast->as()) + visit(*select_query, ast, data); + if (auto * union_select_query = ast->as()) + visit(*union_select_query, ast, data); + } + + static void visit(ASTSelectQuery & select_query, ASTPtr &, Data & data) + { + for (const auto & child : select_query.tables()->children) + { + auto * tables_element = child->as(); + + if (tables_element && tables_element->table_expression) + visit(*tables_element->table_expression->as(), select_query, data); + } + } + + static void visit(ASTSelectWithUnionQuery &, ASTPtr & node, Data & data) + { + if (!data.analyzed) + { + data.analyzed = true; + InterpreterSelectWithUnionQuery interpreter( + node, data.context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify()); + } + } + + static void visit(ASTTableExpression & expression, ASTSelectQuery & select_query, Data & data) + { + if (data.context.getSettingsRef().enable_optimize_predicate_expression && expression.database_and_table_name) + { + if (const auto * identifier = expression.database_and_table_name->as()) + { + const auto & [database, table] = IdentifierSemantic::extractDatabaseAndTable(*identifier); + const auto & storage = data.context.getTable(database.empty() ? data.context.getCurrentDatabase() : database, table); + + if (auto * storage_view = dynamic_cast(storage.get())) + storage_view->getRuntimeViewQuery(&select_query, data.context, true); + } + } + } + }; + + using ExplainAnalyzedSyntaxVisitor = InDepthNodeVisitor; + +} + BlockIO InterpreterExplainQuery::execute() { BlockIO res; @@ -52,9 +121,10 @@ BlockInputStreamPtr InterpreterExplainQuery::executeImpl() } else if (ast.getKind() == ASTExplainQuery::AnalyzedSyntax) { - InterpreterSelectWithUnionQuery interpreter(ast.children.at(0), context, - SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify()); - interpreter.getQuery()->format(IAST::FormatSettings(ss, false)); + ExplainAnalyzedSyntaxVisitor::Data data{.context = context}; + ExplainAnalyzedSyntaxVisitor(data).visit(query); + + ast.children.at(0)->format(IAST::FormatSettings(ss, false)); } res_columns[0]->insert(ss.str()); diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 3c0348eb30a..84e466267ab 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -15,6 +15,7 @@ #include #include +#include "StorageView.h" namespace DB @@ -56,31 +57,7 @@ BlockInputStreams StorageView::read( ASTPtr current_inner_query = inner_query; if (context.getSettings().enable_optimize_predicate_expression) - { - auto new_inner_query = inner_query->clone(); - auto new_outer_query = query_info.query->clone(); - auto * new_outer_select = new_outer_query->as(); - - /// TODO: remove getTableExpressions and getTablesWithColumns - { - const auto & table_expressions = getTableExpressions(*new_outer_select); - const auto & tables_with_columns = getDatabaseAndTablesWithColumnNames(table_expressions, context); - - replaceTableNameWithSubquery(new_outer_select, new_inner_query); - - auto & settings = context.getSettingsRef(); - if (settings.joined_subquery_requires_alias && tables_with_columns.size() > 1) - { - for (auto & pr : tables_with_columns) - if (pr.table.table.empty() && pr.table.alias.empty()) - throw Exception("Not unique subquery in FROM requires an alias (or joined_subquery_requires_alias=0 to disable restriction).", - ErrorCodes::ALIAS_REQUIRED); - } - - if (PredicateExpressionsOptimizer(context, tables_with_columns, context.getSettings()).optimize(*new_outer_select)) - current_inner_query = new_inner_query; - } - } + current_inner_query = getRuntimeViewQuery(*query_info.query->as(), context); QueryPipeline pipeline; /// FIXME res may implicitly use some objects owned be pipeline, but them will be destructed after return @@ -94,6 +71,41 @@ BlockInputStreams StorageView::read( return res; } +ASTPtr StorageView::getRuntimeViewQuery(const ASTSelectQuery & outer_query, const Context & context) +{ + auto temp_outer_query = outer_query.clone(); + auto * new_outer_select = temp_outer_query->as(); + return getRuntimeViewQuery(new_outer_select, context, false); +} + + + +ASTPtr StorageView::getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize) +{ + auto runtime_view_query = inner_query->clone(); + + /// TODO: remove getTableExpressions and getTablesWithColumns + { + const auto & table_expressions = getTableExpressions(*outer_query); + const auto & tables_with_columns = getDatabaseAndTablesWithColumnNames(table_expressions, context); + + replaceTableNameWithSubquery(outer_query, runtime_view_query); + if (context.getSettingsRef().joined_subquery_requires_alias && tables_with_columns.size() > 1) + { + for (auto & pr : tables_with_columns) + if (pr.table.table.empty() && pr.table.alias.empty()) + throw Exception("Not unique subquery in FROM requires an alias (or joined_subquery_requires_alias=0 to disable restriction).", + ErrorCodes::ALIAS_REQUIRED); + } + + if (PredicateExpressionsOptimizer(context, tables_with_columns, context.getSettings()).optimize(*outer_query) && normalize) + InterpreterSelectWithUnionQuery( + runtime_view_query, context, SelectQueryOptions(QueryProcessingStage::FetchColumns).analyze().modify(), {}); + } + + return runtime_view_query; +} + void StorageView::replaceTableNameWithSubquery(ASTSelectQuery * select_query, ASTPtr & subquery) { auto * select_element = select_query->tables()->children[0]->as(); @@ -109,7 +121,8 @@ void StorageView::replaceTableNameWithSubquery(ASTSelectQuery * select_query, AS const auto alias = table_expression->database_and_table_name->tryGetAlias(); table_expression->database_and_table_name = {}; table_expression->subquery = std::make_shared(); - table_expression->children.push_back(subquery); + table_expression->subquery->children.push_back(subquery); + table_expression->children.push_back(table_expression->subquery); if (!alias.empty()) table_expression->subquery->setAlias(alias); } diff --git a/dbms/src/Storages/StorageView.h b/dbms/src/Storages/StorageView.h index bb3997a1578..b411ffa7a84 100644 --- a/dbms/src/Storages/StorageView.h +++ b/dbms/src/Storages/StorageView.h @@ -29,6 +29,10 @@ public: size_t max_block_size, unsigned num_streams) override; + ASTPtr getRuntimeViewQuery(const ASTSelectQuery & outer_query, const Context & context); + + ASTPtr getRuntimeViewQuery(ASTSelectQuery * outer_query, const Context & context, bool normalize); + private: ASTPtr inner_query; From 370615de93245722a814cc76899cee65ca98063d Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Wed, 5 Feb 2020 19:41:03 +0800 Subject: [PATCH 0201/2007] ISSUES-8971 add test for predicate optimizer & view --- ...76_predicate_optimizer_with_view.reference | 4 ++++ .../01076_predicate_optimizer_with_view.sql | 19 +++++++++++++++++++ .../bugs/00597_push_down_predicate.sql | 18 ------------------ 3 files changed, 23 insertions(+), 18 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference create mode 100644 dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql delete mode 100644 dbms/tests/queries/bugs/00597_push_down_predicate.sql diff --git a/dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference b/dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference new file mode 100644 index 00000000000..1e92e7b8596 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.reference @@ -0,0 +1,4 @@ +SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 +SELECT \n date, \n id, \n name, \n value\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 2\n)\nWHERE id = 2 +SELECT id\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n)\nWHERE id = 1 +SELECT id\nFROM \n(\n SELECT \n date, \n id, \n name, \n value\n FROM default.test\n WHERE id = 1\n) AS s\nWHERE id = 1 diff --git a/dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql b/dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql new file mode 100644 index 00000000000..fa55a47e768 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01076_predicate_optimizer_with_view.sql @@ -0,0 +1,19 @@ +DROP TABLE IF EXISTS test; +DROP TABLE IF EXISTS test_view; + +CREATE TABLE test(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192); +CREATE VIEW test_view AS SELECT * FROM test; + +SET enable_debug_queries = 1; +SET enable_optimize_predicate_expression = 1; + +-- Optimize predicate expression with view +ANALYZE SELECT * FROM test_view WHERE id = 1; +ANALYZE SELECT * FROM test_view WHERE id = 2; +ANALYZE SELECT id FROM test_view WHERE id = 1; +ANALYZE SELECT s.id FROM test_view AS s WHERE s.id = 1; + +SELECT * FROM (SELECT toUInt64(b), sum(id) AS b FROM test) WHERE `toUInt64(sum(id))` = 3; -- { serverError 47 } + +DROP TABLE IF EXISTS test; +DROP TABLE IF EXISTS test_view; diff --git a/dbms/tests/queries/bugs/00597_push_down_predicate.sql b/dbms/tests/queries/bugs/00597_push_down_predicate.sql deleted file mode 100644 index 0de9522ab50..00000000000 --- a/dbms/tests/queries/bugs/00597_push_down_predicate.sql +++ /dev/null @@ -1,18 +0,0 @@ -DROP TABLE IF EXISTS test.test; -DROP TABLE IF EXISTS test.test_view; - -CREATE TABLE test.test(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192); -CREATE VIEW test.test_view AS SELECT * FROM test.test; - -SET enable_optimize_predicate_expression = 1; -SET enable_debug_queries = 1; - --- Optimize predicate expression with view --- TODO: simple view is not replaced with subquery inside syntax analyzer -ANALYZE SELECT * FROM test.test_view WHERE id = 1; -ANALYZE SELECT * FROM test.test_view WHERE id = 2; -ANALYZE SELECT id FROM test.test_view WHERE id = 1; -ANALYZE SELECT s.id FROM test.test_view AS s WHERE s.id = 1; - --- TODO: this query shouldn't work, because the name `toUInt64(sum(id))` is undefined for user -SELECT * FROM (SELECT toUInt64(b), sum(id) AS b FROM test.test) WHERE `toUInt64(sum(id))` = 3; From 2723057e809c6a4cd3085734f4e1b6b2cbac3889 Mon Sep 17 00:00:00 2001 From: Aleksei Levushkin Date: Wed, 5 Feb 2020 14:48:59 +0300 Subject: [PATCH 0202/2007] added zlib-ng instead zlib --- contrib/grpc-cmake/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/contrib/grpc-cmake/CMakeLists.txt b/contrib/grpc-cmake/CMakeLists.txt index b1941ae508f..950bcb2b7f6 100644 --- a/contrib/grpc-cmake/CMakeLists.txt +++ b/contrib/grpc-cmake/CMakeLists.txt @@ -15,8 +15,8 @@ endif() set(CMAKE_POSITION_INDEPENDENT_CODE TRUE) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_gRPC_C_CXX_FLAGS}") -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_gRPC_C_CXX_FLAGS}") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${_gRPC_C_CXX_FLAGS} -w") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${_gRPC_C_CXX_FLAGS} -w") set(_gRPC_PROTOBUF_LIBRARY_NAME "libprotobuf") @@ -96,9 +96,9 @@ set(_gRPC_UPB_GRPC_GENERATED_DIR "${GRPC_SOURCE_DIR}/src/core/ext/upb-generated" set(_gRPC_UPB_LIBRARIES upb) # zlib.cmake -set(ZLIB_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/zlib) +set(ZLIB_ROOT_DIR ${GRPC_SOURCE_DIR}/third_party/zlib-ng) include_directories("${ZLIB_ROOT_DIR}") -add_subdirectory(${ZLIB_ROOT_DIR} ${ZLIB_ROOT_DIR}) +# add_subdirectory(${ZLIB_ROOT_DIR} ${ZLIB_ROOT_DIR}) set(_gRPC_ZLIB_LIBRARIES zlibstatic) set(_gRPC_ZLIB_INCLUDE_DIR "${ZLIB_ROOT_DIR}") @@ -1333,7 +1333,7 @@ add_library(grpc++ ${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc ) -target_compile_options(grpc++ PUBLIC -Wno-error=shadow) +target_compile_options(grpc++ PUBLIC -w) target_include_directories(grpc++ PUBLIC ${GRPC_INCLUDE_DIR} @@ -1406,7 +1406,7 @@ add_library(grpc++_unsecure ${GRPC_SOURCE_DIR}/src/cpp/codegen/codegen_init.cc ) -target_compile_options(grpc++_unsecure PUBLIC -Wno-error=shadow) +target_compile_options(grpc++_unsecure PUBLIC -w) target_include_directories(grpc++_unsecure PUBLIC ${GRPC_INCLUDE_DIR} From 784230b7a81499da6316d021667cf92a332b675d Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Thu, 6 Feb 2020 08:30:14 +0800 Subject: [PATCH 0203/2007] ISSUES-8971 fix non from --- dbms/src/Interpreters/InterpreterExplainQuery.cpp | 15 +++++++++------ dbms/src/Storages/StorageView.cpp | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/dbms/src/Interpreters/InterpreterExplainQuery.cpp b/dbms/src/Interpreters/InterpreterExplainQuery.cpp index 31c812a9248..d6272a08f71 100644 --- a/dbms/src/Interpreters/InterpreterExplainQuery.cpp +++ b/dbms/src/Interpreters/InterpreterExplainQuery.cpp @@ -4,21 +4,21 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include #include -#include -#include -#include +#include +#include +#include #include -#include -#include -#include namespace DB @@ -46,6 +46,9 @@ namespace static void visit(ASTSelectQuery & select_query, ASTPtr &, Data & data) { + if (!select_query.tables()) + return; + for (const auto & child : select_query.tables()->children) { auto * tables_element = child->as(); diff --git a/dbms/src/Storages/StorageView.cpp b/dbms/src/Storages/StorageView.cpp index 84e466267ab..d66da3a3bd5 100644 --- a/dbms/src/Storages/StorageView.cpp +++ b/dbms/src/Storages/StorageView.cpp @@ -15,7 +15,6 @@ #include #include -#include "StorageView.h" namespace DB From 513c2e8f1b010c3aafcb08bd5c67c77ebd76845c Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 6 Feb 2020 15:34:11 +0300 Subject: [PATCH 0204/2007] recanonize linux build --- .../01072_random_table_function.reference | 73 ++++++++++++++----- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.reference b/dbms/tests/queries/0_stateless/01072_random_table_function.reference index 93ea1861756..3906b417524 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.reference +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.reference @@ -9,6 +9,7 @@ world hello hello hello +- Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) ['world','hello','world','hello','hello','hello','world','hello','hello'] ['hello','world','world','hello','world','world'] @@ -20,6 +21,7 @@ Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) ['hello','hello'] ['hello','hello'] ['hello','world','hello','hello','world','world','world','world'] +- Nullable(Enum16(\'o\' = -200, \'h\' = 1, \'w\' = 5)) o w @@ -31,6 +33,7 @@ h h w o +- UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 2254772619926532955 2254772619926532955 1234817989 1234817989 54213 -11323 197 -59 9120028858397505560 9120028858397505560 1171957426 1171957426 42674 -22862 178 -78 @@ -42,6 +45,7 @@ UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 7736560050027905187 7736560050027905187 1012211222 1012211222 7702 7702 22 22 2199287578947862030 2199287578947862030 2185722662 -2109244634 31526 31526 38 38 3019483913099890467 3019483913099890467 2647224658 -1647742638 29010 29010 82 82 +- Date DateTime DateTime(\'Europe/Moscow\') 2106-02-07 2009-02-16 23:59:49 2009-02-16 23:59:49 2086-11-02 2007-02-20 10:43:46 2007-02-20 10:43:46 @@ -53,6 +57,7 @@ Date DateTime DateTime(\'Europe/Moscow\') 1991-02-02 2002-01-28 12:47:02 2002-01-28 12:47:02 2056-04-25 2039-04-06 20:11:02 2039-04-06 20:11:02 2049-06-05 2053-11-20 07:10:58 2053-11-20 07:10:58 +- DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 2007-02-20 10:43:46.989 2007-02-20 10:43:46.817989 2007-02-20 10:43:46.817989 2002-10-04 02:54:48.647 2002-10-04 02:54:48.100647 2002-10-04 02:54:48.100647 @@ -64,6 +69,7 @@ DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 1979-01-20 20:39:20.939 1979-01-20 20:39:20.162939 1979-01-20 20:39:20.162939 2063-07-18 01:46:10.215 2063-07-18 01:46:10.908215 2063-07-18 01:46:10.908215 1996-11-02 14:35:41.110 1996-11-02 14:35:41.183110 1996-11-02 14:35:41.183110 +- Float32 Float64 9.783235e37 2.1973467205491123e307 9.285203e37 8.887754501811354e307 @@ -75,6 +81,7 @@ Float32 Float64 8.019563e37 7.539520705557441e307 1.7317079e38 2.143274805821858e307 2.0973474e38 2.9425818885529257e307 +- Decimal32(4) Decimal64(8) Decimal64(8) 123481.7989 22547726199.26532955 4159321346419233104838.6879832895010840 117195.7426 91200288583.97505560 8403779329565810688767.7049545291714611 @@ -86,17 +93,19 @@ Decimal32(4) Decimal64(8) Decimal64(8) 101221.1222 77365600500.27905187 -4693380431928321782727.0243506636623202 -210924.4634 21992875789.47862030 13765369952377767241248.9441272127848016 -164774.2638 30194839130.99890467 -13890064946313418575619.0315227826809939 +- UUID -1f4a8fc0-63ff-735b-7e90-d9ed3e183818 -3f39171b-1263-31fa-5046-2ea9fe2fd033 -9927a60f-01ac-f065-6da8-49def100c0cc -5d736910-493d-c3bf-6b5d-c8601d6440a3 -1e857066-961d-be0e-29e7-5c9efd534f23 -bda66d4f-737b-3622-b60f-aa27fe38ff30 -623d6d82-4422-2885-297f-7b2fec54178b -dcb0e0ca-3a43-5f2e-556e-7945df65729e -678f2360-36ac-d439-8d6d-f92295887e50 -9780b53e-dc0f-4a21-bdb3-9798af1913ad +7e90d9ed-3e18-3818-1f4a-8fc063ff735b +50462ea9-fe2f-d033-3f39-171b126331fa +6da849de-f100-c0cc-9927-a60f01acf065 +6b5dc860-1d64-40a3-5d73-6910493dc3bf +29e75c9e-fd53-4f23-1e85-7066961dbe0e +b60faa27-fe38-ff30-bda6-6d4f737b3622 +297f7b2f-ec54-178b-623d-6d8244222885 +556e7945-df65-729e-dcb0-e0ca3a435f2e +8d6df922-9588-7e50-678f-236036acd439 +bdb39798-af19-13ad-9780-b53edc0f4a21 +- Tuple(Int32, Int64) (1234817989,2254772619926532955) (1171957426,9120028858397505560) @@ -108,6 +117,7 @@ Tuple(Int32, Int64) (1012211222,7736560050027905187) (-2109244634,2199287578947862030) (-1647742638,3019483913099890467) +- Array(Int8) [-59,-78,-25,-72,-40,-84,95,22,38] [82,65,35,-110,-57,-69] @@ -119,6 +129,7 @@ Array(Int8) [-40,84] [-104,-86] [-36,123,44,60,5,25,-5,-127] +- Array(Nullable(Int32)) [1234817989,1171957426,275100647,1033685688,180895192,135557292,716914271,1012211222,-2109244634] [-1647742638,319510849,513356835,-1966518382,-786518841,269162939] @@ -130,17 +141,19 @@ Array(Nullable(Int32)) [-1399152424,-953863084] [733724312,-23652950] [371735004,462118779,148602156,-1055384004,-1041274619,247762201,522289659,822210177] +- Array(Nullable(UUID)) -['1f4a8fc0-63ff-735b-7e90-d9ed3e183818','3f39171b-1263-31fa-5046-2ea9fe2fd033','9927a60f-01ac-f065-6da8-49def100c0cc','5d736910-493d-c3bf-6b5d-c8601d6440a3','1e857066-961d-be0e-29e7-5c9efd534f23','bda66d4f-737b-3622-b60f-aa27fe38ff30','623d6d82-4422-2885-297f-7b2fec54178b','dcb0e0ca-3a43-5f2e-556e-7945df65729e','678f2360-36ac-d439-8d6d-f92295887e50'] -['9780b53e-dc0f-4a21-bdb3-9798af1913ad','c79810de-3635-d333-5ca1-7a81ab302b25','1c756bca-4438-3f17-a766-c8bcbe3ba400','d9072738-ac93-7ed6-167b-3c3c66d35a18','b1e8dec2-de29-3c9f-aaf2-f78fd92df3ce','9cd25f9f-3c0d-f43d-5a46-0194f0be04dd'] -['10a4718d-ab8c-49c6-c785-66ccf112f7d5','02ac2bf5-5634-a5a8-9a18-05ce8d1fb583','8037a13d-2004-08f2-f831-fa2387f5c29a','a99c4373-1121-2691-ecbb-216adbd748c7','ef0986ff-5031-0353-2f21-1de3ea53af08','778064a7-653b-ef7b-c77b-4d769b12b917','a1607e6f-691a-0ff0-b0b3-e454dae7bef7'] -['71c1b47a-c0eb-42b5-eecd-18dc585284fd','72bbf272-9ec5-09ec-f339-b5dac55c037b','26e5bce5-43f7-59b0-84c6-ef509f4c45eb','305fcbff-c366-2033-a8c5-d648f236e754','3a0d329f-f897-84e9-9e87-9501a713e63d','54bda20c-d5cd-a08a-c078-3c4fd81f4f55','43f549d1-3e5b-d5bf-ed32-b4850648bdc8','7eb6ac4f-06e0-ff48-6330-3c7afa5f2644'] -['17b9a4a5-fef8-a3f9-5af4-3b6e67ca62c9','3f524d8e-320d-00dc-c210-e199206550db'] -['005c592e-5081-9f3d-1fcb-5a9e82f39f97','29cf228d-b325-4a34-3eff-e80494a79260'] -['6c08b54b-8cf8-b96d-f087-8b54f5e72d0e'] -['7122e162-ab8b-a84a-6b71-c0846cf0204d','51c1de1a-24c7-18d6-39ed-e9023205610c'] -['f09d6779-1106-d667-e7c9-9a0cad544afe','62060fec-ee13-7c66-5da4-02c8f4d50dc9'] -['df1d0d54-d639-9c9b-2070-622fc9d82203','f23ef5b9-3797-9b0e-b8ac-67ea31b99c3e','e48afe73-9e22-7439-afed-d53b6ea204f4','d7f1ab47-4928-7623-283e-fb3f16aebeba','ea270407-d32f-a407-add2-3ae2d1113ccb','c43e9fff-2980-a1d1-f1bb-ff94d3cffbc2','a0cd54e6-0a2d-07ec-88ad-4f5d29c15b06','5e93413f-2eb9-5363-17ab-e2215b8b19e0'] +['7e90d9ed-3e18-3818-1f4a-8fc063ff735b','50462ea9-fe2f-d033-3f39-171b126331fa','6da849de-f100-c0cc-9927-a60f01acf065','6b5dc860-1d64-40a3-5d73-6910493dc3bf','29e75c9e-fd53-4f23-1e85-7066961dbe0e','b60faa27-fe38-ff30-bda6-6d4f737b3622','297f7b2f-ec54-178b-623d-6d8244222885','556e7945-df65-729e-dcb0-e0ca3a435f2e','8d6df922-9588-7e50-678f-236036acd439'] +['bdb39798-af19-13ad-9780-b53edc0f4a21','5ca17a81-ab30-2b25-c798-10de3635d333','a766c8bc-be3b-a400-1c75-6bca44383f17','167b3c3c-66d3-5a18-d907-2738ac937ed6','aaf2f78f-d92d-f3ce-b1e8-dec2de293c9f','5a460194-f0be-04dd-9cd2-5f9f3c0df43d'] +['c78566cc-f112-f7d5-10a4-718dab8c49c6','9a1805ce-8d1f-b583-02ac-2bf55634a5a8','f831fa23-87f5-c29a-8037-a13d200408f2','ecbb216a-dbd7-48c7-a99c-437311212691','2f211de3-ea53-af08-ef09-86ff50310353','c77b4d76-9b12-b917-7780-64a7653bef7b','b0b3e454-dae7-bef7-a160-7e6f691a0ff0'] +['eecd18dc-5852-84fd-71c1-b47ac0eb42b5','f339b5da-c55c-037b-72bb-f2729ec509ec','84c6ef50-9f4c-45eb-26e5-bce543f759b0','a8c5d648-f236-e754-305f-cbffc3662033','9e879501-a713-e63d-3a0d-329ff89784e9','c0783c4f-d81f-4f55-54bd-a20cd5cda08a','ed32b485-0648-bdc8-43f5-49d13e5bd5bf','63303c7a-fa5f-2644-7eb6-ac4f06e0ff48'] +['5af43b6e-67ca-62c9-17b9-a4a5fef8a3f9','c210e199-2065-50db-3f52-4d8e320d00dc'] +['1fcb5a9e-82f3-9f97-005c-592e50819f3d','3effe804-94a7-9260-29cf-228db3254a34'] +['f0878b54-f5e7-2d0e-6c08-b54b8cf8b96d'] +['6b71c084-6cf0-204d-7122-e162ab8ba84a','39ede902-3205-610c-51c1-de1a24c718d6'] +['e7c99a0c-ad54-4afe-f09d-67791106d667','5da402c8-f4d5-0dc9-6206-0fecee137c66'] +['2070622f-c9d8-2203-df1d-0d54d6399c9b','b8ac67ea-31b9-9c3e-f23e-f5b937979b0e','afedd53b-6ea2-04f4-e48a-fe739e227439','283efb3f-16ae-beba-d7f1-ab4749287623','add23ae2-d111-3ccb-ea27-0407d32fa407','f1bbff94-d3cf-fbc2-c43e-9fff2980a1d1','88ad4f5d-29c1-5b06-a0cd-54e60a2d07ec','17abe221-5b8b-19e0-5e93-413f2eb95363'] +- Tuple(Int32, Array(Int64)) (1234817989,[2254772619926532955,9120028858397505560,4555697903102013946,5784362079052877875,-7410772078432030619,7901646768096461004,6733841386518201279,7736560050027905187,2199287578947862030]) (1171957426,[3019483913099890467,-4781013766399904222,-5327852745410412752,7078934595552553093,2990244123355912075,-2544286630298820818]) @@ -152,6 +165,7 @@ Tuple(Int32, Array(Int64)) (1012211222,[-562393447932771686,-6225026423445182831]) (-2109244634,[-1388479317275096889,-1222297392734207149]) (-1647742638,[3396028458740199176,8610993157653131131,-4072576266223306473,-6818310818869145616,-5713972449102020873,8197031236106666677,-1239306987803343619,8267468115072584172]) +- FixedString(4) Ų ج_ @@ -163,6 +177,7 @@ w W  T +- String String String @@ -183,6 +198,7 @@ String \0g \0> \0XjbW:s< +- Nullable(String) )/VC)%f9 \0ih|;B @@ -194,6 +210,7 @@ Nullable(String) \0g \0> \0XjbW:s< +- Array(String) ['(|ZVAg2F','\0GXjbW','\0<^guT(','\0y M$lZ0','\03','\0p','\0','\0i','\0P'] ['\0"}YRG%B','\0T3(E^> p','\0JTaj','\0)*3','\0k%=p','\0Yub$81`X'] @@ -205,3 +222,19 @@ Array(String) ['\0k','\0 '] ['\0F','\0&h -210924.4634 w 2.143274805821858e307 ('2056-04-25','2039-04-06 20:11:02','2063-07-18 01:46:10.215','8d6df922-9588-7e50-678f-236036acd439') w +[-36,123,44,60,5,25,-5,-127] 2647224658 \0XjbW:s< -164774.2638 o 2.9425818885529257e307 ('2049-06-05','2053-11-20 07:10:58','1996-11-02 14:35:41.110','bdb39798-af19-13ad-9780-b53edc0f4a21') \r +- From 2780250d05d3b894e933358a7de553119712c21e Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Thu, 6 Feb 2020 18:32:00 +0300 Subject: [PATCH 0205/2007] better code near data part writer --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreeDataPartCompact.cpp | 2 +- .../Storages/MergeTree/MergeTreeDataPartCompact.h | 1 + .../MergeTree/MergeTreeDataPartWriterCompact.cpp | 15 +++++++-------- dbms/src/Storages/MergeTree/MergeTreeIOSettings.h | 4 +++- .../Storages/MergeTree/MergeTreeReaderCompact.cpp | 2 +- .../MergeTree/MergedColumnOnlyOutputStream.cpp | 4 ++-- .../MergeTree/MergedColumnOnlyOutputStream.h | 2 +- 8 files changed, 17 insertions(+), 15 deletions(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index aea6a392fa5..7ef141327fc 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1841,7 +1841,7 @@ void MergeTreeData::alterDataPart( nullptr /* offset_columns */, part->index_granularity, &part->index_granularity_info, - "_converting"); + true /* is_writing_temp_files */); in.readPrefix(); out.writePrefix(); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 931a365371c..abe7e34fd1f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -173,7 +173,7 @@ NameToNameMap MergeTreeDataPartCompact::createRenameMapForAlter( analysis_result.expression->add(ExpressionAction::project(projection)); - String data_temp_name = String(DATA_FILE_NAME) + "_converting"; + String data_temp_name = String(DATA_FILE_NAME) + TEMP_FILE_SUFFIX; rename_map[data_temp_name + DATA_FILE_EXTENSION] = DATA_FILE_NAME_WITH_EXTENSION; rename_map[data_temp_name + part_mrk_file_extension] = DATA_FILE_NAME + part_mrk_file_extension; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index fd1d8c18327..ec366038df9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -17,6 +17,7 @@ public: static constexpr auto DATA_FILE_NAME = "data"; static constexpr auto DATA_FILE_EXTENSION = ".bin"; + static constexpr auto TEMP_FILE_SUFFIX = "_temp"; static constexpr auto DATA_FILE_NAME_WITH_EXTENSION = "data.bin"; MergeTreeDataPartCompact( diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index 522359ad635..0a0519cb87e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -1,14 +1,9 @@ #include +#include namespace DB { -namespace -{ - constexpr auto DATA_FILE_NAME = "data"; - constexpr auto DATA_FILE_EXTENSION = ".bin"; -} - MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( const String & part_path_, @@ -24,10 +19,14 @@ MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact( indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_, true) { - String data_file_name = DATA_FILE_NAME + settings.filename_suffix; + using DataPart = MergeTreeDataPartCompact; + String data_file_name = DataPart::DATA_FILE_NAME; + if (settings.is_writing_temp_files) + data_file_name += DataPart::TEMP_FILE_SUFFIX; + stream = std::make_unique( data_file_name, - part_path + data_file_name, DATA_FILE_EXTENSION, + part_path + data_file_name, DataPart::DATA_FILE_EXTENSION, part_path + data_file_name, marks_file_extension, default_codec, settings.max_compress_block_size, diff --git a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h index 8119c076b73..c0e4384c135 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -30,8 +30,10 @@ struct MergeTreeWriterSettings size_t aio_threshold; bool can_use_adaptive_granularity; bool blocks_are_granules_size; - String filename_suffix = ""; + /// true if we write temporary files during alter. + bool is_writing_temp_files = false; size_t estimated_size = 0; + /// used when ALTERing columns if we know that array offsets are not altered. bool skip_offsets = false; }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 1d884bcab0e..0e896843d2e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -28,7 +28,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr settings.save_marks_in_cache, data_part->getColumns().size()) { size_t buffer_size = settings.max_read_buffer_size; - const String full_data_path = path + MergeTreeDataPartCompact::DATA_FILE_NAME + MergeTreeDataPartCompact::DATA_FILE_EXTENSION; + const String full_data_path = path + MergeTreeDataPartCompact::DATA_FILE_NAME_WITH_EXTENSION; if (uncompressed_cache) { diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 93e3592d819..1a6762f14f1 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -10,7 +10,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( WrittenOffsetColumns * offset_columns_, const MergeTreeIndexGranularity & index_granularity, const MergeTreeIndexGranularityInfo * index_granularity_info, - const String & filename_suffix) + bool is_writing_temp_files) : IMergedBlockOutputStream(data_part), header(header_), sync(sync_) { @@ -20,7 +20,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream( index_granularity_info ? index_granularity_info->is_adaptive : data_part->storage.canUseAdaptiveGranularity(), global_settings.min_bytes_to_use_direct_io); - writer_settings.filename_suffix = filename_suffix; + writer_settings.is_writing_temp_files = is_writing_temp_files; writer_settings.skip_offsets = skip_offsets_; writer = data_part->getWriter(header.getNamesAndTypesList(), indices_to_recalc, diff --git a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index cdcf9164882..894370f1d1a 100644 --- a/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/dbms/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -21,7 +21,7 @@ public: WrittenOffsetColumns * offset_columns_ = nullptr, const MergeTreeIndexGranularity & index_granularity = {}, const MergeTreeIndexGranularityInfo * index_granularity_info_ = nullptr, - const String & filename_suffix = ""); + bool is_writing_temp_files = false); Block getHeader() const override { return header; } void write(const Block & block) override; From 487d1ff4912ec036b1c42103b1c15a4e8b630254 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 7 Feb 2020 00:31:53 +0300 Subject: [PATCH 0206/2007] Fix system zlib headers usage --- contrib/CMakeLists.txt | 1 + contrib/boost-cmake/CMakeLists.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 8fb4c612ecb..9a2d3637943 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -262,6 +262,7 @@ if (USE_EMBEDDED_COMPILER AND USE_INTERNAL_LLVM_LIBRARY) set (LLVM_ENABLE_PIC 0 CACHE INTERNAL "") set (LLVM_TARGETS_TO_BUILD "X86;AArch64" CACHE STRING "") add_subdirectory (llvm/llvm) + target_include_directories(LLVMSupport SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) endif () if (USE_INTERNAL_LIBGSASL_LIBRARY) diff --git a/contrib/boost-cmake/CMakeLists.txt b/contrib/boost-cmake/CMakeLists.txt index 54dcd750320..582cc84a552 100644 --- a/contrib/boost-cmake/CMakeLists.txt +++ b/contrib/boost-cmake/CMakeLists.txt @@ -41,4 +41,5 @@ endif() if (USE_INTERNAL_AVRO_LIBRARY) add_boost_lib(iostreams) target_link_libraries(boost_iostreams_internal PUBLIC ${ZLIB_LIBRARIES}) + target_include_directories(boost_iostreams_internal SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) endif() From ae78c9b6bc1545fc1bd972a19674c6293409db44 Mon Sep 17 00:00:00 2001 From: proller Date: Fri, 7 Feb 2020 02:26:45 +0300 Subject: [PATCH 0207/2007] Fix zlib includes --- contrib/arrow-cmake/CMakeLists.txt | 1 + contrib/libxml2-cmake/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 1f09bba8d31..20ff5c49eea 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -348,6 +348,7 @@ if (ARROW_WITH_ZLIB) endif () if (ARROW_WITH_ZSTD) target_link_libraries(${ARROW_LIBRARY} PRIVATE ${ZSTD_LIBRARY}) + target_include_directories(${ARROW_LIBRARY} SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) endif () target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_INCLUDE_DIR}) diff --git a/contrib/libxml2-cmake/CMakeLists.txt b/contrib/libxml2-cmake/CMakeLists.txt index 71127fb9e35..f0b2135a7c8 100644 --- a/contrib/libxml2-cmake/CMakeLists.txt +++ b/contrib/libxml2-cmake/CMakeLists.txt @@ -59,4 +59,4 @@ endif() target_include_directories(libxml2 PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include) target_include_directories(libxml2 PUBLIC ${LIBXML2_SOURCE_DIR}/include) -target_include_directories(libxml2 PRIVATE ${ZLIB_INCLUDE_DIR}/include) +target_include_directories(libxml2 SYSTEM BEFORE PRIVATE ${ZLIB_INCLUDE_DIR}) From 659717638835c21d16f3670776ac2ae0b07e0961 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 30 Jan 2020 23:12:00 +0300 Subject: [PATCH 0208/2007] Added draft version of random table function. Currently unimplemented data generators --- .../DataTypes/DataTypeAggregateFunction.cpp | 4 ++ .../src/DataTypes/DataTypeAggregateFunction.h | 1 + dbms/src/DataTypes/DataTypeArray.cpp | 7 ++ dbms/src/DataTypes/DataTypeArray.h | 1 + dbms/src/DataTypes/DataTypeDecimalBase.cpp | 7 ++ dbms/src/DataTypes/DataTypeDecimalBase.h | 1 + dbms/src/DataTypes/DataTypeEnum.h | 1 + dbms/src/DataTypes/DataTypeFixedString.cpp | 6 ++ dbms/src/DataTypes/DataTypeFixedString.h | 1 + dbms/src/DataTypes/DataTypeLowCardinality.cpp | 6 ++ dbms/src/DataTypes/DataTypeLowCardinality.h | 1 + dbms/src/DataTypes/DataTypeNothing.cpp | 8 +++ dbms/src/DataTypes/DataTypeNothing.h | 1 + dbms/src/DataTypes/DataTypeNullable.cpp | 5 ++ dbms/src/DataTypes/DataTypeNullable.h | 1 + dbms/src/DataTypes/DataTypeNumberBase.cpp | 7 ++ dbms/src/DataTypes/DataTypeNumberBase.h | 1 + dbms/src/DataTypes/DataTypeSet.h | 1 + dbms/src/DataTypes/DataTypeString.cpp | 5 ++ dbms/src/DataTypes/DataTypeString.h | 1 + dbms/src/DataTypes/DataTypeTuple.cpp | 8 +++ dbms/src/DataTypes/DataTypeTuple.h | 1 + dbms/src/DataTypes/IDataType.h | 4 ++ dbms/src/DataTypes/IDataTypeDummy.h | 5 ++ .../TableFunctions/TableFunctionRandom.cpp | 69 +++++++++++++++++++ dbms/src/TableFunctions/TableFunctionRandom.h | 20 ++++++ .../TableFunctions/registerTableFunctions.cpp | 1 + .../TableFunctions/registerTableFunctions.h | 1 + .../01072_random_table_function.sql | 1 + 29 files changed, 176 insertions(+) create mode 100644 dbms/src/TableFunctions/TableFunctionRandom.cpp create mode 100644 dbms/src/TableFunctions/TableFunctionRandom.h create mode 100644 dbms/tests/queries/0_stateless/01072_random_table_function.sql diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp index 8111b1de2fe..f3b26497912 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp @@ -304,6 +304,10 @@ MutableColumnPtr DataTypeAggregateFunction::createColumn() const return ColumnAggregateFunction::create(function); } +MutableColumnPtr DataTypeAggregateFunction::createColumnWithRandomData(size_t) const +{ + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} /// Create empty state Field DataTypeAggregateFunction::getDefault() const diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.h b/dbms/src/DataTypes/DataTypeAggregateFunction.h index 9ae7c67a803..e4c226b2917 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.h +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.h @@ -63,6 +63,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index e2c03805ea8..0500182c61a 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -487,6 +487,13 @@ MutableColumnPtr DataTypeArray::createColumn() const } +MutableColumnPtr DataTypeArray::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + Field DataTypeArray::getDefault() const { return Array(); diff --git a/dbms/src/DataTypes/DataTypeArray.h b/dbms/src/DataTypes/DataTypeArray.h index 1451f27dfbe..ccf269bd357 100644 --- a/dbms/src/DataTypes/DataTypeArray.h +++ b/dbms/src/DataTypes/DataTypeArray.h @@ -94,6 +94,7 @@ public: bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.cpp b/dbms/src/DataTypes/DataTypeDecimalBase.cpp index 7b9a391427c..a0f2bd7bd82 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.cpp +++ b/dbms/src/DataTypes/DataTypeDecimalBase.cpp @@ -41,6 +41,13 @@ MutableColumnPtr DataTypeDecimalBase::createColumn() const return ColumnType::create(0, scale); } +template +MutableColumnPtr DataTypeDecimalBase::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + template void DataTypeDecimalBase::serializeBinary(const Field & field, WriteBuffer & ostr) const { diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.h b/dbms/src/DataTypes/DataTypeDecimalBase.h index 11f7490e80a..d579b965412 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.h +++ b/dbms/src/DataTypes/DataTypeDecimalBase.h @@ -83,6 +83,7 @@ public: Field getDefault() const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return true; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeEnum.h b/dbms/src/DataTypes/DataTypeEnum.h index 2cb677984df..a0408df0279 100644 --- a/dbms/src/DataTypes/DataTypeEnum.h +++ b/dbms/src/DataTypes/DataTypeEnum.h @@ -111,6 +111,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override { return ColumnType::create(); } + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index d30f1003ca0..a148d0b2d22 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -268,6 +268,12 @@ MutableColumnPtr DataTypeFixedString::createColumn() const return ColumnFixedString::create(n); } +MutableColumnPtr DataTypeFixedString::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + Field DataTypeFixedString::getDefault() const { return String(); diff --git a/dbms/src/DataTypes/DataTypeFixedString.h b/dbms/src/DataTypes/DataTypeFixedString.h index 6d1f1c4db83..4f264d3ac86 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.h +++ b/dbms/src/DataTypes/DataTypeFixedString.h @@ -70,6 +70,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.cpp b/dbms/src/DataTypes/DataTypeLowCardinality.cpp index 5db32bd5380..24dc3af48c9 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.cpp +++ b/dbms/src/DataTypes/DataTypeLowCardinality.cpp @@ -934,6 +934,12 @@ MutableColumnPtr DataTypeLowCardinality::createColumn() const return ColumnLowCardinality::create(std::move(dictionary), std::move(indexes)); } +MutableColumnPtr DataTypeLowCardinality::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + Field DataTypeLowCardinality::getDefault() const { return dictionary_type->getDefault(); diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.h b/dbms/src/DataTypes/DataTypeLowCardinality.h index f8c314909b8..9b22acea7e3 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.h +++ b/dbms/src/DataTypes/DataTypeLowCardinality.h @@ -68,6 +68,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNothing.cpp b/dbms/src/DataTypes/DataTypeNothing.cpp index 79fbb002bff..ce4990748f9 100644 --- a/dbms/src/DataTypes/DataTypeNothing.cpp +++ b/dbms/src/DataTypes/DataTypeNothing.cpp @@ -14,6 +14,14 @@ MutableColumnPtr DataTypeNothing::createColumn() const return ColumnNothing::create(0); } + +MutableColumnPtr DataTypeNothing::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + void DataTypeNothing::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { size_t size = column.size(); diff --git a/dbms/src/DataTypes/DataTypeNothing.h b/dbms/src/DataTypes/DataTypeNothing.h index e9421fb15e8..5fbe0acc0a9 100644 --- a/dbms/src/DataTypes/DataTypeNothing.h +++ b/dbms/src/DataTypes/DataTypeNothing.h @@ -19,6 +19,7 @@ public: TypeIndex getTypeId() const override { return TypeIndex::Nothing; } MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; /// These methods read and write zero bytes just to allow to figure out size of column. void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index 397d5ba0a65..6f31e66a1e5 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -488,6 +488,11 @@ MutableColumnPtr DataTypeNullable::createColumn() const return ColumnNullable::create(nested_data_type->createColumn(), ColumnUInt8::create()); } +MutableColumnPtr DataTypeNullable::createColumnWithRandomData(size_t limit) const +{ + return ColumnNullable::create(nested_data_type->createColumnWithRandomData(limit), DataTypeUInt8().createColumnWithRandomData(limit)); +} + Field DataTypeNullable::getDefault() const { return Null(); diff --git a/dbms/src/DataTypes/DataTypeNullable.h b/dbms/src/DataTypes/DataTypeNullable.h index 1766b399c2a..83a76ae0410 100644 --- a/dbms/src/DataTypes/DataTypeNullable.h +++ b/dbms/src/DataTypes/DataTypeNullable.h @@ -76,6 +76,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNumberBase.cpp b/dbms/src/DataTypes/DataTypeNumberBase.cpp index 90356817730..937967d431a 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.cpp +++ b/dbms/src/DataTypes/DataTypeNumberBase.cpp @@ -239,6 +239,13 @@ MutableColumnPtr DataTypeNumberBase::createColumn() const return ColumnVector::create(); } +template +MutableColumnPtr DataTypeNumberBase::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + template bool DataTypeNumberBase::isValueRepresentedByInteger() const { diff --git a/dbms/src/DataTypes/DataTypeNumberBase.h b/dbms/src/DataTypes/DataTypeNumberBase.h index fb752ad5329..5a3dda5fe15 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.h +++ b/dbms/src/DataTypes/DataTypeNumberBase.h @@ -45,6 +45,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return false; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index 7ef0d931279..1d0d56c164b 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -21,6 +21,7 @@ public: // Used for expressions analysis. MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } + MutableColumnPtr createColumnWithRandomData(size_t) const override; // Used only for debugging, making it DUMPABLE Field getDefault() const override { return Tuple(); } diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index ef32fe33690..46478396a68 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -360,6 +360,11 @@ MutableColumnPtr DataTypeString::createColumn() const return ColumnString::create(); } +MutableColumnPtr DataTypeString::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} bool DataTypeString::equals(const IDataType & rhs) const { diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h index 28968eef3f1..4a2c6be42e1 100644 --- a/dbms/src/DataTypes/DataTypeString.h +++ b/dbms/src/DataTypes/DataTypeString.h @@ -54,6 +54,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 4d60177aa4d..5c912b89f2d 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -454,6 +454,14 @@ MutableColumnPtr DataTypeTuple::createColumn() const return ColumnTuple::create(std::move(tuple_columns)); } + +MutableColumnPtr DataTypeTuple::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + Field DataTypeTuple::getDefault() const { return Tuple(ext::map(elems, [] (const DataTypePtr & elem) { return elem->getDefault(); })); diff --git a/dbms/src/DataTypes/DataTypeTuple.h b/dbms/src/DataTypes/DataTypeTuple.h index 06f0f62026e..a3a8fb2847e 100644 --- a/dbms/src/DataTypes/DataTypeTuple.h +++ b/dbms/src/DataTypes/DataTypeTuple.h @@ -81,6 +81,7 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & reader, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/IDataType.h b/dbms/src/DataTypes/IDataType.h index 2f9f113b955..6c4ea791981 100644 --- a/dbms/src/DataTypes/IDataType.h +++ b/dbms/src/DataTypes/IDataType.h @@ -287,6 +287,10 @@ public: */ virtual MutableColumnPtr createColumn() const = 0; + /** Create column for corresponding type and fill with random values. + */ + virtual MutableColumnPtr createColumnWithRandomData(size_t size) const = 0; + /** Create ColumnConst for corresponding type, with specified size and value. */ ColumnPtr createColumnConst(size_t size, const Field & field) const; diff --git a/dbms/src/DataTypes/IDataTypeDummy.h b/dbms/src/DataTypes/IDataTypeDummy.h index f27359e5f74..e346689274f 100644 --- a/dbms/src/DataTypes/IDataTypeDummy.h +++ b/dbms/src/DataTypes/IDataTypeDummy.h @@ -42,6 +42,11 @@ public: throw Exception("Method createColumn() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); } + MutableColumnPtr createColumnWithRandomData(size_t) const override + { + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } + Field getDefault() const override { throw Exception("Method getDefault() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp new file mode 100644 index 00000000000..f7ffe977698 --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -0,0 +1,69 @@ +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include +#include +#include + +#include "registerTableFunctions.h" + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; +} + +StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const +{ + ASTs & args_func = ast_function->children; + + if (args_func.size() != 1) + throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::LOGICAL_ERROR); + + ASTs & args = args_func.at(0)->children; + + if (args.size() > 2) + throw Exception("Table function '" + getName() + "' requires one or two arguments: structure (and limit).", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + /// Parsing first argument as table structure and creating a sample block + std::string structure = args[0]->as().value.safeGet(); + + UInt64 limit = 1; + /// Parsing second argument if present + if (args.size() == 2) + limit = args[1]->as().value.safeGet(); + + if (!limit) + throw Exception("Table function '" + getName() + "' limit should not be 0.", ErrorCodes::BAD_ARGUMENTS); + + ColumnsDescription columns = parseColumnsListFromString(structure, context); + + Block res_block; + for (const auto & name_type : columns.getOrdinary()) + Column c = name_type.type->createColumnWithRandomData(limit) ; + res_block.insert({ c, name_type.type, name_type.name }); + + auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); + res->startup(); + return res; +} + +void registerTableFunctionRandom(TableFunctionFactory & factory) +{ + factory.registerFunction(TableFunctionFactory::CaseInsensitive); +} + +} diff --git a/dbms/src/TableFunctions/TableFunctionRandom.h b/dbms/src/TableFunctions/TableFunctionRandom.h new file mode 100644 index 00000000000..c4f8e2bca37 --- /dev/null +++ b/dbms/src/TableFunctions/TableFunctionRandom.h @@ -0,0 +1,20 @@ +#pragma once + +#include + +namespace DB +{ +/* random(structure, limit) - creates a temporary storage filling columns with random data + * random is case-insensitive table function + */ +class TableFunctionRandom : public ITableFunction +{ +public: + static constexpr auto name = "generate"; + std::string getName() const override { return name; } +private: + StoragePtr executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const override; +}; + + +} diff --git a/dbms/src/TableFunctions/registerTableFunctions.cpp b/dbms/src/TableFunctions/registerTableFunctions.cpp index 35021cd46d0..91b6b94440c 100644 --- a/dbms/src/TableFunctions/registerTableFunctions.cpp +++ b/dbms/src/TableFunctions/registerTableFunctions.cpp @@ -15,6 +15,7 @@ void registerTableFunctions() registerTableFunctionURL(factory); registerTableFunctionValues(factory); registerTableFunctionInput(factory); + registerTableFunctionRandom(factory); #if USE_AWS_S3 registerTableFunctionS3(factory); diff --git a/dbms/src/TableFunctions/registerTableFunctions.h b/dbms/src/TableFunctions/registerTableFunctions.h index 66f2dda90ea..8ae5ab339f4 100644 --- a/dbms/src/TableFunctions/registerTableFunctions.h +++ b/dbms/src/TableFunctions/registerTableFunctions.h @@ -12,6 +12,7 @@ void registerTableFunctionFile(TableFunctionFactory & factory); void registerTableFunctionURL(TableFunctionFactory & factory); void registerTableFunctionValues(TableFunctionFactory & factory); void registerTableFunctionInput(TableFunctionFactory & factory); +void registerTableFunctionRandom(TableFunctionFactory & factory); #if USE_AWS_S3 void registerTableFunctionS3(TableFunctionFactory & factory); diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.sql b/dbms/tests/queries/0_stateless/01072_random_table_function.sql new file mode 100644 index 00000000000..fb217befea5 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.sql @@ -0,0 +1 @@ +SELECT * FROM random(3) From 7a4143ddc3268ade09d9bfa8d976361a163806b8 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 31 Jan 2020 09:36:29 +0300 Subject: [PATCH 0209/2007] fix --- dbms/src/TableFunctions/TableFunctionRandom.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index f7ffe977698..b68bde17550 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "registerTableFunctions.h" @@ -44,7 +45,7 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C UInt64 limit = 1; /// Parsing second argument if present if (args.size() == 2) - limit = args[1]->as().value.safeGet(); + limit = args[1]->as().value.safeGet(); if (!limit) throw Exception("Table function '" + getName() + "' limit should not be 0.", ErrorCodes::BAD_ARGUMENTS); @@ -53,8 +54,8 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C Block res_block; for (const auto & name_type : columns.getOrdinary()) - Column c = name_type.type->createColumnWithRandomData(limit) ; - res_block.insert({ c, name_type.type, name_type.name }); + MutableColumnPtr column = name_type.type->createColumnWithRandomData(limit); + res_block.insert({ column, name_type.type, name_type.name }); auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); From 11c748d91bc6f210343e4632311ea73c4e58911f Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Fri, 31 Jan 2020 10:58:41 +0300 Subject: [PATCH 0210/2007] build fix --- dbms/src/DataTypes/DataTypeEnum.cpp | 8 ++++++++ dbms/src/DataTypes/DataTypeSet.h | 5 ++++- dbms/src/TableFunctions/TableFunctionRandom.cpp | 4 +++- .../queries/0_stateless/01072_random_table_function.sql | 2 +- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index 5ca6296f43d..fcaf9f2c8a3 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -347,6 +347,14 @@ Field DataTypeEnum::castToValue(const Field & value_or_name) const } +template +MutableColumnPtr DataTypeEnum::createColumnWithRandomData(size_t limit) const +{ + (void)limit; + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); +} + + /// Explicit instantiations. template class DataTypeEnum; template class DataTypeEnum; diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index 1d0d56c164b..f468881cfe9 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -21,7 +21,10 @@ public: // Used for expressions analysis. MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } - MutableColumnPtr createColumnWithRandomData(size_t) const override; + MutableColumnPtr createColumnWithRandomData(size_t) const override + { + throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); + } // Used only for debugging, making it DUMPABLE Field getDefault() const override { return Tuple(); } diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index b68bde17550..3d4bb1d3247 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -54,8 +54,10 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C Block res_block; for (const auto & name_type : columns.getOrdinary()) + { MutableColumnPtr column = name_type.type->createColumnWithRandomData(limit); - res_block.insert({ column, name_type.type, name_type.name }); + res_block.insert({std::move(column), name_type.type, name_type.name}); + } auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.sql b/dbms/tests/queries/0_stateless/01072_random_table_function.sql index fb217befea5..21f0925439d 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.sql +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.sql @@ -1 +1 @@ -SELECT * FROM random(3) +SELECT * FROM generate('id int', 3) From 5fae4f88a4bd4958f7d61f74754f113f241a4f34 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Sun, 2 Feb 2020 00:22:00 +0300 Subject: [PATCH 0211/2007] implemented couple of generators --- .../DataTypes/DataTypeAggregateFunction.cpp | 4 - .../src/DataTypes/DataTypeAggregateFunction.h | 1 - dbms/src/DataTypes/DataTypeArray.cpp | 7 - dbms/src/DataTypes/DataTypeArray.h | 1 - dbms/src/DataTypes/DataTypeDecimalBase.cpp | 7 - dbms/src/DataTypes/DataTypeDecimalBase.h | 1 - dbms/src/DataTypes/DataTypeEnum.cpp | 8 - dbms/src/DataTypes/DataTypeEnum.h | 1 - dbms/src/DataTypes/DataTypeFixedString.cpp | 6 - dbms/src/DataTypes/DataTypeFixedString.h | 1 - dbms/src/DataTypes/DataTypeInterval.h | 2 +- dbms/src/DataTypes/DataTypeLowCardinality.cpp | 6 - dbms/src/DataTypes/DataTypeLowCardinality.h | 1 - dbms/src/DataTypes/DataTypeNothing.cpp | 8 - dbms/src/DataTypes/DataTypeNothing.h | 1 - dbms/src/DataTypes/DataTypeNullable.cpp | 5 - dbms/src/DataTypes/DataTypeNullable.h | 1 - dbms/src/DataTypes/DataTypeNumberBase.cpp | 7 - dbms/src/DataTypes/DataTypeNumberBase.h | 1 - dbms/src/DataTypes/DataTypeSet.h | 4 - dbms/src/DataTypes/DataTypeString.cpp | 5 - dbms/src/DataTypes/DataTypeString.h | 1 - dbms/src/DataTypes/DataTypeTuple.cpp | 8 - dbms/src/DataTypes/DataTypeTuple.h | 1 - dbms/src/DataTypes/IDataType.h | 4 - dbms/src/DataTypes/IDataTypeDummy.h | 5 - .../TableFunctions/TableFunctionRandom.cpp | 228 +++++++++++++++++- .../01072_random_table_function.sql | 46 +++- 28 files changed, 273 insertions(+), 98 deletions(-) diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp index f3b26497912..8111b1de2fe 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.cpp @@ -304,10 +304,6 @@ MutableColumnPtr DataTypeAggregateFunction::createColumn() const return ColumnAggregateFunction::create(function); } -MutableColumnPtr DataTypeAggregateFunction::createColumnWithRandomData(size_t) const -{ - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} /// Create empty state Field DataTypeAggregateFunction::getDefault() const diff --git a/dbms/src/DataTypes/DataTypeAggregateFunction.h b/dbms/src/DataTypes/DataTypeAggregateFunction.h index e4c226b2917..9ae7c67a803 100644 --- a/dbms/src/DataTypes/DataTypeAggregateFunction.h +++ b/dbms/src/DataTypes/DataTypeAggregateFunction.h @@ -63,7 +63,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeArray.cpp b/dbms/src/DataTypes/DataTypeArray.cpp index 0500182c61a..e2c03805ea8 100644 --- a/dbms/src/DataTypes/DataTypeArray.cpp +++ b/dbms/src/DataTypes/DataTypeArray.cpp @@ -487,13 +487,6 @@ MutableColumnPtr DataTypeArray::createColumn() const } -MutableColumnPtr DataTypeArray::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - Field DataTypeArray::getDefault() const { return Array(); diff --git a/dbms/src/DataTypes/DataTypeArray.h b/dbms/src/DataTypes/DataTypeArray.h index ccf269bd357..1451f27dfbe 100644 --- a/dbms/src/DataTypes/DataTypeArray.h +++ b/dbms/src/DataTypes/DataTypeArray.h @@ -94,7 +94,6 @@ public: bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.cpp b/dbms/src/DataTypes/DataTypeDecimalBase.cpp index a0f2bd7bd82..7b9a391427c 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.cpp +++ b/dbms/src/DataTypes/DataTypeDecimalBase.cpp @@ -41,13 +41,6 @@ MutableColumnPtr DataTypeDecimalBase::createColumn() const return ColumnType::create(0, scale); } -template -MutableColumnPtr DataTypeDecimalBase::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - template void DataTypeDecimalBase::serializeBinary(const Field & field, WriteBuffer & ostr) const { diff --git a/dbms/src/DataTypes/DataTypeDecimalBase.h b/dbms/src/DataTypes/DataTypeDecimalBase.h index d579b965412..11f7490e80a 100644 --- a/dbms/src/DataTypes/DataTypeDecimalBase.h +++ b/dbms/src/DataTypes/DataTypeDecimalBase.h @@ -83,7 +83,6 @@ public: Field getDefault() const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return true; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeEnum.cpp b/dbms/src/DataTypes/DataTypeEnum.cpp index fcaf9f2c8a3..5ca6296f43d 100644 --- a/dbms/src/DataTypes/DataTypeEnum.cpp +++ b/dbms/src/DataTypes/DataTypeEnum.cpp @@ -347,14 +347,6 @@ Field DataTypeEnum::castToValue(const Field & value_or_name) const } -template -MutableColumnPtr DataTypeEnum::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - /// Explicit instantiations. template class DataTypeEnum; template class DataTypeEnum; diff --git a/dbms/src/DataTypes/DataTypeEnum.h b/dbms/src/DataTypes/DataTypeEnum.h index a0408df0279..2cb677984df 100644 --- a/dbms/src/DataTypes/DataTypeEnum.h +++ b/dbms/src/DataTypes/DataTypeEnum.h @@ -111,7 +111,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override { return ColumnType::create(); } - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/DataTypeFixedString.cpp b/dbms/src/DataTypes/DataTypeFixedString.cpp index a148d0b2d22..d30f1003ca0 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.cpp +++ b/dbms/src/DataTypes/DataTypeFixedString.cpp @@ -268,12 +268,6 @@ MutableColumnPtr DataTypeFixedString::createColumn() const return ColumnFixedString::create(n); } -MutableColumnPtr DataTypeFixedString::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - Field DataTypeFixedString::getDefault() const { return String(); diff --git a/dbms/src/DataTypes/DataTypeFixedString.h b/dbms/src/DataTypes/DataTypeFixedString.h index 4f264d3ac86..6d1f1c4db83 100644 --- a/dbms/src/DataTypes/DataTypeFixedString.h +++ b/dbms/src/DataTypes/DataTypeFixedString.h @@ -70,7 +70,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeInterval.h b/dbms/src/DataTypes/DataTypeInterval.h index 111a2489d65..d66b329185d 100644 --- a/dbms/src/DataTypes/DataTypeInterval.h +++ b/dbms/src/DataTypes/DataTypeInterval.h @@ -12,7 +12,7 @@ namespace DB * Mostly the same as Int64. * But also tagged with interval kind. * - * Intended isage is for temporary elements in expressions, + * Intended usage is for temporary elements in expressions, * not for storing values in tables. */ class DataTypeInterval final : public DataTypeNumberBase diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.cpp b/dbms/src/DataTypes/DataTypeLowCardinality.cpp index 24dc3af48c9..5db32bd5380 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.cpp +++ b/dbms/src/DataTypes/DataTypeLowCardinality.cpp @@ -934,12 +934,6 @@ MutableColumnPtr DataTypeLowCardinality::createColumn() const return ColumnLowCardinality::create(std::move(dictionary), std::move(indexes)); } -MutableColumnPtr DataTypeLowCardinality::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - Field DataTypeLowCardinality::getDefault() const { return dictionary_type->getDefault(); diff --git a/dbms/src/DataTypes/DataTypeLowCardinality.h b/dbms/src/DataTypes/DataTypeLowCardinality.h index 9b22acea7e3..f8c314909b8 100644 --- a/dbms/src/DataTypes/DataTypeLowCardinality.h +++ b/dbms/src/DataTypes/DataTypeLowCardinality.h @@ -68,7 +68,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNothing.cpp b/dbms/src/DataTypes/DataTypeNothing.cpp index ce4990748f9..79fbb002bff 100644 --- a/dbms/src/DataTypes/DataTypeNothing.cpp +++ b/dbms/src/DataTypes/DataTypeNothing.cpp @@ -14,14 +14,6 @@ MutableColumnPtr DataTypeNothing::createColumn() const return ColumnNothing::create(0); } - -MutableColumnPtr DataTypeNothing::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - void DataTypeNothing::serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const { size_t size = column.size(); diff --git a/dbms/src/DataTypes/DataTypeNothing.h b/dbms/src/DataTypes/DataTypeNothing.h index 5fbe0acc0a9..e9421fb15e8 100644 --- a/dbms/src/DataTypes/DataTypeNothing.h +++ b/dbms/src/DataTypes/DataTypeNothing.h @@ -19,7 +19,6 @@ public: TypeIndex getTypeId() const override { return TypeIndex::Nothing; } MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; /// These methods read and write zero bytes just to allow to figure out size of column. void serializeBinaryBulk(const IColumn & column, WriteBuffer & ostr, size_t offset, size_t limit) const override; diff --git a/dbms/src/DataTypes/DataTypeNullable.cpp b/dbms/src/DataTypes/DataTypeNullable.cpp index 6f31e66a1e5..397d5ba0a65 100644 --- a/dbms/src/DataTypes/DataTypeNullable.cpp +++ b/dbms/src/DataTypes/DataTypeNullable.cpp @@ -488,11 +488,6 @@ MutableColumnPtr DataTypeNullable::createColumn() const return ColumnNullable::create(nested_data_type->createColumn(), ColumnUInt8::create()); } -MutableColumnPtr DataTypeNullable::createColumnWithRandomData(size_t limit) const -{ - return ColumnNullable::create(nested_data_type->createColumnWithRandomData(limit), DataTypeUInt8().createColumnWithRandomData(limit)); -} - Field DataTypeNullable::getDefault() const { return Null(); diff --git a/dbms/src/DataTypes/DataTypeNullable.h b/dbms/src/DataTypes/DataTypeNullable.h index 83a76ae0410..1766b399c2a 100644 --- a/dbms/src/DataTypes/DataTypeNullable.h +++ b/dbms/src/DataTypes/DataTypeNullable.h @@ -76,7 +76,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeNumberBase.cpp b/dbms/src/DataTypes/DataTypeNumberBase.cpp index 937967d431a..90356817730 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.cpp +++ b/dbms/src/DataTypes/DataTypeNumberBase.cpp @@ -239,13 +239,6 @@ MutableColumnPtr DataTypeNumberBase::createColumn() const return ColumnVector::create(); } -template -MutableColumnPtr DataTypeNumberBase::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - template bool DataTypeNumberBase::isValueRepresentedByInteger() const { diff --git a/dbms/src/DataTypes/DataTypeNumberBase.h b/dbms/src/DataTypes/DataTypeNumberBase.h index 5a3dda5fe15..fb752ad5329 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.h +++ b/dbms/src/DataTypes/DataTypeNumberBase.h @@ -45,7 +45,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; bool isParametric() const override { return false; } bool haveSubtypes() const override { return false; } diff --git a/dbms/src/DataTypes/DataTypeSet.h b/dbms/src/DataTypes/DataTypeSet.h index f468881cfe9..7ef0d931279 100644 --- a/dbms/src/DataTypes/DataTypeSet.h +++ b/dbms/src/DataTypes/DataTypeSet.h @@ -21,10 +21,6 @@ public: // Used for expressions analysis. MutableColumnPtr createColumn() const override { return ColumnSet::create(0, nullptr); } - MutableColumnPtr createColumnWithRandomData(size_t) const override - { - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } // Used only for debugging, making it DUMPABLE Field getDefault() const override { return Tuple(); } diff --git a/dbms/src/DataTypes/DataTypeString.cpp b/dbms/src/DataTypes/DataTypeString.cpp index 46478396a68..ef32fe33690 100644 --- a/dbms/src/DataTypes/DataTypeString.cpp +++ b/dbms/src/DataTypes/DataTypeString.cpp @@ -360,11 +360,6 @@ MutableColumnPtr DataTypeString::createColumn() const return ColumnString::create(); } -MutableColumnPtr DataTypeString::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} bool DataTypeString::equals(const IDataType & rhs) const { diff --git a/dbms/src/DataTypes/DataTypeString.h b/dbms/src/DataTypes/DataTypeString.h index 4a2c6be42e1..28968eef3f1 100644 --- a/dbms/src/DataTypes/DataTypeString.h +++ b/dbms/src/DataTypes/DataTypeString.h @@ -54,7 +54,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & protobuf, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; diff --git a/dbms/src/DataTypes/DataTypeTuple.cpp b/dbms/src/DataTypes/DataTypeTuple.cpp index 5c912b89f2d..4d60177aa4d 100644 --- a/dbms/src/DataTypes/DataTypeTuple.cpp +++ b/dbms/src/DataTypes/DataTypeTuple.cpp @@ -454,14 +454,6 @@ MutableColumnPtr DataTypeTuple::createColumn() const return ColumnTuple::create(std::move(tuple_columns)); } - -MutableColumnPtr DataTypeTuple::createColumnWithRandomData(size_t limit) const -{ - (void)limit; - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); -} - - Field DataTypeTuple::getDefault() const { return Tuple(ext::map(elems, [] (const DataTypePtr & elem) { return elem->getDefault(); })); diff --git a/dbms/src/DataTypes/DataTypeTuple.h b/dbms/src/DataTypes/DataTypeTuple.h index a3a8fb2847e..06f0f62026e 100644 --- a/dbms/src/DataTypes/DataTypeTuple.h +++ b/dbms/src/DataTypes/DataTypeTuple.h @@ -81,7 +81,6 @@ public: void deserializeProtobuf(IColumn & column, ProtobufReader & reader, bool allow_add_row, bool & row_added) const override; MutableColumnPtr createColumn() const override; - MutableColumnPtr createColumnWithRandomData(size_t) const override; Field getDefault() const override; void insertDefaultInto(IColumn & column) const override; diff --git a/dbms/src/DataTypes/IDataType.h b/dbms/src/DataTypes/IDataType.h index 6c4ea791981..2f9f113b955 100644 --- a/dbms/src/DataTypes/IDataType.h +++ b/dbms/src/DataTypes/IDataType.h @@ -287,10 +287,6 @@ public: */ virtual MutableColumnPtr createColumn() const = 0; - /** Create column for corresponding type and fill with random values. - */ - virtual MutableColumnPtr createColumnWithRandomData(size_t size) const = 0; - /** Create ColumnConst for corresponding type, with specified size and value. */ ColumnPtr createColumnConst(size_t size, const Field & field) const; diff --git a/dbms/src/DataTypes/IDataTypeDummy.h b/dbms/src/DataTypes/IDataTypeDummy.h index e346689274f..f27359e5f74 100644 --- a/dbms/src/DataTypes/IDataTypeDummy.h +++ b/dbms/src/DataTypes/IDataTypeDummy.h @@ -42,11 +42,6 @@ public: throw Exception("Method createColumn() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); } - MutableColumnPtr createColumnWithRandomData(size_t) const override - { - throw Exception("Method createColumnWithRandomData() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); - } - Field getDefault() const override { throw Exception("Method getDefault() is not implemented for data type " + getName(), ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index 3d4bb1d3247..1391ecdc74b 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -4,11 +4,17 @@ #include #include #include +#include +#include + #include #include #include +#include +#include + #include #include #include @@ -24,6 +30,226 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int BAD_ARGUMENTS; + extern const int BAD_TYPE_OF_FIELD; + extern const int LOGICAL_ERROR; +} + +MutableColumnPtr createColumnWithRandomData(DataTypePtr type, UInt64 limit) +{ + TypeIndex idx = type->getTypeId(); + MutableColumnPtr column = type->createColumn(); + + switch (idx) + { + case TypeIndex::Nothing: + for (UInt64 i = 0; i < limit; ++i) + { + column->insertDefault(); + } + throw Exception("Random Generator not implemented for type 'Nothing'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::UInt8: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt16: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt32: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt64: + { + pcg64 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::UInt128: + throw Exception("Random Generator not implemented for type 'UInt128'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Int8: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int16: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int32: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int64: + { + pcg64 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Int128: + throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Float32: + { + pcg32 generator(randomSeed()); + double d; + for (UInt64 i = 0; i < limit; ++i) + { + d = std::numeric_limits::max(); + column->insert( (d / pcg32::max()) * generator() ); + } + } + break; + case TypeIndex::Float64: + { + pcg64 generator(randomSeed()); + double d; + for (UInt64 i = 0; i < limit; ++i) + { + d = std::numeric_limits::max(); + column->insert( (d / pcg64::max()) * generator() ); + } + } + break; + case TypeIndex::Date: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::DateTime: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::DateTime64: + { + UInt32 scale; + if (auto * ptype = typeid_cast(type.get())) + scale = ptype->getScale(); + else + throw Exception("Static cast to DataTypeDateTime64 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + UInt32 fractional = static_cast(generator()) % intExp10(scale); + UInt32 whole = static_cast(generator()); + DateTime64 dt = DecimalUtils::decimalFromComponents(whole, fractional, scale); + column->insert(DecimalField(dt, scale)); + } + } + break; + case TypeIndex::String: + throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::FixedString: + throw Exception("Random Generator not implemented for type 'FixedString'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Enum8: + throw Exception("Random Generator not implemented for type 'Enum8'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Enum16: + throw Exception("Random Generator not implemented for type 'Enum16'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Decimal32: + { + pcg32 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Decimal64: + { + pcg64 generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Decimal128: + throw Exception("Random Generator not implemented for type 'Decimal128'.", ErrorCodes::NOT_IMPLEMENTED); +/* + { + UInt32 scale = 0; + if (auto * ptype = typeid_cast *>(type.get())) + scale = ptype->getScale(); + else + throw Exception("Static cast to Decimal128 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + + pcg128_once_insecure generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) + { + column->insert(DecimalField(static_cast(generator()), scale)); + } + } + break; +*/ + case TypeIndex::UUID: + { + pcg128_once_insecure generator(randomSeed()); + for (UInt64 i = 0; i < limit; ++i) { + column->insert(static_cast(generator())); + } + } + break; + case TypeIndex::Array: + throw Exception("Random Generator not implemented for type 'Array'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Tuple: + throw Exception("Random Generator not implemented for type 'Tuple'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Set: + throw Exception("Random Generator not implemented for type 'Set'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Interval: + throw Exception("Type 'Interval' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); + case TypeIndex::Nullable: + throw Exception("Random Generator not implemented for type 'Nullable'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::Function: + throw Exception("Random Generator not implemented for type 'Function'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::AggregateFunction: + throw Exception("Random Generator not implemented for type 'AggregateFunction'.", ErrorCodes::NOT_IMPLEMENTED); + case TypeIndex::LowCardinality: + throw Exception("Random Generator not implemented for type 'LowCardinality'.", ErrorCodes::NOT_IMPLEMENTED); + } + return column; } StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const @@ -55,7 +281,7 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C Block res_block; for (const auto & name_type : columns.getOrdinary()) { - MutableColumnPtr column = name_type.type->createColumnWithRandomData(limit); + MutableColumnPtr column = createColumnWithRandomData(name_type.type, limit); res_block.insert({std::move(column), name_type.type, name_type.name}); } diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.sql b/dbms/tests/queries/0_stateless/01072_random_table_function.sql index 21f0925439d..c81d630d9b5 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.sql +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.sql @@ -1 +1,45 @@ -SELECT * FROM generate('id int', 3) +SELECT +toTypeName(ui64), toTypeName(i64), +toTypeName(ui32), toTypeName(i32), +toTypeName(ui16), toTypeName(i16), +toTypeName(ui8), toTypeName(i8) +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 1); +SELECT +ui64, i64, +ui32, i32, +ui16, i16, +ui8, i8 +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10); +SELECT +toTypeName(d), toTypeName(dt), toTypeName(dtm) +FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 1); +SELECT +d, dt, dtm +FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 10) FORMAT JSONEachRow;; +SELECT +toTypeName(dt64), toTypeName(dts64), toTypeName(dtms64) +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 1); +SELECT +dt64, dts64, dtms64 +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10) FORMAT JSONEachRow; +SELECT +dt64, dts64, dtms64 +FROM generate('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Europe/Moscow\')', 10); +SELECT + toTypeName(f32), toTypeName(f64) +FROM generate('f32 Float32, f64 Float64', 1); +SELECT + f32, f64 +FROM generate('f32 Float32, f64 Float64', 10) FORMAT JSONEachRow; +SELECT + toTypeName(d32), toTypeName(d64) +FROM generate('d32 Decimal32(4), d64 Decimal64(8)', 1); +SELECT + d32, d64 +FROM generate('d32 Decimal32(4), d64 Decimal64(8)', 10) FORMAT JSONEachRow; +SELECT + toTypeName(i) +FROM generate('i Interval', 10); +SELECT + i +FROM generate('i Interval', 10) FORMAT JSONEachRow; From 907037bcca01bd56ab7bbcc399c26158af0fde8e Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 4 Feb 2020 13:59:26 +0300 Subject: [PATCH 0212/2007] Fix everything, add docs. --- dbms/src/DataTypes/DataTypeNumberBase.cpp | 2 +- .../TableFunctions/TableFunctionRandom.cpp | 440 ++++++++++++------ .../01072_random_table_function.reference | 207 ++++++++ .../01072_random_table_function.sql | 133 +++++- .../table_functions/generate.md | 38 ++ .../table_functions/generate.md | 1 + .../table_functions/generate.md | 37 ++ .../table_functions/generate.md | 1 + 8 files changed, 705 insertions(+), 154 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01072_random_table_function.reference create mode 100644 docs/en/query_language/table_functions/generate.md create mode 120000 docs/ja/query_language/table_functions/generate.md create mode 100644 docs/ru/query_language/table_functions/generate.md create mode 120000 docs/zh/query_language/table_functions/generate.md diff --git a/dbms/src/DataTypes/DataTypeNumberBase.cpp b/dbms/src/DataTypes/DataTypeNumberBase.cpp index 90356817730..ce01269bc4d 100644 --- a/dbms/src/DataTypes/DataTypeNumberBase.cpp +++ b/dbms/src/DataTypes/DataTypeNumberBase.cpp @@ -257,7 +257,7 @@ template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; -template class DataTypeNumberBase; +template class DataTypeNumberBase; // used only in UUID template class DataTypeNumberBase; template class DataTypeNumberBase; template class DataTypeNumberBase; diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index 1391ecdc74b..d70c8a73c63 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -4,8 +4,17 @@ #include #include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include #include @@ -34,222 +43,355 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -MutableColumnPtr createColumnWithRandomData(DataTypePtr type, UInt64 limit) +void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, + UInt64 max_array_length, UInt64 max_string_length, UInt64 random_seed) { TypeIndex idx = type->getTypeId(); - MutableColumnPtr column = type->createColumn(); + if (!random_seed) + random_seed = randomSeed(); + (void) max_string_length; switch (idx) { case TypeIndex::Nothing: - for (UInt64 i = 0; i < limit; ++i) - { - column->insertDefault(); - } throw Exception("Random Generator not implemented for type 'Nothing'.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::UInt8: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt16: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg64 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::UInt128: - throw Exception("Random Generator not implemented for type 'UInt128'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("There is no DataType 'UInt128' support.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::Int8: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int16: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg64 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::Int128: - throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("There is no DataType 'Int128' support.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::Float32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + double d = 1.0; + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - double d; - for (UInt64 i = 0; i < limit; ++i) - { - d = std::numeric_limits::max(); - column->insert( (d / pcg32::max()) * generator() ); - } + d = std::numeric_limits::max(); + data[i] = (d / pcg32::max()) * generator(); } break; + } case TypeIndex::Float64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + double d = 1.0; + for (UInt64 i = 0; i < limit; ++i) { - pcg64 generator(randomSeed()); - double d; - for (UInt64 i = 0; i < limit; ++i) - { - d = std::numeric_limits::max(); - column->insert( (d / pcg64::max()) * generator() ); - } + d = std::numeric_limits::max(); + data[i] = (d / pcg64::max()) * generator(); } break; + } case TypeIndex::Date: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::DateTime: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } + data[i] = static_cast(generator()); } break; + } case TypeIndex::DateTime64: + { + UInt32 scale; + if (auto * ptype = typeid_cast(type.get())) + scale = ptype->getScale(); + else + throw Exception("Static cast to DataTypeDateTime64 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { - UInt32 scale; - if (auto * ptype = typeid_cast(type.get())) - scale = ptype->getScale(); - else - throw Exception("Static cast to DataTypeDateTime64 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - UInt32 fractional = static_cast(generator()) % intExp10(scale); - UInt32 whole = static_cast(generator()); - DateTime64 dt = DecimalUtils::decimalFromComponents(whole, fractional, scale); - column->insert(DecimalField(dt, scale)); - } + UInt32 fractional = static_cast(generator()) % intExp10(scale); + UInt32 whole = static_cast(generator()); + DateTime64 dt = DecimalUtils::decimalFromComponents(whole, fractional, scale); + data[i] = dt; } break; + } case TypeIndex::String: - throw Exception("Random Generator not implemented for type '" + String(TypeName::get()) + "'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::FixedString: - throw Exception("Random Generator not implemented for type 'FixedString'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Enum8: - throw Exception("Random Generator not implemented for type 'Enum8'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Enum16: - throw Exception("Random Generator not implemented for type 'Enum16'.", ErrorCodes::NOT_IMPLEMENTED); - case TypeIndex::Decimal32: - { - pcg32 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } - } - break; - case TypeIndex::Decimal64: - { - pcg64 generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(static_cast(generator())); - } - } - break; - case TypeIndex::Decimal128: - throw Exception("Random Generator not implemented for type 'Decimal128'.", ErrorCodes::NOT_IMPLEMENTED); -/* - { - UInt32 scale = 0; - if (auto * ptype = typeid_cast *>(type.get())) - scale = ptype->getScale(); - else - throw Exception("Static cast to Decimal128 failed ", ErrorCodes::BAD_TYPE_OF_FIELD); + { + auto & column_string = typeid_cast(column); + auto & offsets = column_string.getOffsets(); + auto & chars = column_string.getChars(); - pcg128_once_insecure generator(randomSeed()); - for (UInt64 i = 0; i < limit; ++i) - { - column->insert(DecimalField(static_cast(generator()), scale)); - } - } - break; -*/ - case TypeIndex::UUID: + UInt64 offset = 0; { - pcg128_once_insecure generator(randomSeed()); + pcg32 generator(random_seed); + offsets.resize(limit); for (UInt64 i = 0; i < limit; ++i) { - column->insert(static_cast(generator())); + offset += 1 + static_cast(generator()) % max_string_length; + offsets[i] = offset - 1; + } + chars.resize(offset); + for (UInt64 i = 0; i < offset; ++i) { + chars[i] = 32 + generator() % 95; + } + // add terminating zero char + for (auto & i : offsets) + { + chars[i] = 0; } } break; + } + case TypeIndex::FixedString: + { + auto & column_string = typeid_cast(column); + size_t len = column_string.sizeOfValueIfFixed(); + auto & chars = column_string.getChars(); + + UInt64 num_chars = static_cast(len) * limit; + { + pcg32 generator(random_seed); + chars.resize(num_chars); + for (UInt64 i = 0; i < num_chars; ++i) { + chars[i] = static_cast(generator()); + } + } + break; + } + case TypeIndex::Enum8: + { + auto values = typeid_cast *>(type.get())->getValues(); + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + + UInt8 size = values.size(); + UInt8 off; + for (UInt64 i = 0; i < limit; ++i) + { + off = static_cast(generator()) % size; + data[i] = values[off].second; + } + break; + } + case TypeIndex::Enum16: + { + auto values = typeid_cast *>(type.get())->getValues(); + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + + UInt16 size = values.size(); + UInt8 off; + for (UInt64 i = 0; i < limit; ++i) + { + off = static_cast(generator()) % size; + data[i] = values[off].second; + } + break; + } + case TypeIndex::Decimal32: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg32 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) + { + data[i] = static_cast(generator()); + } + break; + } + case TypeIndex::Decimal64: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) + { + data[i] = static_cast(generator()); + } + break; + } + case TypeIndex::Decimal128: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) + { + Int128 x = static_cast(generator()) << 64 | static_cast(generator()); + data[i] = x; + } + } + break; + case TypeIndex::UUID: + { + auto & data = typeid_cast &>(column).getData(); + data.resize(limit); + pcg64 generator(random_seed); + for (UInt64 i = 0; i < limit; ++i) { + auto x = UInt128(generator(), generator()); + data[i] = x; + } + } + break; case TypeIndex::Array: - throw Exception("Random Generator not implemented for type 'Array'.", ErrorCodes::NOT_IMPLEMENTED); + { + auto & column_array = typeid_cast(column); + auto nested_type = typeid_cast(type.get())->getNestedType(); + + auto & offsets = column_array.getOffsets(); + IColumn & data = column_array.getData(); + + UInt64 offset = 0; + { + pcg32 generator(random_seed); + offsets.resize(limit); + for (UInt64 i = 0; i < limit; ++i) { + offset += static_cast(generator()) % max_array_length; + offsets[i] = offset; + } + } + fillColumnWithRandomData(data, nested_type, offset, max_array_length, max_string_length, random_seed); + break; + } case TypeIndex::Tuple: - throw Exception("Random Generator not implemented for type 'Tuple'.", ErrorCodes::NOT_IMPLEMENTED); + { + auto &column_tuple = typeid_cast(column); + auto elements = typeid_cast(type.get())->getElements(); + + for (size_t i = 0; i < column_tuple.tupleSize(); ++i) + { + fillColumnWithRandomData(column_tuple.getColumn(i), elements[i], limit, max_array_length, max_string_length, random_seed); + } + break; + } case TypeIndex::Set: - throw Exception("Random Generator not implemented for type 'Set'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Type 'Set' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); case TypeIndex::Interval: throw Exception("Type 'Interval' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); case TypeIndex::Nullable: - throw Exception("Random Generator not implemented for type 'Nullable'.", ErrorCodes::NOT_IMPLEMENTED); + { + auto & column_nullable = typeid_cast(column); + auto nested_type = typeid_cast(type.get())->getNestedType(); + + auto & null_map = column_nullable.getNullMapData(); + IColumn & nested_column = column_nullable.getNestedColumn(); + + fillColumnWithRandomData(nested_column, nested_type, limit, max_array_length, max_string_length, random_seed); + + pcg32 generator(random_seed); + null_map.resize(limit); + for (UInt64 i = 0; i < limit; ++i) { + null_map[i] = generator() < 1024; + } + break; + } case TypeIndex::Function: - throw Exception("Random Generator not implemented for type 'Function'.", ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Type 'Funclion' can not be stored in a table.", ErrorCodes::LOGICAL_ERROR); case TypeIndex::AggregateFunction: throw Exception("Random Generator not implemented for type 'AggregateFunction'.", ErrorCodes::NOT_IMPLEMENTED); case TypeIndex::LowCardinality: throw Exception("Random Generator not implemented for type 'LowCardinality'.", ErrorCodes::NOT_IMPLEMENTED); } - return column; } StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const Context & context, const std::string & table_name) const @@ -261,30 +403,48 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C ASTs & args = args_func.at(0)->children; - if (args.size() > 2) - throw Exception("Table function '" + getName() + "' requires one or two arguments: structure (and limit).", + if (args.size() > 5) + throw Exception("Table function '" + getName() + "' requires at most five arguments: "\ + " structure, limit, max_array_length, max_string_length, random_seed.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); /// Parsing first argument as table structure and creating a sample block std::string structure = args[0]->as().value.safeGet(); UInt64 limit = 1; + UInt64 max_array_length = 10; + UInt64 max_string_length = 10; + UInt64 random_seed = 0; // zero for random + /// Parsing second argument if present - if (args.size() == 2) + if (args.size() >= 2) limit = args[1]->as().value.safeGet(); if (!limit) throw Exception("Table function '" + getName() + "' limit should not be 0.", ErrorCodes::BAD_ARGUMENTS); + if (args.size() >= 3) + max_array_length = args[1]->as().value.safeGet(); + + if (args.size() >= 4) + max_string_length = args[1]->as().value.safeGet(); + + if (args.size() == 5) + random_seed = args[1]->as().value.safeGet(); + ColumnsDescription columns = parseColumnsListFromString(structure, context); Block res_block; for (const auto & name_type : columns.getOrdinary()) { - MutableColumnPtr column = createColumnWithRandomData(name_type.type, limit); + MutableColumnPtr column = name_type.type->createColumn(); res_block.insert({std::move(column), name_type.type, name_type.name}); } + for (auto & ctn : res_block.getColumnsWithTypeAndName()) + { + fillColumnWithRandomData(ctn.column->assumeMutableRef(), ctn.type, limit, max_array_length, max_string_length, random_seed); + } auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); res->startup(); return res; diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.reference b/dbms/tests/queries/0_stateless/01072_random_table_function.reference new file mode 100644 index 00000000000..93ea1861756 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.reference @@ -0,0 +1,207 @@ +Enum8(\'hello\' = 1, \'world\' = 5) +world +hello +world +hello +hello +hello +world +hello +hello +hello +Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) +['world','hello','world','hello','hello','hello','world','hello','hello'] +['hello','world','world','hello','world','world'] +['hello','world','hello','hello','world','world','world'] +['world','world','world','world','world','world','hello','hello'] +['world','hello'] +['hello','hello'] +['world'] +['hello','hello'] +['hello','hello'] +['hello','world','hello','hello','world','world','world','world'] +Nullable(Enum16(\'o\' = -200, \'h\' = 1, \'w\' = 5)) +o +w +w +w +h +w +h +h +w +o +UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 +2254772619926532955 2254772619926532955 1234817989 1234817989 54213 -11323 197 -59 +9120028858397505560 9120028858397505560 1171957426 1171957426 42674 -22862 178 -78 +4555697903102013946 4555697903102013946 275100647 275100647 46055 -19481 231 -25 +5784362079052877875 5784362079052877875 1033685688 1033685688 51896 -13640 184 -72 +11035971995277520997 -7410772078432030619 180895192 180895192 15832 15832 216 -40 +7901646768096461004 7901646768096461004 135557292 135557292 28844 28844 172 -84 +6733841386518201279 6733841386518201279 716914271 716914271 15967 15967 95 95 +7736560050027905187 7736560050027905187 1012211222 1012211222 7702 7702 22 22 +2199287578947862030 2199287578947862030 2185722662 -2109244634 31526 31526 38 38 +3019483913099890467 3019483913099890467 2647224658 -1647742638 29010 29010 82 82 +Date DateTime DateTime(\'Europe/Moscow\') +2106-02-07 2009-02-16 23:59:49 2009-02-16 23:59:49 +2086-11-02 2007-02-20 10:43:46 2007-02-20 10:43:46 +2096-02-04 1978-09-20 03:50:47 1978-09-20 03:50:47 +2106-02-07 2002-10-04 02:54:48 2002-10-04 02:54:48 +2013-05-07 1975-09-25 19:39:52 1975-09-25 19:39:52 +2048-12-21 1974-04-19 01:48:12 1974-04-19 01:48:12 +2013-09-19 1992-09-19 18:51:11 1992-09-19 18:51:11 +1991-02-02 2002-01-28 12:47:02 2002-01-28 12:47:02 +2056-04-25 2039-04-06 20:11:02 2039-04-06 20:11:02 +2049-06-05 2053-11-20 07:10:58 2053-11-20 07:10:58 +DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') +2007-02-20 10:43:46.989 2007-02-20 10:43:46.817989 2007-02-20 10:43:46.817989 +2002-10-04 02:54:48.647 2002-10-04 02:54:48.100647 2002-10-04 02:54:48.100647 +1974-04-19 01:48:12.192 1974-04-19 01:48:12.895192 1974-04-19 01:48:12.895192 +2002-01-28 12:47:02.271 2002-01-28 12:47:02.914271 2002-01-28 12:47:02.914271 +2053-11-20 07:10:58.662 2053-11-20 07:10:58.722662 2053-11-20 07:10:58.722662 +1986-04-08 19:07:15.849 1986-04-08 19:07:15.510849 1986-04-08 19:07:15.510849 +2081-03-06 04:00:55.914 2081-03-06 04:00:55.448914 2081-03-06 04:00:55.448914 +1979-01-20 20:39:20.939 1979-01-20 20:39:20.162939 1979-01-20 20:39:20.162939 +2063-07-18 01:46:10.215 2063-07-18 01:46:10.908215 2063-07-18 01:46:10.908215 +1996-11-02 14:35:41.110 1996-11-02 14:35:41.183110 1996-11-02 14:35:41.183110 +Float32 Float64 +9.783235e37 2.1973467205491123e307 +9.285203e37 8.887754501811354e307 +2.1795718e37 4.4396706606805647e307 +8.1897013e37 5.637042481600483e307 +1.4331993e37 1.07549012514996e308 +1.0739954e37 7.700402896226395e307 +5.67998e37 6.562339881458101e307 +8.019563e37 7.539520705557441e307 +1.7317079e38 2.143274805821858e307 +2.0973474e38 2.9425818885529257e307 +Decimal32(4) Decimal64(8) Decimal64(8) +123481.7989 22547726199.26532955 4159321346419233104838.6879832895010840 +117195.7426 91200288583.97505560 8403779329565810688767.7049545291714611 +27510.0647 45556979031.02013946 -13670461591942827725055.0250490776469300 +103368.5688 57843620790.52877875 12421744869005473959544.2499747955622051 +18089.5192 -74107720784.32030619 4056969511333950153663.4915186231430947 +13555.7292 79016467680.96461004 -8819413736166121578589.4583420666183888 +71691.4271 67338413865.18201279 13058329479868658041313.8432372419860363 +101221.1222 77365600500.27905187 -4693380431928321782727.0243506636623202 +-210924.4634 21992875789.47862030 13765369952377767241248.9441272127848016 +-164774.2638 30194839130.99890467 -13890064946313418575619.0315227826809939 +UUID +1f4a8fc0-63ff-735b-7e90-d9ed3e183818 +3f39171b-1263-31fa-5046-2ea9fe2fd033 +9927a60f-01ac-f065-6da8-49def100c0cc +5d736910-493d-c3bf-6b5d-c8601d6440a3 +1e857066-961d-be0e-29e7-5c9efd534f23 +bda66d4f-737b-3622-b60f-aa27fe38ff30 +623d6d82-4422-2885-297f-7b2fec54178b +dcb0e0ca-3a43-5f2e-556e-7945df65729e +678f2360-36ac-d439-8d6d-f92295887e50 +9780b53e-dc0f-4a21-bdb3-9798af1913ad +Tuple(Int32, Int64) +(1234817989,2254772619926532955) +(1171957426,9120028858397505560) +(275100647,4555697903102013946) +(1033685688,5784362079052877875) +(180895192,-7410772078432030619) +(135557292,7901646768096461004) +(716914271,6733841386518201279) +(1012211222,7736560050027905187) +(-2109244634,2199287578947862030) +(-1647742638,3019483913099890467) +Array(Int8) +[-59,-78,-25,-72,-40,-84,95,22,38] +[82,65,35,-110,-57,-69] +[72,119,-78,-58,13,39,-71] +[81,107,-11,-63,-59,69,-80,-122] +[87,-76] +[22,-84] +[-45] +[-40,84] +[-104,-86] +[-36,123,44,60,5,25,-5,-127] +Array(Nullable(Int32)) +[1234817989,1171957426,275100647,1033685688,180895192,135557292,716914271,1012211222,-2109244634] +[-1647742638,319510849,513356835,-1966518382,-786518841,269162939] +[285701960,1943908215,-1343029326,1474183110,846934541,1007818023,-1664171079] +[195050577,371018347,734173429,2001591233,-1812297275,1172704837,-728923984,774864518] +[-462583209,-1520633676] +[-638906858,1986832300] +[378774483] +[-1399152424,-953863084] +[733724312,-23652950] +[371735004,462118779,148602156,-1055384004,-1041274619,247762201,522289659,822210177] +Array(Nullable(UUID)) +['1f4a8fc0-63ff-735b-7e90-d9ed3e183818','3f39171b-1263-31fa-5046-2ea9fe2fd033','9927a60f-01ac-f065-6da8-49def100c0cc','5d736910-493d-c3bf-6b5d-c8601d6440a3','1e857066-961d-be0e-29e7-5c9efd534f23','bda66d4f-737b-3622-b60f-aa27fe38ff30','623d6d82-4422-2885-297f-7b2fec54178b','dcb0e0ca-3a43-5f2e-556e-7945df65729e','678f2360-36ac-d439-8d6d-f92295887e50'] +['9780b53e-dc0f-4a21-bdb3-9798af1913ad','c79810de-3635-d333-5ca1-7a81ab302b25','1c756bca-4438-3f17-a766-c8bcbe3ba400','d9072738-ac93-7ed6-167b-3c3c66d35a18','b1e8dec2-de29-3c9f-aaf2-f78fd92df3ce','9cd25f9f-3c0d-f43d-5a46-0194f0be04dd'] +['10a4718d-ab8c-49c6-c785-66ccf112f7d5','02ac2bf5-5634-a5a8-9a18-05ce8d1fb583','8037a13d-2004-08f2-f831-fa2387f5c29a','a99c4373-1121-2691-ecbb-216adbd748c7','ef0986ff-5031-0353-2f21-1de3ea53af08','778064a7-653b-ef7b-c77b-4d769b12b917','a1607e6f-691a-0ff0-b0b3-e454dae7bef7'] +['71c1b47a-c0eb-42b5-eecd-18dc585284fd','72bbf272-9ec5-09ec-f339-b5dac55c037b','26e5bce5-43f7-59b0-84c6-ef509f4c45eb','305fcbff-c366-2033-a8c5-d648f236e754','3a0d329f-f897-84e9-9e87-9501a713e63d','54bda20c-d5cd-a08a-c078-3c4fd81f4f55','43f549d1-3e5b-d5bf-ed32-b4850648bdc8','7eb6ac4f-06e0-ff48-6330-3c7afa5f2644'] +['17b9a4a5-fef8-a3f9-5af4-3b6e67ca62c9','3f524d8e-320d-00dc-c210-e199206550db'] +['005c592e-5081-9f3d-1fcb-5a9e82f39f97','29cf228d-b325-4a34-3eff-e80494a79260'] +['6c08b54b-8cf8-b96d-f087-8b54f5e72d0e'] +['7122e162-ab8b-a84a-6b71-c0846cf0204d','51c1de1a-24c7-18d6-39ed-e9023205610c'] +['f09d6779-1106-d667-e7c9-9a0cad544afe','62060fec-ee13-7c66-5da4-02c8f4d50dc9'] +['df1d0d54-d639-9c9b-2070-622fc9d82203','f23ef5b9-3797-9b0e-b8ac-67ea31b99c3e','e48afe73-9e22-7439-afed-d53b6ea204f4','d7f1ab47-4928-7623-283e-fb3f16aebeba','ea270407-d32f-a407-add2-3ae2d1113ccb','c43e9fff-2980-a1d1-f1bb-ff94d3cffbc2','a0cd54e6-0a2d-07ec-88ad-4f5d29c15b06','5e93413f-2eb9-5363-17ab-e2215b8b19e0'] +Tuple(Int32, Array(Int64)) +(1234817989,[2254772619926532955,9120028858397505560,4555697903102013946,5784362079052877875,-7410772078432030619,7901646768096461004,6733841386518201279,7736560050027905187,2199287578947862030]) +(1171957426,[3019483913099890467,-4781013766399904222,-5327852745410412752,7078934595552553093,2990244123355912075,-2544286630298820818]) +(275100647,[6155991081669718686,7462222003717329977,-8255668614967296432,-7529819295378879967,-4777308097681484883,-4064480117123591373,6674750820081216293]) +(1033685688,[2050663721809231639,-6384194708780112896,-2808232718275215658,1619954721090656792,-5627002805867168609,-6128563945701772338,-7146544521171569603,6504888450989032669]) +(180895192,[1199208254069819846,-4069733657855461419]) +(135557292,[192577216783361448,-7343112807738526333]) +(716914271,[-9207713629233477390]) +(1012211222,[-562393447932771686,-6225026423445182831]) +(-2109244634,[-1388479317275096889,-1222297392734207149]) +(-1647742638,[3396028458740199176,8610993157653131131,-4072576266223306473,-6818310818869145616,-5713972449102020873,8197031236106666677,-1239306987803343619,8267468115072584172]) +FixedString(4) +Ų +ج_ +&RA# +ǻH +w\r +\'Qk +E +W + +T +String +String +String +String +String +String +String +String +String +String +)/VC)%f9 +\0ih|;B +\0J"Z,kd +\0m"m]$35 +\00 +\0( +\0 +\0g +\0> +\0XjbW:s< +Nullable(String) +)/VC)%f9 +\0ih|;B +\0J"Z,kd +\0m"m]$35 +\00 +\0( +\0 +\0g +\0> +\0XjbW:s< +Array(String) +['(|ZVAg2F','\0GXjbW','\0<^guT(','\0y M$lZ0','\03','\0p','\0','\0i','\0P'] +['\0"}YRG%B','\0T3(E^> p','\0JTaj','\0)*3','\0k%=p','\0Yub$81`X'] +['','\0\\p]|]','\05','\0k$C/pnA'] +['\0ryz{*p',''] +['\07`mjt*G',''] +['\0~g'] +['\0k','\0 '] +['\0F','\0&h diff --git a/docs/ja/query_language/table_functions/generate.md b/docs/ja/query_language/table_functions/generate.md new file mode 120000 index 00000000000..de0b0a41754 --- /dev/null +++ b/docs/ja/query_language/table_functions/generate.md @@ -0,0 +1 @@ +en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/ru/query_language/table_functions/generate.md b/docs/ru/query_language/table_functions/generate.md new file mode 100644 index 00000000000..11d7f7073a9 --- /dev/null +++ b/docs/ru/query_language/table_functions/generate.md @@ -0,0 +1,37 @@ +# generate + +Генерирует случайные данные с заданной схемой. +Позволяет заполнять тестовые таблицы данными. +Поддерживает все типы данных, которые могут храниться в таблице, за исключением LowCardinality, AggregateFunction. + +```sql +generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); +``` + +**Входные параметры** +- `name` — название соответствующего столбца. +- `TypeName` — тип соответствующего столбца. +- `limit` — количество строк для генерации. +- `max_array_length` — максимальная длина массива для всех сгенерированных массивов. По умолчанию `10`. +- `max_string_length` — максимальная длина строки для всех генерируемых строк. По умолчанию `10`. +- `random_seed` — укажите состояние генератора случайных чисел вручную, чтобы получить стабильные результаты. По умолчанию `0` - генератор инициализируется случайным состоянием. + +**Возвращаемое значение** + +Объект таблицы с запрошенной схемой. + +## Пример + + +```sql +SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); +``` +```text +┌─a────────┬────────────d─┬─c──────────────────────────────────────────────────────────────────┐ +│ [77] │ -124167.6723 │ ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') │ +│ [32,110] │ -141397.7312 │ ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') │ +│ [68] │ -67417.0770 │ ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') │ +└──────────┴──────────────┴────────────────────────────────────────────────────────────────────┘ +``` + +[Оригинальная статья](https://clickhouse.tech/docs/ru/query_language/table_functions/generate/) diff --git a/docs/zh/query_language/table_functions/generate.md b/docs/zh/query_language/table_functions/generate.md new file mode 120000 index 00000000000..de0b0a41754 --- /dev/null +++ b/docs/zh/query_language/table_functions/generate.md @@ -0,0 +1 @@ +en/query_language/table_functions/generate.md \ No newline at end of file From 9e542f6cb0a9b0788adb86bce159827878cb8fc9 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 4 Feb 2020 15:25:09 +0300 Subject: [PATCH 0213/2007] doc fixes --- docs/en/query_language/table_functions/generate.md | 2 +- docs/ru/query_language/table_functions/generate.md | 2 +- docs/toc_en.yml | 1 + docs/toc_fa.yml | 1 + docs/toc_ja.yml | 3 ++- docs/toc_ru.yml | 1 + docs/toc_zh.yml | 1 + 7 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/en/query_language/table_functions/generate.md b/docs/en/query_language/table_functions/generate.md index c04ebff8a1a..ed9e2150b03 100644 --- a/docs/en/query_language/table_functions/generate.md +++ b/docs/en/query_language/table_functions/generate.md @@ -2,7 +2,7 @@ Generates random data with given schema. Allows to populate test tables with data. -Supports all data types that can be stored in table except LowCardinality, AggregateFunction. +Supports all data types that can be stored in table except `LowCardinality` and `AggregateFunction`. ```sql generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); diff --git a/docs/ru/query_language/table_functions/generate.md b/docs/ru/query_language/table_functions/generate.md index 11d7f7073a9..53544d16e7d 100644 --- a/docs/ru/query_language/table_functions/generate.md +++ b/docs/ru/query_language/table_functions/generate.md @@ -2,7 +2,7 @@ Генерирует случайные данные с заданной схемой. Позволяет заполнять тестовые таблицы данными. -Поддерживает все типы данных, которые могут храниться в таблице, за исключением LowCardinality, AggregateFunction. +Поддерживает все типы данных, которые могут храниться в таблице, за исключением `LowCardinality` и `AggregateFunction`. ```sql generate('name TypeName[, name TypeName]...', 'limit'[, 'max_array_length'[, 'max_string_length'[, 'random_seed']]]); diff --git a/docs/toc_en.yml b/docs/toc_en.yml index 8558216b15b..76d115045e7 100644 --- a/docs/toc_en.yml +++ b/docs/toc_en.yml @@ -142,6 +142,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_fa.yml b/docs/toc_fa.yml index bd1e84d590e..280d5a6f53a 100644 --- a/docs/toc_fa.yml +++ b/docs/toc_fa.yml @@ -168,6 +168,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': diff --git a/docs/toc_ja.yml b/docs/toc_ja.yml index f47bc065890..6661300a97e 100644 --- a/docs/toc_ja.yml +++ b/docs/toc_ja.yml @@ -140,7 +140,8 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' - - 'Dictionaries': + - 'generate': 'query_language/table_functions/generate.md' +- 'Dictionaries': - 'Introduction': 'query_language/dicts/index.md' - 'External Dictionaries': - 'General Description': 'query_language/dicts/external_dicts.md' diff --git a/docs/toc_ru.yml b/docs/toc_ru.yml index 5999ac74b56..06f196fd2f5 100644 --- a/docs/toc_ru.yml +++ b/docs/toc_ru.yml @@ -141,6 +141,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - 'Словари': - 'Введение': 'query_language/dicts/index.md' - 'Внешние словари': diff --git a/docs/toc_zh.yml b/docs/toc_zh.yml index e85c6b50f27..c7ec13a1943 100644 --- a/docs/toc_zh.yml +++ b/docs/toc_zh.yml @@ -167,6 +167,7 @@ nav: - 'odbc': 'query_language/table_functions/odbc.md' - 'hdfs': 'query_language/table_functions/hdfs.md' - 'input': 'query_language/table_functions/input.md' + - 'generate': 'query_language/table_functions/generate.md' - '字典': - '介绍': 'query_language/dicts/index.md' - '外部字典': From 41ffe2d9f9b14dca40a554b2d95c0d628fe6b29c Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 4 Feb 2020 16:29:08 +0300 Subject: [PATCH 0214/2007] style fixes --- .../src/TableFunctions/TableFunctionRandom.cpp | 18 ++++++++++++------ .../query_language/table_functions/generate.md | 1 + .../query_language/table_functions/generate.md | 2 +- .../query_language/table_functions/generate.md | 2 +- 4 files changed, 15 insertions(+), 8 deletions(-) create mode 120000 docs/fa/query_language/table_functions/generate.md diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index d70c8a73c63..fff1ed83539 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -224,12 +224,14 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { pcg32 generator(random_seed); offsets.resize(limit); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { offset += 1 + static_cast(generator()) % max_string_length; offsets[i] = offset - 1; } chars.resize(offset); - for (UInt64 i = 0; i < offset; ++i) { + for (UInt64 i = 0; i < offset; ++i) + { chars[i] = 32 + generator() % 95; } // add terminating zero char @@ -250,7 +252,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { pcg32 generator(random_seed); chars.resize(num_chars); - for (UInt64 i = 0; i < num_chars; ++i) { + for (UInt64 i = 0; i < num_chars; ++i) + { chars[i] = static_cast(generator()); } } @@ -327,7 +330,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, auto & data = typeid_cast &>(column).getData(); data.resize(limit); pcg64 generator(random_seed); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { auto x = UInt128(generator(), generator()); data[i] = x; } @@ -345,7 +349,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { pcg32 generator(random_seed); offsets.resize(limit); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { offset += static_cast(generator()) % max_array_length; offsets[i] = offset; } @@ -380,7 +385,8 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, pcg32 generator(random_seed); null_map.resize(limit); - for (UInt64 i = 0; i < limit; ++i) { + for (UInt64 i = 0; i < limit; ++i) + { null_map[i] = generator() < 1024; } break; diff --git a/docs/fa/query_language/table_functions/generate.md b/docs/fa/query_language/table_functions/generate.md new file mode 120000 index 00000000000..141c05da1e3 --- /dev/null +++ b/docs/fa/query_language/table_functions/generate.md @@ -0,0 +1 @@ +../../../en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/ja/query_language/table_functions/generate.md b/docs/ja/query_language/table_functions/generate.md index de0b0a41754..141c05da1e3 120000 --- a/docs/ja/query_language/table_functions/generate.md +++ b/docs/ja/query_language/table_functions/generate.md @@ -1 +1 @@ -en/query_language/table_functions/generate.md \ No newline at end of file +../../../en/query_language/table_functions/generate.md \ No newline at end of file diff --git a/docs/zh/query_language/table_functions/generate.md b/docs/zh/query_language/table_functions/generate.md index de0b0a41754..141c05da1e3 120000 --- a/docs/zh/query_language/table_functions/generate.md +++ b/docs/zh/query_language/table_functions/generate.md @@ -1 +1 @@ -en/query_language/table_functions/generate.md \ No newline at end of file +../../../en/query_language/table_functions/generate.md \ No newline at end of file From ceaff363b228532a9ad4d6e50dbd4bb79d7e575a Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 6 Feb 2020 15:34:11 +0300 Subject: [PATCH 0215/2007] recanonize linux build --- .../01072_random_table_function.reference | 73 ++++++++++++++----- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.reference b/dbms/tests/queries/0_stateless/01072_random_table_function.reference index 93ea1861756..3906b417524 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.reference +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.reference @@ -9,6 +9,7 @@ world hello hello hello +- Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) ['world','hello','world','hello','hello','hello','world','hello','hello'] ['hello','world','world','hello','world','world'] @@ -20,6 +21,7 @@ Array(Nullable(Enum8(\'hello\' = 1, \'world\' = 5))) ['hello','hello'] ['hello','hello'] ['hello','world','hello','hello','world','world','world','world'] +- Nullable(Enum16(\'o\' = -200, \'h\' = 1, \'w\' = 5)) o w @@ -31,6 +33,7 @@ h h w o +- UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 2254772619926532955 2254772619926532955 1234817989 1234817989 54213 -11323 197 -59 9120028858397505560 9120028858397505560 1171957426 1171957426 42674 -22862 178 -78 @@ -42,6 +45,7 @@ UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 7736560050027905187 7736560050027905187 1012211222 1012211222 7702 7702 22 22 2199287578947862030 2199287578947862030 2185722662 -2109244634 31526 31526 38 38 3019483913099890467 3019483913099890467 2647224658 -1647742638 29010 29010 82 82 +- Date DateTime DateTime(\'Europe/Moscow\') 2106-02-07 2009-02-16 23:59:49 2009-02-16 23:59:49 2086-11-02 2007-02-20 10:43:46 2007-02-20 10:43:46 @@ -53,6 +57,7 @@ Date DateTime DateTime(\'Europe/Moscow\') 1991-02-02 2002-01-28 12:47:02 2002-01-28 12:47:02 2056-04-25 2039-04-06 20:11:02 2039-04-06 20:11:02 2049-06-05 2053-11-20 07:10:58 2053-11-20 07:10:58 +- DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 2007-02-20 10:43:46.989 2007-02-20 10:43:46.817989 2007-02-20 10:43:46.817989 2002-10-04 02:54:48.647 2002-10-04 02:54:48.100647 2002-10-04 02:54:48.100647 @@ -64,6 +69,7 @@ DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 1979-01-20 20:39:20.939 1979-01-20 20:39:20.162939 1979-01-20 20:39:20.162939 2063-07-18 01:46:10.215 2063-07-18 01:46:10.908215 2063-07-18 01:46:10.908215 1996-11-02 14:35:41.110 1996-11-02 14:35:41.183110 1996-11-02 14:35:41.183110 +- Float32 Float64 9.783235e37 2.1973467205491123e307 9.285203e37 8.887754501811354e307 @@ -75,6 +81,7 @@ Float32 Float64 8.019563e37 7.539520705557441e307 1.7317079e38 2.143274805821858e307 2.0973474e38 2.9425818885529257e307 +- Decimal32(4) Decimal64(8) Decimal64(8) 123481.7989 22547726199.26532955 4159321346419233104838.6879832895010840 117195.7426 91200288583.97505560 8403779329565810688767.7049545291714611 @@ -86,17 +93,19 @@ Decimal32(4) Decimal64(8) Decimal64(8) 101221.1222 77365600500.27905187 -4693380431928321782727.0243506636623202 -210924.4634 21992875789.47862030 13765369952377767241248.9441272127848016 -164774.2638 30194839130.99890467 -13890064946313418575619.0315227826809939 +- UUID -1f4a8fc0-63ff-735b-7e90-d9ed3e183818 -3f39171b-1263-31fa-5046-2ea9fe2fd033 -9927a60f-01ac-f065-6da8-49def100c0cc -5d736910-493d-c3bf-6b5d-c8601d6440a3 -1e857066-961d-be0e-29e7-5c9efd534f23 -bda66d4f-737b-3622-b60f-aa27fe38ff30 -623d6d82-4422-2885-297f-7b2fec54178b -dcb0e0ca-3a43-5f2e-556e-7945df65729e -678f2360-36ac-d439-8d6d-f92295887e50 -9780b53e-dc0f-4a21-bdb3-9798af1913ad +7e90d9ed-3e18-3818-1f4a-8fc063ff735b +50462ea9-fe2f-d033-3f39-171b126331fa +6da849de-f100-c0cc-9927-a60f01acf065 +6b5dc860-1d64-40a3-5d73-6910493dc3bf +29e75c9e-fd53-4f23-1e85-7066961dbe0e +b60faa27-fe38-ff30-bda6-6d4f737b3622 +297f7b2f-ec54-178b-623d-6d8244222885 +556e7945-df65-729e-dcb0-e0ca3a435f2e +8d6df922-9588-7e50-678f-236036acd439 +bdb39798-af19-13ad-9780-b53edc0f4a21 +- Tuple(Int32, Int64) (1234817989,2254772619926532955) (1171957426,9120028858397505560) @@ -108,6 +117,7 @@ Tuple(Int32, Int64) (1012211222,7736560050027905187) (-2109244634,2199287578947862030) (-1647742638,3019483913099890467) +- Array(Int8) [-59,-78,-25,-72,-40,-84,95,22,38] [82,65,35,-110,-57,-69] @@ -119,6 +129,7 @@ Array(Int8) [-40,84] [-104,-86] [-36,123,44,60,5,25,-5,-127] +- Array(Nullable(Int32)) [1234817989,1171957426,275100647,1033685688,180895192,135557292,716914271,1012211222,-2109244634] [-1647742638,319510849,513356835,-1966518382,-786518841,269162939] @@ -130,17 +141,19 @@ Array(Nullable(Int32)) [-1399152424,-953863084] [733724312,-23652950] [371735004,462118779,148602156,-1055384004,-1041274619,247762201,522289659,822210177] +- Array(Nullable(UUID)) -['1f4a8fc0-63ff-735b-7e90-d9ed3e183818','3f39171b-1263-31fa-5046-2ea9fe2fd033','9927a60f-01ac-f065-6da8-49def100c0cc','5d736910-493d-c3bf-6b5d-c8601d6440a3','1e857066-961d-be0e-29e7-5c9efd534f23','bda66d4f-737b-3622-b60f-aa27fe38ff30','623d6d82-4422-2885-297f-7b2fec54178b','dcb0e0ca-3a43-5f2e-556e-7945df65729e','678f2360-36ac-d439-8d6d-f92295887e50'] -['9780b53e-dc0f-4a21-bdb3-9798af1913ad','c79810de-3635-d333-5ca1-7a81ab302b25','1c756bca-4438-3f17-a766-c8bcbe3ba400','d9072738-ac93-7ed6-167b-3c3c66d35a18','b1e8dec2-de29-3c9f-aaf2-f78fd92df3ce','9cd25f9f-3c0d-f43d-5a46-0194f0be04dd'] -['10a4718d-ab8c-49c6-c785-66ccf112f7d5','02ac2bf5-5634-a5a8-9a18-05ce8d1fb583','8037a13d-2004-08f2-f831-fa2387f5c29a','a99c4373-1121-2691-ecbb-216adbd748c7','ef0986ff-5031-0353-2f21-1de3ea53af08','778064a7-653b-ef7b-c77b-4d769b12b917','a1607e6f-691a-0ff0-b0b3-e454dae7bef7'] -['71c1b47a-c0eb-42b5-eecd-18dc585284fd','72bbf272-9ec5-09ec-f339-b5dac55c037b','26e5bce5-43f7-59b0-84c6-ef509f4c45eb','305fcbff-c366-2033-a8c5-d648f236e754','3a0d329f-f897-84e9-9e87-9501a713e63d','54bda20c-d5cd-a08a-c078-3c4fd81f4f55','43f549d1-3e5b-d5bf-ed32-b4850648bdc8','7eb6ac4f-06e0-ff48-6330-3c7afa5f2644'] -['17b9a4a5-fef8-a3f9-5af4-3b6e67ca62c9','3f524d8e-320d-00dc-c210-e199206550db'] -['005c592e-5081-9f3d-1fcb-5a9e82f39f97','29cf228d-b325-4a34-3eff-e80494a79260'] -['6c08b54b-8cf8-b96d-f087-8b54f5e72d0e'] -['7122e162-ab8b-a84a-6b71-c0846cf0204d','51c1de1a-24c7-18d6-39ed-e9023205610c'] -['f09d6779-1106-d667-e7c9-9a0cad544afe','62060fec-ee13-7c66-5da4-02c8f4d50dc9'] -['df1d0d54-d639-9c9b-2070-622fc9d82203','f23ef5b9-3797-9b0e-b8ac-67ea31b99c3e','e48afe73-9e22-7439-afed-d53b6ea204f4','d7f1ab47-4928-7623-283e-fb3f16aebeba','ea270407-d32f-a407-add2-3ae2d1113ccb','c43e9fff-2980-a1d1-f1bb-ff94d3cffbc2','a0cd54e6-0a2d-07ec-88ad-4f5d29c15b06','5e93413f-2eb9-5363-17ab-e2215b8b19e0'] +['7e90d9ed-3e18-3818-1f4a-8fc063ff735b','50462ea9-fe2f-d033-3f39-171b126331fa','6da849de-f100-c0cc-9927-a60f01acf065','6b5dc860-1d64-40a3-5d73-6910493dc3bf','29e75c9e-fd53-4f23-1e85-7066961dbe0e','b60faa27-fe38-ff30-bda6-6d4f737b3622','297f7b2f-ec54-178b-623d-6d8244222885','556e7945-df65-729e-dcb0-e0ca3a435f2e','8d6df922-9588-7e50-678f-236036acd439'] +['bdb39798-af19-13ad-9780-b53edc0f4a21','5ca17a81-ab30-2b25-c798-10de3635d333','a766c8bc-be3b-a400-1c75-6bca44383f17','167b3c3c-66d3-5a18-d907-2738ac937ed6','aaf2f78f-d92d-f3ce-b1e8-dec2de293c9f','5a460194-f0be-04dd-9cd2-5f9f3c0df43d'] +['c78566cc-f112-f7d5-10a4-718dab8c49c6','9a1805ce-8d1f-b583-02ac-2bf55634a5a8','f831fa23-87f5-c29a-8037-a13d200408f2','ecbb216a-dbd7-48c7-a99c-437311212691','2f211de3-ea53-af08-ef09-86ff50310353','c77b4d76-9b12-b917-7780-64a7653bef7b','b0b3e454-dae7-bef7-a160-7e6f691a0ff0'] +['eecd18dc-5852-84fd-71c1-b47ac0eb42b5','f339b5da-c55c-037b-72bb-f2729ec509ec','84c6ef50-9f4c-45eb-26e5-bce543f759b0','a8c5d648-f236-e754-305f-cbffc3662033','9e879501-a713-e63d-3a0d-329ff89784e9','c0783c4f-d81f-4f55-54bd-a20cd5cda08a','ed32b485-0648-bdc8-43f5-49d13e5bd5bf','63303c7a-fa5f-2644-7eb6-ac4f06e0ff48'] +['5af43b6e-67ca-62c9-17b9-a4a5fef8a3f9','c210e199-2065-50db-3f52-4d8e320d00dc'] +['1fcb5a9e-82f3-9f97-005c-592e50819f3d','3effe804-94a7-9260-29cf-228db3254a34'] +['f0878b54-f5e7-2d0e-6c08-b54b8cf8b96d'] +['6b71c084-6cf0-204d-7122-e162ab8ba84a','39ede902-3205-610c-51c1-de1a24c718d6'] +['e7c99a0c-ad54-4afe-f09d-67791106d667','5da402c8-f4d5-0dc9-6206-0fecee137c66'] +['2070622f-c9d8-2203-df1d-0d54d6399c9b','b8ac67ea-31b9-9c3e-f23e-f5b937979b0e','afedd53b-6ea2-04f4-e48a-fe739e227439','283efb3f-16ae-beba-d7f1-ab4749287623','add23ae2-d111-3ccb-ea27-0407d32fa407','f1bbff94-d3cf-fbc2-c43e-9fff2980a1d1','88ad4f5d-29c1-5b06-a0cd-54e60a2d07ec','17abe221-5b8b-19e0-5e93-413f2eb95363'] +- Tuple(Int32, Array(Int64)) (1234817989,[2254772619926532955,9120028858397505560,4555697903102013946,5784362079052877875,-7410772078432030619,7901646768096461004,6733841386518201279,7736560050027905187,2199287578947862030]) (1171957426,[3019483913099890467,-4781013766399904222,-5327852745410412752,7078934595552553093,2990244123355912075,-2544286630298820818]) @@ -152,6 +165,7 @@ Tuple(Int32, Array(Int64)) (1012211222,[-562393447932771686,-6225026423445182831]) (-2109244634,[-1388479317275096889,-1222297392734207149]) (-1647742638,[3396028458740199176,8610993157653131131,-4072576266223306473,-6818310818869145616,-5713972449102020873,8197031236106666677,-1239306987803343619,8267468115072584172]) +- FixedString(4) Ų ج_ @@ -163,6 +177,7 @@ w W  T +- String String String @@ -183,6 +198,7 @@ String \0g \0> \0XjbW:s< +- Nullable(String) )/VC)%f9 \0ih|;B @@ -194,6 +210,7 @@ Nullable(String) \0g \0> \0XjbW:s< +- Array(String) ['(|ZVAg2F','\0GXjbW','\0<^guT(','\0y M$lZ0','\03','\0p','\0','\0i','\0P'] ['\0"}YRG%B','\0T3(E^> p','\0JTaj','\0)*3','\0k%=p','\0Yub$81`X'] @@ -205,3 +222,19 @@ Array(String) ['\0k','\0 '] ['\0F','\0&h -210924.4634 w 2.143274805821858e307 ('2056-04-25','2039-04-06 20:11:02','2063-07-18 01:46:10.215','8d6df922-9588-7e50-678f-236036acd439') w +[-36,123,44,60,5,25,-5,-127] 2647224658 \0XjbW:s< -164774.2638 o 2.9425818885529257e307 ('2049-06-05','2053-11-20 07:10:58','1996-11-02 14:35:41.110','bdb39798-af19-13ad-9780-b53edc0f4a21') \r +- From 75920565c98452844ee299ee8e2e327e0be0198e Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 31 Jan 2020 18:45:58 +0300 Subject: [PATCH 0216/2007] Improve flush performance in SystemLog It started having problems under heavy workloads such as saving all memory allocations to trace_log table. To improve that, stop popping entries one by one from the log queue, and actually stop using the queue completely, because we don't use most of its methods or its backpressure feature. Just keep a vector of messages and flush it to disk periodically. --- dbms/src/Interpreters/SystemLog.h | 225 ++++++++++++++---------------- dbms/src/Interpreters/TextLog.cpp | 11 ++ dbms/src/Interpreters/TextLog.h | 8 +- 3 files changed, 124 insertions(+), 120 deletions(-) diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 301bceba525..8dc1f911da2 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -6,7 +6,6 @@ #include #include #include -#include #include #include #include @@ -53,6 +52,10 @@ namespace DB }; */ +namespace ErrorCodes +{ + extern const int TIMEOUT_EXCEEDED; +} #define DBMS_SYSTEM_LOG_QUEUE_SIZE 1048576 @@ -119,55 +122,45 @@ public: void shutdown(); protected: + Logger * log; + +private: + /* Saving thread data */ Context & context; const String database_name; const String table_name; const String storage_def; StoragePtr table; + bool is_prepared = false; const size_t flush_interval_milliseconds; - std::atomic is_shutdown{false}; - - enum class EntryType - { - LOG_ELEMENT = 0, - AUTO_FLUSH, - FORCE_FLUSH, - SHUTDOWN, - }; - - using QueueItem = std::pair; - - /// Queue is bounded. But its size is quite large to not block in all normal cases. - ConcurrentBoundedQueue queue {DBMS_SYSTEM_LOG_QUEUE_SIZE}; - - /** Data that was pulled from queue. Data is accumulated here before enough time passed. - * It's possible to implement double-buffering, but we assume that insertion into table is faster - * than accumulation of large amount of log records (for example, for query log - processing of large amount of queries). - */ - std::vector data; - - Logger * log; - - /** In this thread, data is pulled from 'queue' and stored in 'data', and then written into table. - */ ThreadFromGlobalPool saving_thread; - void threadFunction(); + /* Data shared between callers of add()/flush()/shutdown(), and the saving thread */ + std::mutex mutex; + /// Queue is bounded. But its size is quite large to not block in all normal cases. + std::vector queue; + // An always-incrementing index of the first message currently in the queue. + // We use it to give a global sequential index to every message, so that we can wait + // until a particular message is flushed. This is used to implement synchronous log + // flushing for SYSTEM FLUSH LOGS. + uint64_t queue_front_index = 0; + bool is_shutdown = false; + std::condition_variable flush_event; + // Requested to flush logs up to this index, exclusive + uint64_t requested_flush_before = 0; + // Flushed log up to this index, exclusive + uint64_t flushed_before = 0; + + void savingThreadFunction(); /** Creates new table if it does not exist. * Renames old table if its structure is not suitable. * This cannot be done in constructor to avoid deadlock while renaming a table under locked Context when SystemLog object is created. */ - bool is_prepared = false; void prepareTable(); - std::mutex flush_mutex; - std::mutex condvar_mutex; - std::condition_variable flush_condvar; - bool force_flushing = false; - /// flushImpl can be executed only in saving_thread. - void flushImpl(EntryType reason); + void flushImpl(const std::vector & to_flush, uint64_t to_flush_end); }; @@ -183,51 +176,86 @@ SystemLog::SystemLog(Context & context_, { log = &Logger::get("SystemLog (" + database_name + "." + table_name + ")"); - data.reserve(DBMS_SYSTEM_LOG_QUEUE_SIZE); - saving_thread = ThreadFromGlobalPool([this] { threadFunction(); }); + saving_thread = ThreadFromGlobalPool([this] { savingThreadFunction(); }); } template void SystemLog::add(const LogElement & element) { + std::unique_lock lock(mutex); + if (is_shutdown) return; - /// Without try we could block here in case of queue overflow. - if (!queue.tryPush({EntryType::LOG_ELEMENT, element})) - LOG_ERROR(log, "SystemLog queue is full"); + if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE / 2) + { + // The queue more than half full, time to flush. + const uint64_t queue_end = queue_front_index + queue.size(); + if (requested_flush_before < queue_end) + { + requested_flush_before = queue_end; + } + flush_event.notify_all(); + } + + if (queue.size() >= DBMS_SYSTEM_LOG_QUEUE_SIZE) + { + // TextLog sets its logger level to 0, so this log is a noop and there + // is no recursive logging. + LOG_ERROR(log, "Queue is full for system log '" + demangle(typeid(*this).name()) + "'."); + return; + } + + queue.push_back(element); } template void SystemLog::flush() { + std::unique_lock lock(mutex); + if (is_shutdown) return; - std::lock_guard flush_lock(flush_mutex); - force_flushing = true; + const uint64_t queue_end = queue_front_index + queue.size(); - /// Tell thread to execute extra flush. - queue.push({EntryType::FORCE_FLUSH, {}}); + if (requested_flush_before < queue_end) + { + requested_flush_before = queue_end; + } + flush_event.notify_all(); - /// Wait for flush being finished. - std::unique_lock lock(condvar_mutex); - while (force_flushing) - flush_condvar.wait(lock); + const int timeout_seconds = 60; + bool result = flush_event.wait_for(lock, std::chrono::seconds(timeout_seconds), + [&] { return flushed_before >= queue_end; }); + + if (!result) + { + throw Exception("Timeout exceeded (" + toString(timeout_seconds) + " s) while flushing system log '" + demangle(typeid(*this).name()) + "'.", + ErrorCodes::TIMEOUT_EXCEEDED); + } } template void SystemLog::shutdown() { - bool old_val = false; - if (!is_shutdown.compare_exchange_strong(old_val, true)) - return; + { + std::unique_lock lock(mutex); + + if (is_shutdown) + { + return; + } + + is_shutdown = true; + + /// Tell thread to shutdown. + flush_event.notify_all(); + } - /// Tell thread to shutdown. - queue.push({EntryType::SHUTDOWN, {}}); saving_thread.join(); } @@ -240,71 +268,41 @@ SystemLog::~SystemLog() template -void SystemLog::threadFunction() +void SystemLog::savingThreadFunction() { setThreadName("SystemLogFlush"); - Stopwatch time_after_last_write; - bool first = true; - - while (true) + bool exit_this_thread = false; + while (!exit_this_thread) { try { - if (first) + std::vector to_flush; + // The end index (exclusive, like std end()) of the messages we are + // going to flush. + uint64_t to_flush_end = 0; + { - time_after_last_write.restart(); - first = false; + std::unique_lock lock(mutex); + flush_event.wait_for(lock, std::chrono::milliseconds(flush_interval_milliseconds), + [&] () { return requested_flush_before > flushed_before || is_shutdown; }); + + queue_front_index += queue.size(); + to_flush_end = queue_front_index; + queue.swap(to_flush); + + exit_this_thread = is_shutdown; } - QueueItem element; - bool has_element = false; - - /// data.size() is increased only in this function - /// TODO: get rid of data and queue duality - - if (data.empty()) + if (to_flush.empty()) { - queue.pop(element); - has_element = true; - } - else - { - size_t milliseconds_elapsed = time_after_last_write.elapsed() / 1000000; - if (milliseconds_elapsed < flush_interval_milliseconds) - has_element = queue.tryPop(element, flush_interval_milliseconds - milliseconds_elapsed); + continue; } - if (has_element) - { - if (element.first == EntryType::SHUTDOWN) - { - /// NOTE: MergeTree engine can write data even it is already in shutdown state. - flushImpl(element.first); - break; - } - else if (element.first == EntryType::FORCE_FLUSH) - { - flushImpl(element.first); - time_after_last_write.restart(); - continue; - } - else - data.push_back(element.second); - } - - size_t milliseconds_elapsed = time_after_last_write.elapsed() / 1000000; - if (milliseconds_elapsed >= flush_interval_milliseconds) - { - /// Write data to a table. - flushImpl(EntryType::AUTO_FLUSH); - time_after_last_write.restart(); - } + flushImpl(to_flush, to_flush_end); } catch (...) { - /// In case of exception we lost accumulated data - to avoid locking. - data.clear(); tryLogCurrentException(__PRETTY_FUNCTION__); } } @@ -312,13 +310,10 @@ void SystemLog::threadFunction() template -void SystemLog::flushImpl(EntryType reason) +void SystemLog::flushImpl(const std::vector & to_flush, uint64_t to_flush_end) { try { - if ((reason == EntryType::AUTO_FLUSH || reason == EntryType::SHUTDOWN) && data.empty()) - return; - LOG_TRACE(log, "Flushing system log"); /// We check for existence of the table and create it as needed at every flush. @@ -327,13 +322,9 @@ void SystemLog::flushImpl(EntryType reason) prepareTable(); Block block = LogElement::createBlock(); - for (const LogElement & elem : data) + for (const auto & elem : to_flush) elem.appendToBlock(block); - /// Clear queue early, because insertion to the table could lead to generation of more log entrites - /// and pushing them to already full queue will lead to deadlock. - data.clear(); - /// We write to table indirectly, using InterpreterInsertQuery. /// This is needed to support DEFAULT-columns in table. @@ -352,15 +343,11 @@ void SystemLog::flushImpl(EntryType reason) catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); - /// In case of exception, also clean accumulated data - to avoid locking. - data.clear(); - } - if (reason == EntryType::FORCE_FLUSH) - { - std::lock_guard lock(condvar_mutex); - force_flushing = false; - flush_condvar.notify_one(); } + + std::unique_lock lock(mutex); + flushed_before = to_flush_end; + flush_event.notify_all(); } diff --git a/dbms/src/Interpreters/TextLog.cpp b/dbms/src/Interpreters/TextLog.cpp index c09877c2df5..b5f1d987b91 100644 --- a/dbms/src/Interpreters/TextLog.cpp +++ b/dbms/src/Interpreters/TextLog.cpp @@ -70,4 +70,15 @@ void TextLogElement::appendToBlock(Block & block) const columns[i++]->insert(source_line); } +TextLog::TextLog(Context & context_, const String & database_name_, + const String & table_name_, const String & storage_def_, + size_t flush_interval_milliseconds_) + : SystemLog(context_, database_name_, table_name_, + storage_def_, flush_interval_milliseconds_) +{ + // SystemLog methods may write text logs, so we disable logging for the text + // log table to avoid recursion. + log->setLevel(0); +} + } diff --git a/dbms/src/Interpreters/TextLog.h b/dbms/src/Interpreters/TextLog.h index fff45f14328..73c38429662 100644 --- a/dbms/src/Interpreters/TextLog.h +++ b/dbms/src/Interpreters/TextLog.h @@ -30,7 +30,13 @@ struct TextLogElement class TextLog : public SystemLog { - using SystemLog::SystemLog; +public: + TextLog( + Context & context_, + const String & database_name_, + const String & table_name_, + const String & storage_def_, + size_t flush_interval_milliseconds_); }; } From b125e6b393f3ca00894795ecffaf22c3bd57a785 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 6 Feb 2020 18:07:08 +0300 Subject: [PATCH 0217/2007] Log which configuration files we loaded. --- dbms/src/Common/Config/ConfigProcessor.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/dbms/src/Common/Config/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp index acf58abfbb4..e8e040360a4 100644 --- a/dbms/src/Common/Config/ConfigProcessor.cpp +++ b/dbms/src/Common/Config/ConfigProcessor.cpp @@ -427,8 +427,8 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & // Skip non-config and temporary files if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, ".")) files.push_back(file.path()); + } } - } std::sort(files.begin(), files.end()); @@ -440,6 +440,8 @@ XMLDocumentPtr ConfigProcessor::processConfig( zkutil::ZooKeeperNodeCache * zk_node_cache, const zkutil::EventPtr & zk_changed_event) { + LOG_DEBUG(log, "Processing configuration file '" + path + "'."); + XMLDocumentPtr config = dom_parser.parse(path); std::vector contributing_files; @@ -449,6 +451,8 @@ XMLDocumentPtr ConfigProcessor::processConfig( { try { + LOG_DEBUG(log, "Merging configuration file '" + merge_file + "'."); + XMLDocumentPtr with = dom_parser.parse(merge_file); merge(config, with); contributing_files.push_back(merge_file); @@ -484,6 +488,8 @@ XMLDocumentPtr ConfigProcessor::processConfig( } if (!include_from_path.empty()) { + LOG_DEBUG(log, "Including configuration file '" + include_from_path + "'."); + contributing_files.push_back(include_from_path); include_from = dom_parser.parse(include_from_path); } @@ -613,6 +619,7 @@ void ConfigProcessor::savePreprocessedConfig(const LoadedConfig & loaded_config, Poco::File(preprocessed_path_parent).createDirectories(); } DOMWriter().writeNode(preprocessed_path, loaded_config.preprocessed_xml); + LOG_DEBUG(log, "Saved preprocessed configuration to '" << preprocessed_path << "'."); } catch (Poco::Exception & e) { From ae0044efe87b2f43d985324455efb69d62605011 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Fri, 7 Feb 2020 22:37:24 +0300 Subject: [PATCH 0218/2007] Update ConfigProcessor.cpp --- dbms/src/Common/Config/ConfigProcessor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Common/Config/ConfigProcessor.cpp b/dbms/src/Common/Config/ConfigProcessor.cpp index e8e040360a4..ce96bb70ddc 100644 --- a/dbms/src/Common/Config/ConfigProcessor.cpp +++ b/dbms/src/Common/Config/ConfigProcessor.cpp @@ -427,8 +427,8 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & // Skip non-config and temporary files if (file.isFile() && (extension == "xml" || extension == "conf") && !startsWith(base_name, ".")) files.push_back(file.path()); - } } + } std::sort(files.begin(), files.end()); From bddbc956ec3c40b4a076c969c78ebd9a93bb4029 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 7 Feb 2020 23:59:51 +0300 Subject: [PATCH 0219/2007] Add symlynks for lld, because gcc is not able to find it without 'ld.' prefix. --- docker/packager/binary/Dockerfile | 3 +++ docker/packager/deb/Dockerfile | 3 +++ 2 files changed, 6 insertions(+) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 9018afb1153..05e2f2c3b97 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -57,6 +57,9 @@ RUN apt-get update -y \ rename \ wget +RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld +RUN ln -s /usr/bin/lld-8 /usr/bin/ldd + ENV CC=clang-8 ENV CXX=clang++-8 diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index 986768b7a95..dc150e9ba14 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -88,6 +88,9 @@ RUN curl -O https://clickhouse-builds.s3.yandex.net/utils/dpkg-deb RUN chmod +x dpkg-deb RUN cp dpkg-deb /usr/bin +RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld +RUN ln -s /usr/bin/lld-8 /usr/bin/ldd + COPY build.sh / CMD ["/bin/bash", "/build.sh"] From 71d46bb5219e620126347643708990dda8a25f0e Mon Sep 17 00:00:00 2001 From: liyang Date: Sat, 8 Feb 2020 14:31:03 +0800 Subject: [PATCH 0220/2007] modify to suggestions --- dbms/src/Functions/array/arrayScalarProduct.h | 5 +-- dbms/src/Functions/array/array_auc.cpp | 43 ++++++++++++++++--- .../functions/array_functions.md | 28 ++++++++++-- 3 files changed, 63 insertions(+), 13 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index c450f41f48a..87f618b5b4f 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -109,11 +109,10 @@ public: { const DataTypeArray * array_type = checkAndGetDataType(arguments[i].get()); if (!array_type) - throw Exception("All argument for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception("All arguments for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto & nested_type = array_type->getNestedType(); - WhichDataType which(nested_type); - bool is_number = which.isNativeInt() || which.isNativeUInt() || which.isFloat(); + bool is_number = isNativeNumber(nested_type) if (!is_number) { throw Exception( diff --git a/dbms/src/Functions/array/array_auc.cpp b/dbms/src/Functions/array/array_auc.cpp index 41739446e46..861b617e3ea 100644 --- a/dbms/src/Functions/array/array_auc.cpp +++ b/dbms/src/Functions/array/array_auc.cpp @@ -18,16 +18,35 @@ public: struct ScoreLabel { ResultType score; - UInt8 label; + bool label; }; - static DataTypePtr getReturnType(const DataTypePtr & /* nested_type1 */, const DataTypePtr & nested_type2) + static DataTypePtr getReturnType(const DataTypePtr & /* score_type */, const DataTypePtr & label_type) { - WhichDataType which2(nested_type2); - if (!which2.isUInt8()) + WhichDataType which(label_type); + // Labels values are either {0, 1} or {-1, 1}, and its type must be one of (Enum8, UInt8, Int8) + if (!which.isUInt8() && !which.isEnum8() && !which.isInt8()) { throw Exception(std::string(NameArrayAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } + + if (which.isEnum8()) + { + auto type8 = checkAndGetDataType(label_type.get()); + if (type8) + throw Exception(std::string(NameArrayAUC::name) + "lable type not valid Enum8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + std::set valSet; + const auto & values = type8->getValues(); + for (const auto & value : values) + { + valSet.insert(value.second); + } + + if (valSet != {0, 1} || valSet != {-1, 1}) + throw Exception( + std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } return std::make_shared>(); } @@ -45,25 +64,35 @@ public: if (score_len == 0) return {}; - // Order pairs of score and lable by score ascending + // Calculate positive and negative label number and restore scores and labels in vector size_t num_pos = 0; size_t num_neg = 0; + std::set labelValSet; std::vector pairs(score_len); for (size_t i = 0; i < score_len; ++i) { pairs[i].score = scores[i + score_offset]; - pairs[i].label = (labels[i + label_offset] ? 1 : 0); + pairs[i].label = (labels[i + label_offset] == 1); if (pairs[i].label) ++num_pos; else ++num_neg; + + labelValSet.insert(labels[i + label_offset]); } + + // Label values must be {0, 1} or {-1, 1} + if (labelValSet != {0, 1} && labelValSet != {-1, 1}) + throw Exception( + std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + // Order pairs of score and lable by score ascending std::sort(pairs.begin(), pairs.end(), [](const auto & lhs, const auto & rhs) { return lhs.score < rhs.score; }); // Calculate AUC size_t curr_cnt = 0; size_t curr_pos_cnt = 0; - size_t curr_sum = 0; + Int64 curr_sum = 0; ResultType last_score = -1; ResultType rank_sum = 0; for (size_t i = 0; i < pairs.size(); ++i) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 35aa9c98dfb..3574f37a509 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -896,11 +896,33 @@ Result: │ [('a','d'),('b','e'),('c','f')] │ └────────────────────────────────────────────┘ ``` -## arrayAUC(arr_scores, arr_labels) +## arrayAUC {#arrayauc} +Calculate AUC(Area Under the Curve, which is a concept in machine learning, see more details: https://en.wikipedia.org/wiki/Receiver_operating_characteristic#Area_under_the_curve). -Returns AUC (Area Under the Curve, which is a concept in machine learning, see more details: https://developers.google.com/machine-learning/crash-course/classification/roc-and-auc). +**Syntax** +```sql +arrayAUC(arr_scores, arr_labels) +``` -`arr_scores` represents scores prediction model gives, while `arr_labels` represents labels of samples, usually 1 for positive sample and 0 for negtive sample. +**Parameters** +- `arr_scores` — scores prediction model gives. +- `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negtive sample. +**Returned value** +return AUC value with type Float64. + +**Example** +Query: +```sql +select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +``` + +Result: + +```text +┌─arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1])─┐ +│ 0.75 │ +└────────────────────────────────────────-----──┘ +``` [Original article](https://clickhouse.yandex/docs/en/query_language/functions/array_functions/) From 9d35b8b6ea45ffd54fb255f9c13f0a9064f588aa Mon Sep 17 00:00:00 2001 From: Andrew Onyshchuk Date: Sat, 8 Feb 2020 19:23:09 -0600 Subject: [PATCH 0221/2007] Use correct implementation for reinterpretAsFixedString --- dbms/src/Functions/FunctionsReinterpret.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Functions/FunctionsReinterpret.h b/dbms/src/Functions/FunctionsReinterpret.h index b0eebdee88e..59cdf7bc93d 100644 --- a/dbms/src/Functions/FunctionsReinterpret.h +++ b/dbms/src/Functions/FunctionsReinterpret.h @@ -261,7 +261,7 @@ using FunctionReinterpretAsDate = FunctionReinterpretStringAs; using FunctionReinterpretAsString = FunctionReinterpretAsStringImpl; -using FunctionReinterpretAsFixedString = FunctionReinterpretAsStringImpl; +using FunctionReinterpretAsFixedString = FunctionReinterpretAsFixedStringImpl; } From f0f0768e81504e9af675337a0d3d39d1b0b7351f Mon Sep 17 00:00:00 2001 From: liyang Date: Sun, 9 Feb 2020 10:45:58 +0800 Subject: [PATCH 0222/2007] fix bugs --- dbms/src/Functions/array/arrayScalarProduct.h | 6 ++--- dbms/src/Functions/array/array_auc.cpp | 26 ++++++++++--------- .../0_stateless/01064_array_auc.reference | 15 +++++++++++ .../queries/0_stateless/01064_array_auc.sql | 17 +++++++++++- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/dbms/src/Functions/array/arrayScalarProduct.h b/dbms/src/Functions/array/arrayScalarProduct.h index 87f618b5b4f..1352038195b 100644 --- a/dbms/src/Functions/array/arrayScalarProduct.h +++ b/dbms/src/Functions/array/arrayScalarProduct.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -112,12 +113,9 @@ public: throw Exception("All arguments for function " + getName() + " must be an array.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); auto & nested_type = array_type->getNestedType(); - bool is_number = isNativeNumber(nested_type) - if (!is_number) - { + if (!isNativeNumber(nested_type) && !isEnum(nested_type)) throw Exception( getName() + " cannot process values of type " + nested_type->getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } nested_types[i] = nested_type; } diff --git a/dbms/src/Functions/array/array_auc.cpp b/dbms/src/Functions/array/array_auc.cpp index 861b617e3ea..3ed9cd275d3 100644 --- a/dbms/src/Functions/array/array_auc.cpp +++ b/dbms/src/Functions/array/array_auc.cpp @@ -14,6 +14,10 @@ class ArrayAUCImpl { public: using ResultType = Float64; + using LabelValueSet = std::set; + using LabelValueSets = std::vector; + + inline static const LabelValueSets expect_label_value_sets = {{0, 1}, {-1, 1}}; struct ScoreLabel { @@ -27,23 +31,22 @@ public: // Labels values are either {0, 1} or {-1, 1}, and its type must be one of (Enum8, UInt8, Int8) if (!which.isUInt8() && !which.isEnum8() && !which.isInt8()) { - throw Exception(std::string(NameArrayAUC::name) + "lable type must be UInt8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + throw Exception( + std::string(NameArrayAUC::name) + "lable type must be UInt8, Enum8 or Int8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } if (which.isEnum8()) { auto type8 = checkAndGetDataType(label_type.get()); - if (type8) + if (!type8) throw Exception(std::string(NameArrayAUC::name) + "lable type not valid Enum8", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - std::set valSet; + LabelValueSet value_set; const auto & values = type8->getValues(); for (const auto & value : values) - { - valSet.insert(value.second); - } + value_set.insert(value.second); - if (valSet != {0, 1} || valSet != {-1, 1}) + if (std::find(expect_label_value_sets.begin(), expect_label_value_sets.end(), value_set) == expect_label_value_sets.end()) throw Exception( std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); } @@ -67,7 +70,7 @@ public: // Calculate positive and negative label number and restore scores and labels in vector size_t num_pos = 0; size_t num_neg = 0; - std::set labelValSet; + LabelValueSet label_value_set; std::vector pairs(score_len); for (size_t i = 0; i < score_len; ++i) { @@ -78,11 +81,11 @@ public: else ++num_neg; - labelValSet.insert(labels[i + label_offset]); + label_value_set.insert(labels[i + label_offset]); } // Label values must be {0, 1} or {-1, 1} - if (labelValSet != {0, 1} && labelValSet != {-1, 1}) + if (std::find(expect_label_value_sets.begin(), expect_label_value_sets.end(), label_value_set) == expect_label_value_sets.end()) throw Exception( std::string(NameArrayAUC::name) + "lable values must be {0, 1} or {-1, 1}", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -119,6 +122,7 @@ public: } }; + /// auc(array_score, array_label) - Calculate AUC with array of score and label using FunctionArrayAUC = FunctionArrayScalarProduct; @@ -126,6 +130,4 @@ void registerFunctionArrayAUC(FunctionFactory & factory) { factory.registerFunction(); } - - } diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.reference b/dbms/tests/queries/0_stateless/01064_array_auc.reference index 39c64a9f26e..8c17bba359a 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.reference +++ b/dbms/tests/queries/0_stateless/01064_array_auc.reference @@ -1 +1,16 @@ 0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.75 +0.25 +0.25 +0.25 +0.25 +0.25 +0.125 +0.25 diff --git a/dbms/tests/queries/0_stateless/01064_array_auc.sql b/dbms/tests/queries/0_stateless/01064_array_auc.sql index ca270937f63..de05c47c51b 100644 --- a/dbms/tests/queries/0_stateless/01064_array_auc.sql +++ b/dbms/tests/queries/0_stateless/01064_array_auc.sql @@ -1 +1,16 @@ -select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast([0, 0, 1, 1] as Array(Int8))); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast([-1, -1, 1, 1] as Array(Int8))); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = 0, 'true' = 1)))); +select arrayAUC([0.1, 0.4, 0.35, 0.8], cast(['false', 'false', 'true', 'true'] as Array(Enum8('false' = -1, 'true' = 1)))); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt8)), [0, 0, 1, 1]); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt16)), [0, 0, 1, 1]); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt32)), [0, 0, 1, 1]); +select arrayAUC(cast([10, 40, 35, 80] as Array(UInt64)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int8)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int16)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int32)), [0, 0, 1, 1]); +select arrayAUC(cast([-10, -40, -35, -80] as Array(Int64)), [0, 0, 1, 1]); +select arrayAUC(cast([-0.1, -0.4, -0.35, -0.8] as Array(Float32)) , [0, 0, 1, 1]); +select arrayAUC([0, 3, 5, 6, 7.5, 8], [1, 0, 1, 0, 0, 0]); +select arrayAUC([0.1, 0.35, 0.4, 0.8], [1, 0, 1, 0]); \ No newline at end of file From 2b683f6fc97ef266f2bc7d94dd1ef3569ca45b33 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Sun, 9 Feb 2020 17:15:29 +0300 Subject: [PATCH 0223/2007] boop --- dbms/src/Interpreters/SystemLog.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/SystemLog.h b/dbms/src/Interpreters/SystemLog.h index 8dc1f911da2..e80e8bb4878 100644 --- a/dbms/src/Interpreters/SystemLog.h +++ b/dbms/src/Interpreters/SystemLog.h @@ -137,7 +137,7 @@ private: /* Data shared between callers of add()/flush()/shutdown(), and the saving thread */ std::mutex mutex; - /// Queue is bounded. But its size is quite large to not block in all normal cases. + // Queue is bounded. But its size is quite large to not block in all normal cases. std::vector queue; // An always-incrementing index of the first message currently in the queue. // We use it to give a global sequential index to every message, so that we can wait From fd87a5bb2d72ce1bf6df1ae7d7f760e936199954 Mon Sep 17 00:00:00 2001 From: liyang Date: Mon, 10 Feb 2020 09:45:30 +0800 Subject: [PATCH 0224/2007] fix error --- docs/en/query_language/functions/array_functions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/query_language/functions/array_functions.md b/docs/en/query_language/functions/array_functions.md index 3574f37a509..101ba8a8b7f 100644 --- a/docs/en/query_language/functions/array_functions.md +++ b/docs/en/query_language/functions/array_functions.md @@ -914,7 +914,7 @@ return AUC value with type Float64. **Example** Query: ```sql -select auc([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) +select arrayAUC([0.1, 0.4, 0.35, 0.8], [0, 0, 1, 1]) ``` Result: From 954ea60a6aea854dbbe81e22a8f49996fd01bfb0 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 10 Feb 2020 12:05:14 +0300 Subject: [PATCH 0225/2007] Remove redundant compilers, linkers and libs --- docker/packager/binary/Dockerfile | 15 --------------- docker/packager/deb/Dockerfile | 15 --------------- 2 files changed, 30 deletions(-) diff --git a/docker/packager/binary/Dockerfile b/docker/packager/binary/Dockerfile index 05e2f2c3b97..6d30578f269 100644 --- a/docker/packager/binary/Dockerfile +++ b/docker/packager/binary/Dockerfile @@ -13,7 +13,6 @@ RUN apt-get --allow-unauthenticated update -y \ apt-transport-https \ ca-certificates -RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" >> /etc/apt/sources.list RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list RUN add-apt-repository ppa:ubuntu-toolchain-r/test @@ -24,23 +23,10 @@ RUN apt-get update -y \ cmake \ ccache \ curl \ - gcc-7 \ - g++-7 \ gcc-8 \ g++-8 \ gcc-9 \ g++-9 \ - clang-6.0 \ - lld-6.0 \ - libclang-6.0-dev \ - liblld-6.0-dev \ - llvm-6.0 \ - libllvm6.0 \ - llvm-6.0-dev \ - clang-7 \ - lld-7 \ - libclang-7-dev \ - liblld-7-dev \ clang-8 \ lld-8 \ libclang-8-dev \ @@ -58,7 +44,6 @@ RUN apt-get update -y \ wget RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld -RUN ln -s /usr/bin/lld-8 /usr/bin/ldd ENV CC=clang-8 ENV CXX=clang++-8 diff --git a/docker/packager/deb/Dockerfile b/docker/packager/deb/Dockerfile index dc150e9ba14..b937d9818ae 100644 --- a/docker/packager/deb/Dockerfile +++ b/docker/packager/deb/Dockerfile @@ -14,30 +14,16 @@ RUN apt-get --allow-unauthenticated update -y \ ca-certificates -RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-7 main" >> /etc/apt/sources.list RUN echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" >> /etc/apt/sources.list RUN add-apt-repository ppa:ubuntu-toolchain-r/test RUN apt-get --allow-unauthenticated update -y \ && env DEBIAN_FRONTEND=noninteractive \ apt-get --allow-unauthenticated install --yes --no-install-recommends \ - gcc-7 \ - g++-7 \ gcc-8 \ g++-8 \ gcc-9 \ g++-9 \ - clang-6.0 \ - lld-6.0 \ - libclang-6.0-dev \ - liblld-6.0-dev \ - llvm-6.0 \ - libllvm6.0 \ - llvm-6.0-dev \ - clang-7 \ - lld-7 \ - libclang-7-dev \ - liblld-7-dev \ clang-8 \ lld-8 \ libclang-8-dev \ @@ -89,7 +75,6 @@ RUN chmod +x dpkg-deb RUN cp dpkg-deb /usr/bin RUN ln -s /usr/bin/lld-8 /usr/bin/ld.lld -RUN ln -s /usr/bin/lld-8 /usr/bin/ldd COPY build.sh / From 1c41d73d9b232d674e1a64524d154a03ccc55099 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Mon, 10 Feb 2020 12:27:57 +0300 Subject: [PATCH 0226/2007] small refactoring + comments adding --- dbms/programs/copier/Internals.h | 8 ++++++++ dbms/src/Parsers/ASTLiteral.h | 4 ++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 dbms/programs/copier/Internals.h diff --git a/dbms/programs/copier/Internals.h b/dbms/programs/copier/Internals.h new file mode 100644 index 00000000000..a25ae7b973c --- /dev/null +++ b/dbms/programs/copier/Internals.h @@ -0,0 +1,8 @@ +// +// Created by jakalletti on 2/7/20. +// + +#ifndef CLICKHOUSE_INTERNALS_H +#define CLICKHOUSE_INTERNALS_H + +#endif //CLICKHOUSE_INTERNALS_H diff --git a/dbms/src/Parsers/ASTLiteral.h b/dbms/src/Parsers/ASTLiteral.h index 552f5da04a2..1d307a4101e 100644 --- a/dbms/src/Parsers/ASTLiteral.h +++ b/dbms/src/Parsers/ASTLiteral.h @@ -21,8 +21,8 @@ public: std::optional begin; std::optional end; - ASTLiteral(Field && value_) : value(value_) {} - ASTLiteral(const Field & value_) : value(value_) {} + explicit ASTLiteral(Field && value_) : value(value_) {} + explicit ASTLiteral(const Field & value_) : value(value_) {} /** Get the text that identifies this element. */ String getID(char delim) const override { return "Literal" + (delim + applyVisitor(FieldVisitorDump(), value)); } From d59a9a7dedcd83d69b57e5fc524f3d87c751d8a5 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 10 Feb 2020 15:54:03 +0300 Subject: [PATCH 0227/2007] less flappy --- dbms/tests/performance/general_purpose_hashes_on_UUID.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/performance/general_purpose_hashes_on_UUID.xml b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml index c7fb0a3676b..9dbac4e33cc 100644 --- a/dbms/tests/performance/general_purpose_hashes_on_UUID.xml +++ b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml @@ -41,8 +41,8 @@ table - numbers(1000000) - numbers_mt(10000000) + numbers(10000000) + numbers_mt(100000000) From 93dba03e8115635712136766a98a30374b364134 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 30 Jan 2020 15:51:47 +0300 Subject: [PATCH 0228/2007] SHOW CREATE for DatabaseMemory --- dbms/src/Databases/DatabaseMemory.cpp | 24 +++++++++++++-- dbms/src/Databases/DatabaseMemory.h | 3 ++ dbms/src/Databases/DatabasesCommon.cpp | 30 ++++++++++++------- dbms/src/Databases/DatabasesCommon.h | 3 ++ .../01069_database_memory.reference | 1 + .../0_stateless/01069_database_memory.sql | 3 ++ 6 files changed, 50 insertions(+), 14 deletions(-) diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index 28a6a8a7af0..c54d1abfb8f 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -7,6 +7,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int UNKNOWN_TABLE; +} + DatabaseMemory::DatabaseMemory(const String & name_) : DatabaseWithOwnTablesBase(name_, "DatabaseMemory(" + name_ + ")") , data_path("data/" + escapeForFileName(database_name) + "/") @@ -16,16 +21,20 @@ void DatabaseMemory::createTable( const Context & /*context*/, const String & table_name, const StoragePtr & table, - const ASTPtr & /*query*/) + const ASTPtr & query) { - attachTable(table_name, table); + std::lock_guard lock{mutex}; + attachTableUnlocked(table_name, table); + create_queries.emplace(table_name, query); } void DatabaseMemory::removeTable( const Context & /*context*/, const String & table_name) { - detachTable(table_name); + std::lock_guard lock{mutex}; + detachTableUnlocked(table_name); + create_queries.erase(table_name); } ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context & /*context*/) const @@ -37,4 +46,13 @@ ASTPtr DatabaseMemory::getCreateDatabaseQuery(const Context & /*context*/) const return create_query; } +ASTPtr DatabaseMemory::getCreateTableQueryImpl(const Context &, const String & table_name, bool throw_on_error) const +{ + std::lock_guard lock{mutex}; + auto it = create_queries.find(table_name); + if (it == create_queries.end() && throw_on_error) + throw Exception("No table " + table_name + " in database " + database_name, ErrorCodes::UNKNOWN_TABLE); + return it->second; +} + } diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 01789c11901..63fa896fefa 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -33,6 +33,7 @@ public: const Context & context, const String & table_name) override; + ASTPtr getCreateTableQueryImpl(const Context & /*context*/, const String & name, bool throw_on_error) const override; ASTPtr getCreateDatabaseQuery(const Context & /*context*/) const override; /// DatabaseMemory allows to create tables, which store data on disk. @@ -44,6 +45,8 @@ public: private: String data_path; + using NameToASTCreate = std::unordered_map; + NameToASTCreate create_queries; }; } diff --git a/dbms/src/Databases/DatabasesCommon.cpp b/dbms/src/Databases/DatabasesCommon.cpp index becdad672a1..756019eccf1 100644 --- a/dbms/src/Databases/DatabasesCommon.cpp +++ b/dbms/src/Databases/DatabasesCommon.cpp @@ -66,18 +66,21 @@ bool DatabaseWithOwnTablesBase::empty(const Context & /*context*/) const StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) { - StoragePtr res; - { - std::lock_guard lock(mutex); - if (dictionaries.count(table_name)) - throw Exception("Cannot detach dictionary " + database_name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE); + std::lock_guard lock(mutex); + return detachTableUnlocked(table_name); +} - auto it = tables.find(table_name); - if (it == tables.end()) - throw Exception("Table " + backQuote(database_name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - res = it->second; - tables.erase(it); - } +StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name) +{ + StoragePtr res; + if (dictionaries.count(table_name)) + throw Exception("Cannot detach dictionary " + database_name + "." + table_name + " as table, use DETACH DICTIONARY query.", ErrorCodes::UNKNOWN_TABLE); + + auto it = tables.find(table_name); + if (it == tables.end()) + throw Exception("Table " + backQuote(database_name) + "." + backQuote(table_name) + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + res = it->second; + tables.erase(it); return res; } @@ -85,6 +88,11 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(const String & table_name) void DatabaseWithOwnTablesBase::attachTable(const String & table_name, const StoragePtr & table) { std::lock_guard lock(mutex); + attachTableUnlocked(table_name, table); +} + +void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, const StoragePtr & table) +{ if (!tables.emplace(table_name, table).second) throw Exception("Table " + database_name + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } diff --git a/dbms/src/Databases/DatabasesCommon.h b/dbms/src/Databases/DatabasesCommon.h index 291c0e56e8f..1cefb8949bc 100644 --- a/dbms/src/Databases/DatabasesCommon.h +++ b/dbms/src/Databases/DatabasesCommon.h @@ -46,6 +46,9 @@ protected: Poco::Logger * log; DatabaseWithOwnTablesBase(const String & name_, const String & logger); + + void attachTableUnlocked(const String & table_name, const StoragePtr & table); + StoragePtr detachTableUnlocked(const String & table_name); }; } diff --git a/dbms/tests/queries/0_stateless/01069_database_memory.reference b/dbms/tests/queries/0_stateless/01069_database_memory.reference index b2fa2bd204c..35175dd4b62 100644 --- a/dbms/tests/queries/0_stateless/01069_database_memory.reference +++ b/dbms/tests/queries/0_stateless/01069_database_memory.reference @@ -5,3 +5,4 @@ CREATE DATABASE memory_01069 ENGINE = Memory() 4 3 4 +CREATE TABLE memory_01069.file (`n` UInt8) ENGINE = File(CSV) diff --git a/dbms/tests/queries/0_stateless/01069_database_memory.sql b/dbms/tests/queries/0_stateless/01069_database_memory.sql index 645790e3f5e..ed006a5ce04 100644 --- a/dbms/tests/queries/0_stateless/01069_database_memory.sql +++ b/dbms/tests/queries/0_stateless/01069_database_memory.sql @@ -15,4 +15,7 @@ DROP TABLE memory_01069.mt; SELECT * FROM memory_01069.mt ORDER BY n; -- { serverError 60 } SELECT * FROM memory_01069.file ORDER BY n; +SHOW CREATE TABLE memory_01069.mt; -- { serverError 60 } +SHOW CREATE TABLE memory_01069.file; + DROP DATABASE memory_01069; From a508c25d05688cf79fc4dbf6f8fb02b4daafc4f4 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Thu, 30 Jan 2020 22:00:51 +0300 Subject: [PATCH 0229/2007] store temporary tables in DatabaseMemory --- dbms/src/Databases/DatabaseMemory.cpp | 2 +- dbms/src/Databases/DatabaseMemory.h | 3 +- dbms/src/Interpreters/Context.cpp | 115 ++++++++++++++---- dbms/src/Interpreters/Context.h | 9 +- .../Interpreters/InterpreterCreateQuery.cpp | 2 +- .../src/Interpreters/InterpreterDropQuery.cpp | 2 +- .../Interpreters/InterpreterSelectQuery.cpp | 2 +- 7 files changed, 105 insertions(+), 30 deletions(-) diff --git a/dbms/src/Databases/DatabaseMemory.cpp b/dbms/src/Databases/DatabaseMemory.cpp index c54d1abfb8f..c83c91d253c 100644 --- a/dbms/src/Databases/DatabaseMemory.cpp +++ b/dbms/src/Databases/DatabaseMemory.cpp @@ -51,7 +51,7 @@ ASTPtr DatabaseMemory::getCreateTableQueryImpl(const Context &, const String & t std::lock_guard lock{mutex}; auto it = create_queries.find(table_name); if (it == create_queries.end() && throw_on_error) - throw Exception("No table " + table_name + " in database " + database_name, ErrorCodes::UNKNOWN_TABLE); + throw Exception("There is no metadata of table " + table_name + " in database " + database_name, ErrorCodes::UNKNOWN_TABLE); return it->second; } diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 63fa896fefa..6a0752ce778 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -19,7 +19,8 @@ namespace DB class DatabaseMemory : public DatabaseWithOwnTablesBase { public: - DatabaseMemory(const String & name_); + //FIXME default name + DatabaseMemory(const String & name_ = "_temporary_and_external_tables"); String getEngineName() const override { return "Memory"; } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index d290f8dc4f4..7a9013b106b 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -58,6 +58,8 @@ #include #include +#include + namespace ProfileEvents { extern const Event ContextLock; @@ -99,6 +101,57 @@ namespace ErrorCodes extern const int ACCESS_DENIED; } +struct TemporaryTableHolder : boost::noncopyable +{ + static constexpr const char * database_name = "_temporary_and_external_tables"; + + TemporaryTableHolder(const Context & context_, + DatabaseMemory & external_tables_, + const StoragePtr & table, + const ASTPtr & query = {}) + : context(context_), external_tables(external_tables_) + { + if (query) + { + ASTCreateQuery & create = dynamic_cast(*query); + if (create.uuid == UUIDHelpers::Nil) + create.uuid = UUIDHelpers::generateV4(); + id = create.uuid; + } + else + id = UUIDHelpers::generateV4(); + external_tables.createTable(context, "_data_" + toString(id), table, query); + } + + TemporaryTableHolder(TemporaryTableHolder && other) + : context(other.context), external_tables(other.external_tables), id(other.id) + { + other.id = UUIDHelpers::Nil; + } + + ~TemporaryTableHolder() + { + external_tables.removeTable(context, "_data_" + toString(id)); + } + + StorageID getGlobalTableID() const + { + return StorageID{database_name, "_data_" + toString(id), id}; + } + + StoragePtr getTable() const + { + auto table = external_tables.tryGetTable(context, "_data_" + toString(id)); + if (!table) + throw Exception("Temporary table " + getGlobalTableID().getNameForLogs() + " not found", ErrorCodes::LOGICAL_ERROR); + return table; + } + + const Context & context; + DatabaseMemory & external_tables; + UUID id; +}; + /** Set of known objects (environment), that could be used in query. * Shared (global) part. Order of members (especially, order of destruction) is very important. @@ -134,6 +187,8 @@ struct ContextShared mutable VolumePtr tmp_volume; /// Volume for the the temporary files that occur when processing the request. Databases databases; /// List of databases and tables in them. + DatabaseMemory temporary_and_external_tables; + mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::optional external_dictionaries_loader; mutable std::optional external_models_loader; @@ -785,9 +840,8 @@ void Context::removeDependency(const StorageID & from, const StorageID & where) Dependencies Context::getDependencies(const StorageID & from) const { auto lock = getLock(); - - String db = resolveDatabase(from.database_name, current_database); - ViewDependencies::const_iterator iter = shared->view_dependencies.find(StorageID(db, from.table_name, from.uuid)); + StorageID resolved = resolveStorageIDUnlocked(from); + ViewDependencies::const_iterator iter = shared->view_dependencies.find(resolved); if (iter == shared->view_dependencies.end()) return {}; @@ -820,7 +874,7 @@ bool Context::isDatabaseExist(const String & database_name) const bool Context::isExternalTableExist(const String & table_name) const { - return external_tables.end() != external_tables.find(table_name); + return external_tables_mapping.count(table_name); } @@ -872,8 +926,8 @@ Tables Context::getExternalTables() const auto lock = getLock(); Tables res; - for (auto & table : external_tables) - res[table.first] = table.second.first; + for (auto & table : external_tables_mapping) + res[table.first] = table.second->getTable(); if (session_context && session_context != this) { @@ -891,11 +945,11 @@ Tables Context::getExternalTables() const StoragePtr Context::tryGetExternalTable(const String & table_name) const { - TableAndCreateASTs::const_iterator jt = external_tables.find(table_name); - if (external_tables.end() == jt) + auto it = external_tables_mapping.find(table_name); + if (external_tables_mapping.end() == it) return StoragePtr(); - return jt->second.first; + return it->second->getTable(); } StoragePtr Context::getTable(const String & database_name, const String & table_name) const @@ -965,10 +1019,11 @@ StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional(*this, shared->temporary_and_external_tables, storage, ast)); } @@ -984,16 +1039,9 @@ bool Context::hasScalar(const String & name) const } -StoragePtr Context::tryRemoveExternalTable(const String & table_name) +bool Context::removeExternalTable(const String & table_name) { - TableAndCreateASTs::const_iterator it = external_tables.find(table_name); - - if (external_tables.end() == it) - return StoragePtr(); - - auto storage = it->second.first; - external_tables.erase(it); - return storage; + return external_tables_mapping.erase(table_name); } @@ -1080,11 +1128,11 @@ DatabasePtr Context::detachDatabase(const String & database_name) ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { - TableAndCreateASTs::const_iterator jt = external_tables.find(table_name); - if (external_tables.end() == jt) + auto it = external_tables_mapping.find(table_name); + if (external_tables_mapping.end() == it) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - return jt->second.second; + return shared->temporary_and_external_tables.getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); } Settings Context::getSettings() const @@ -2163,6 +2211,27 @@ void Context::resetInputCallbacks() input_blocks_reader = {}; } +StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const +{ + if (storage_id.uuid != UUIDHelpers::Nil) + { + //TODO maybe update table and db name? + //TODO add flag `resolved` to StorageID and check access rights if it was not previously resolved + return storage_id; + } + if (storage_id.database_name.empty()) + { + auto it = external_tables_mapping.find(storage_id.getTableName()); + if (it != external_tables_mapping.end()) + return it->second->getGlobalTableID(); /// Do not check access rights for session-local table + if (current_database.empty()) + throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); + storage_id.database_name = current_database; + } + checkDatabaseAccessRightsImpl(storage_id.database_name); + return storage_id; +} + SessionCleaner::~SessionCleaner() { diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 07fa6b06c1f..3e0997db151 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -143,6 +143,8 @@ struct SubscriptionForUserChange SubscriptionForUserChange & operator =(const SubscriptionForUserChange &) { subscription = {}; return *this; } }; +struct TemporaryTableHolder; + /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) * and copied part (which can be its own for each session or query). @@ -178,7 +180,9 @@ private: String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification. /// Thus, used in HTTP interface. If not specified - then some globally default format is used. // TODO maybe replace with DatabaseMemory? - TableAndCreateASTs external_tables; /// Temporary tables. + //TableAndCreateASTs external_tables; /// Temporary tables. + using TemporaryTablesMapping = std::map>; + TemporaryTablesMapping external_tables_mapping; Scalars scalars; StoragePtr view_source; /// Temporary StorageValues used to generate alias columns for materialized views Tables table_function_results; /// Temporary tables obtained by execution of table functions. Keyed by AST tree id. @@ -316,7 +320,8 @@ public: void addExternalTable(const String & table_name, const StoragePtr & storage, const ASTPtr & ast = {}); void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; - StoragePtr tryRemoveExternalTable(const String & table_name); + bool removeExternalTable(const String & table_name); + StorageID resolveStorageIDUnlocked(StorageID storage_id) const; StoragePtr executeTableFunction(const ASTPtr & table_expression); diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 471f38cfee8..4ca606d5a98 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -616,7 +616,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, throw Exception("Table " + create.database + "." + table_name + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } } - else if (context.tryGetExternalTable(table_name) && create.if_not_exists) + else if (context.isExternalTableExist(table_name) && create.if_not_exists) return false; StoragePtr res; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index c51365ad2ba..94d077abe8c 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -221,7 +221,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, } else if (kind == ASTDropQuery::Kind::Drop) { - context_handle.tryRemoveExternalTable(table_name); + context_handle.removeExternalTable(table_name); table->shutdown(); /// If table was already dropped by anyone, an exception will be thrown auto table_lock = table->lockExclusively(context.getCurrentQueryId()); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index ac7ea12d898..29c25115469 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -357,7 +357,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// Save the new temporary tables in the query context for (const auto & it : query_analyzer->getExternalTables()) - if (!context->tryGetExternalTable(it.first)) + if (!context->isExternalTableExist(it.first)) context->addExternalTable(it.first, it.second); } From b7197ba531e2249637866ba7d1890be28de30156 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 16:40:42 +0300 Subject: [PATCH 0230/2007] fix build --- dbms/src/Core/UUID.h | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/dbms/src/Core/UUID.h b/dbms/src/Core/UUID.h index 4f8fdced814..d5a6232a72b 100644 --- a/dbms/src/Core/UUID.h +++ b/dbms/src/Core/UUID.h @@ -2,10 +2,24 @@ #include #include +#include namespace DB { STRONG_TYPEDEF(UInt128, UUID) +namespace UUIDHelpers +{ + inline UUID generateV4() + { + UInt128 res{thread_local_rng(), thread_local_rng()}; + res.low = (res.low & 0xffffffffffff0fffull) | 0x0000000000004000ull; + res.high = (res.high & 0x3fffffffffffffffull) | 0x8000000000000000ull; + return UUID{res}; + } + + const UUID Nil = UUID(UInt128(0, 0)); +} + } From 4ff7bf78aba768561f444e1aa1bff4309f5995de Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 3 Feb 2020 15:54:36 +0300 Subject: [PATCH 0231/2007] add DatabaseCatalog --- dbms/programs/local/LocalServer.cpp | 2 + dbms/programs/server/Server.cpp | 2 + dbms/src/Databases/DatabaseMemory.h | 3 +- dbms/src/Interpreters/Context.cpp | 232 ++++++------------ dbms/src/Interpreters/Context.h | 27 +- dbms/src/Interpreters/DatabaseCatalog.cpp | 228 +++++++++++++++++ dbms/src/Interpreters/DatabaseCatalog.h | 91 +++++++ .../InterpreterShowTablesQuery.cpp | 3 +- 8 files changed, 411 insertions(+), 177 deletions(-) create mode 100644 dbms/src/Interpreters/DatabaseCatalog.cpp create mode 100644 dbms/src/Interpreters/DatabaseCatalog.h diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index 5cfceaeb592..e9fca03d7e8 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -200,6 +201,7 @@ try loadMetadataSystem(*context); attachSystemTables(); loadMetadata(*context); + context->getDatabaseCatalog().loadDatabases(); LOG_DEBUG(log, "Loaded metadata."); } else diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index 0cd357bbd94..fc53bf1c462 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -551,6 +552,7 @@ int Server::main(const std::vector & /*args*/) attachSystemTablesServer(*global_context->getDatabase("system"), has_zookeeper); /// Then, load remaining databases loadMetadata(*global_context); + global_context->getDatabaseCatalog().loadDatabases(); } catch (...) { diff --git a/dbms/src/Databases/DatabaseMemory.h b/dbms/src/Databases/DatabaseMemory.h index 6a0752ce778..63fa896fefa 100644 --- a/dbms/src/Databases/DatabaseMemory.h +++ b/dbms/src/Databases/DatabaseMemory.h @@ -19,8 +19,7 @@ namespace DB class DatabaseMemory : public DatabaseWithOwnTablesBase { public: - //FIXME default name - DatabaseMemory(const String & name_ = "_temporary_and_external_tables"); + DatabaseMemory(const String & name_); String getEngineName() const override { return "Memory"; } diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 7a9013b106b..07d6c001074 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -59,6 +59,7 @@ #include #include +#include namespace ProfileEvents { @@ -103,10 +104,8 @@ namespace ErrorCodes struct TemporaryTableHolder : boost::noncopyable { - static constexpr const char * database_name = "_temporary_and_external_tables"; - TemporaryTableHolder(const Context & context_, - DatabaseMemory & external_tables_, + IDatabase & external_tables_, const StoragePtr & table, const ASTPtr & query = {}) : context(context_), external_tables(external_tables_) @@ -136,7 +135,7 @@ struct TemporaryTableHolder : boost::noncopyable StorageID getGlobalTableID() const { - return StorageID{database_name, "_data_" + toString(id), id}; + return StorageID{DatabaseCatalog::TEMPORARY_DATABASE, "_data_" + toString(id), id}; } StoragePtr getTable() const @@ -148,7 +147,7 @@ struct TemporaryTableHolder : boost::noncopyable } const Context & context; - DatabaseMemory & external_tables; + IDatabase & external_tables; UUID id; }; @@ -186,8 +185,7 @@ struct ContextShared String tmp_path; /// Path to the temporary files that occur when processing the request. mutable VolumePtr tmp_volume; /// Volume for the the temporary files that occur when processing the request. - Databases databases; /// List of databases and tables in them. - DatabaseMemory temporary_and_external_tables; + std::shared_ptr database_catalog; /// Manages databases and tables in them. mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::optional external_dictionaries_loader; @@ -319,28 +317,7 @@ struct ContextShared if (system_logs) system_logs->shutdown(); - /** At this point, some tables may have threads that block our mutex. - * To shutdown them correctly, we will copy the current list of tables, - * and ask them all to finish their work. - * Then delete all objects with tables. - */ - - Databases current_databases; - - { - std::lock_guard lock(mutex); - current_databases = databases; - } - - /// We still hold "databases" in Context (instead of std::move) for Buffer tables to flush data correctly. - - for (auto & database : current_databases) - database.second->shutdown(); - - { - std::lock_guard lock(mutex); - databases.clear(); - } + database_catalog->shutdown(); /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). /// TODO: Get rid of this. @@ -385,6 +362,7 @@ Context Context::createGlobal() res.row_policy = std::make_shared(); res.access_rights = std::make_shared(); res.shared = std::make_shared(); + res.shared->database_catalog = std::make_shared(res); return res; } @@ -406,19 +384,6 @@ MergeList & Context::getMergeList() { return shared->merge_list; } const MergeList & Context::getMergeList() const { return shared->merge_list; } -const Databases Context::getDatabases() const -{ - auto lock = getLock(); - return shared->databases; -} - -Databases Context::getDatabases() -{ - auto lock = getLock(); - return shared->databases; -} - - Context::SessionKey Context::getSessionKey(const String & session_id) const { auto & user_name = client_info.current_user; @@ -526,50 +491,66 @@ std::chrono::steady_clock::duration Context::closeSessions() const return shared->close_interval; } - -static String resolveDatabase(const String & database_name, const String & current_database) +DatabaseCatalog & Context::getDatabaseCatalog() const { + return *shared->database_catalog; +} + +Databases Context::getDatabases() const +{ + return shared->database_catalog->getDatabases(); +} + +String Context::resolveDatabase(const String & database_name) const +{ + String res = database_name.empty() ? getCurrentDatabase() : database_name; + if (res.empty()) + throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); + return res; +} + +String Context::resolveDatabaseAndCheckAccess(const String & database_name) const +{ + auto lock = getLock(); String res = database_name.empty() ? current_database : database_name; if (res.empty()) throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); return res; } +//StorageID Context::resolveDatabase(StorageID table_id) const +//{ +// table_id.database_name = resolveDatabase(table_id.database_name); +// return table_id; +//} -const DatabasePtr Context::getDatabase(const String & database_name) const +DatabasePtr Context::getDatabase(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - assertDatabaseExists(db); - return shared->databases[db]; + auto db = resolveDatabaseAndCheckAccess(database_name); + return shared->database_catalog->getDatabase(db, *this); } -DatabasePtr Context::getDatabase(const String & database_name) +DatabasePtr Context::tryGetDatabase(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - assertDatabaseExists(db); - return shared->databases[db]; + String db = resolveDatabase(database_name); + return shared->database_catalog->tryGetDatabase(database_name, *this); } -const DatabasePtr Context::tryGetDatabase(const String & database_name) const +bool Context::isDatabaseExist(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - auto it = shared->databases.find(db); - if (it == shared->databases.end()) - return {}; - return it->second; + String db = resolveDatabaseAndCheckAccess(database_name); + return shared->database_catalog->isDatabaseExist(database_name); } -DatabasePtr Context::tryGetDatabase(const String & database_name) +void Context::addDatabase(const String & database_name, const DatabasePtr & database) { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - auto it = shared->databases.find(db); - if (it == shared->databases.end()) - return {}; - return it->second; + shared->database_catalog->attachDatabase(database_name, database, *this); +} + + +DatabasePtr Context::detachDatabase(const String & database_name) +{ + return shared->database_catalog->detachDatabase(database_name, *this); } String Context::getPath() const @@ -850,26 +831,17 @@ Dependencies Context::getDependencies(const StorageID & from) const bool Context::isTableExist(const String & database_name, const String & table_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - Databases::const_iterator it = shared->databases.find(db); - return shared->databases.end() != it - && it->second->isTableExist(*this, table_name); + //FIXME do we need resolve temporary tables here? + auto table_id = resolveStorageID({database_name, table_name}); + return shared->database_catalog->isTableExist(table_id, *this); } bool Context::isDictionaryExists(const String & database_name, const String & dictionary_name) const { auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - Databases::const_iterator it = shared->databases.find(db); - return shared->databases.end() != it && it->second->isDictionaryExist(*this, dictionary_name); -} - -bool Context::isDatabaseExist(const String & database_name) const -{ - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - return shared->databases.end() != shared->databases.find(db); + String db = resolveDatabaseAndCheckAccess(database_name); + auto db_ptr = shared->database_catalog->tryGetDatabase(database_name, *this); + return db_ptr && db_ptr->isDictionaryExist(*this, dictionary_name); } bool Context::isExternalTableExist(const String & table_name) const @@ -880,32 +852,18 @@ bool Context::isExternalTableExist(const String & table_name) const void Context::assertTableDoesntExist(const String & database_name, const String & table_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - Databases::const_iterator it = shared->databases.find(db); - if (shared->databases.end() != it && it->second->isTableExist(*this, table_name)) - throw Exception("Table " + backQuoteIfNeed(db) + "." + backQuoteIfNeed(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); + //FIXME do we need resolve temporary tables here? (and do we need this method?) + auto table_id = resolveStorageID({database_name, table_name}); + shared->database_catalog->assertTableDoesntExist(table_id, *this); } void Context::assertDatabaseExists(const String & database_name) const { - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - if (shared->databases.end() == shared->databases.find(db)) - throw Exception("Database " + backQuoteIfNeed(db) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); + String db = resolveDatabaseAndCheckAccess(database_name); + shared->database_catalog->assertDatabaseExists(db); } - -void Context::assertDatabaseDoesntExist(const String & database_name) const -{ - auto lock = getLock(); - String db = resolveDatabase(database_name, current_database); - if (shared->databases.end() != shared->databases.find(db)) - throw Exception("Database " + backQuoteIfNeed(db) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); -} - - const Scalars & Context::getScalars() const { return scalars; @@ -979,41 +937,8 @@ StoragePtr Context::tryGetTable(const StorageID & table_id) const StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional * exception) const { - String db; - DatabasePtr database; - - { - auto lock = getLock(); - - if (table_id.database_name.empty()) - { - StoragePtr res = tryGetExternalTable(table_id.table_name); - if (res) - return res; - } - - db = resolveDatabase(table_id.database_name, current_database); - - Databases::const_iterator it = shared->databases.find(db); - if (shared->databases.end() == it) - { - if (exception) - exception->emplace("Database " + backQuoteIfNeed(db) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - return {}; - } - - database = it->second; - } - - auto table = database->tryGetTable(*this, table_id.table_name); - if (!table) - { - if (exception) - exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); - return {}; - } - - return table; + auto resolved_id = resolveStorageID(table_id); + return shared->database_catalog->getTable(resolved_id, *this, exception); } @@ -1023,7 +948,8 @@ void Context::addExternalTable(const String & table_name, const StoragePtr & sto if (external_tables_mapping.end() != external_tables_mapping.find(table_name)) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); - external_tables_mapping.emplace(table_name, std::make_shared(*this, shared->temporary_and_external_tables, storage, ast)); + auto holder = std::make_shared(*this, *shared->database_catalog->getDatabaseForTemporaryTables(), storage, ast); + external_tables_mapping.emplace(table_name, std::move(holder)); } @@ -1106,33 +1032,13 @@ std::unique_ptr Context::getDDLGuard(const String & database, const St return std::make_unique(shared->ddl_guards[database], std::move(lock), table); } - -void Context::addDatabase(const String & database_name, const DatabasePtr & database) -{ - auto lock = getLock(); - - assertDatabaseDoesntExist(database_name); - shared->databases[database_name] = database; -} - - -DatabasePtr Context::detachDatabase(const String & database_name) -{ - auto lock = getLock(); - auto res = getDatabase(database_name); - shared->databases.erase(database_name); - - return res; -} - - ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { auto it = external_tables_mapping.find(table_name); if (external_tables_mapping.end() == it) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - return shared->temporary_and_external_tables.getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); + return shared->database_catalog->getDatabaseForTemporaryTables()->getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); } Settings Context::getSettings() const @@ -1215,6 +1121,7 @@ void Context::checkSettingsConstraints(const SettingsChanges & changes) String Context::getCurrentDatabase() const { + auto lock = getLock(); return current_database; } @@ -2211,6 +2118,12 @@ void Context::resetInputCallbacks() input_blocks_reader = {}; } +StorageID Context::resolveStorageID(StorageID storage_id) const +{ + auto lock = getLock(); + return resolveStorageIDUnlocked(std::move(storage_id)); +} + StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const { if (storage_id.uuid != UUIDHelpers::Nil) @@ -2228,7 +2141,6 @@ StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); storage_id.database_name = current_database; } - checkDatabaseAccessRightsImpl(storage_id.database_name); return storage_id; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 3e0997db151..da316dc4e4a 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -144,6 +144,8 @@ struct SubscriptionForUserChange }; struct TemporaryTableHolder; +class DatabaseCatalog; +using DatabaseCatalogPtr = std::shared_ptr; /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) @@ -300,6 +302,8 @@ public: void addDependencyUnsafe(const StorageID & from, const StorageID & where); void removeDependencyUnsafe(const StorageID & from, const StorageID & where); + DatabaseCatalog & getDatabaseCatalog() const; + /// Checking the existence of the table/database. Database can be empty - in this case the current database is used. bool isTableExist(const String & database_name, const String & table_name) const; bool isDatabaseExist(const String & database_name) const; @@ -307,7 +311,12 @@ public: bool isExternalTableExist(const String & table_name) const; void assertTableDoesntExist(const String & database_name, const String & table_name) const; void assertDatabaseExists(const String & database_name) const; - void assertDatabaseDoesntExist(const String & database_name) const; + + String resolveDatabase(const String & database_name) const; + String resolveDatabaseAndCheckAccess(const String & database_name) const; + //StorageID resolveDatabase(StorageID table_id) const; + StorageID resolveStorageID(StorageID storage_id) const; + StorageID resolveStorageIDUnlocked(StorageID storage_id) const; const Scalars & getScalars() const; const Block & getScalar(const String & name) const; @@ -321,7 +330,6 @@ public: void addScalar(const String & name, const Block & block); bool hasScalar(const String & name) const; bool removeExternalTable(const String & table_name); - StorageID resolveStorageIDUnlocked(StorageID storage_id) const; StoragePtr executeTableFunction(const ASTPtr & table_expression); @@ -410,13 +418,10 @@ public: /// Get query for the CREATE table. ASTPtr getCreateExternalTableQuery(const String & table_name) const; - const DatabasePtr getDatabase(const String & database_name) const; - DatabasePtr getDatabase(const String & database_name); - const DatabasePtr tryGetDatabase(const String & database_name) const; - DatabasePtr tryGetDatabase(const String & database_name); + DatabasePtr getDatabase(const String & database_name) const; + DatabasePtr tryGetDatabase(const String & database_name) const; - const Databases getDatabases() const; - Databases getDatabases(); + Databases getDatabases() const; std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) const; void releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout); @@ -617,12 +622,6 @@ private: void calculateUserSettings(); void calculateAccessRights(); - /** Check if the current client has access to the specified database. - * If access is denied, throw an exception. - * NOTE: This method should always be called when the `shared->mutex` mutex is acquired. - */ - void checkDatabaseAccessRightsImpl(const std::string & database_name) const; - template void checkAccessImpl(const Args &... args) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp new file mode 100644 index 00000000000..e53a3298e36 --- /dev/null +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -0,0 +1,228 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_DATABASE; + extern const int UNKNOWN_TABLE; + extern const int TABLE_ALREADY_EXISTS; + extern const int DATABASE_ALREADY_EXISTS; +} + +void DatabaseCatalog::loadDatabases() +{ + + auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE); + attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables, global_context); +} + +void DatabaseCatalog::shutdown() +{ + /** At this point, some tables may have threads that block our mutex. + * To shutdown them correctly, we will copy the current list of tables, + * and ask them all to finish their work. + * Then delete all objects with tables. + */ + + Databases current_databases; + { + std::lock_guard lock(databases_mutex); + current_databases = databases; + } + + /// We still hold "databases" (instead of std::move) for Buffer tables to flush data correctly. + + for (auto & database : current_databases) + database.second->shutdown(); + + + std::lock_guard lock(databases_mutex); + databases.clear(); + for (auto & elem : uuid_map) + { + std::lock_guard map_lock(elem.mutex); + elem.map.clear(); + } +} + +DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const +{ + assert(uuid != UUIDHelpers::Nil && 0 <= getFirstLevelIdx(uuid) && getFirstLevelIdx(uuid) < uuid_map.size()); + const UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto it = map_part.map.find(uuid); + if (it == map_part.map.end()) + return {}; + return it->second; +} + +//String DatabaseCatalog::resolveDatabase(const String & database_name, const String & current_database) +//{ +// String res = database_name.empty() ? current_database : database_name; +// if (res.empty()) +// throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); +// return res; +//} + +StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const +{ + //if (table_id.hasUUID()) + //{ + // auto db_and_table = tryGetByUUID(table_id.uuid); + // if (!db_and_table.first || !db_and_table.second) + // { + // assert(!db_and_table.first && !db_and_table.second); + // if (exception) + // exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + // return {}; +// + // } + // return db_and_table.second; + //} + + std::lock_guard _lock{databases_mutex}; + + auto it = databases.find(table_id.getDatabaseName()); + if (databases.end() == it) + { + if (exception) + exception->emplace("Database " + backQuoteIfNeed(table_id.getDatabaseName()) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); + return {}; + } + + auto database = it->second; + auto table = database->tryGetTable(local_context, table_id.table_name); + if (!table && exception) + exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); + + return table; +} + +void DatabaseCatalog::assertDatabaseExists(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + assertDatabaseExistsUnlocked(database_name); +} + +void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + assertDatabaseDoesntExistUnlocked(database_name); +} + +void DatabaseCatalog::assertDatabaseExistsUnlocked(const String & database_name) const +{ + if (databases.end() == databases.find(database_name)) + throw Exception("Database " + backQuoteIfNeed(database_name) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); +} + + +void DatabaseCatalog::assertDatabaseDoesntExistUnlocked(const String & database_name) const +{ + if (databases.end() != databases.find(database_name)) + throw Exception("Database " + backQuoteIfNeed(database_name) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); +} + +void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database, const Context & /*local_context*/) +{ + //local_context.checkDatabaseAccessRights(database_name); + std::lock_guard lock{databases_mutex}; + assertDatabaseDoesntExistUnlocked(database_name); + databases[database_name] = database; +} + + +DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, const Context & local_context) +{ + //local_context.checkDatabaseAccessRights(database_name); + std::lock_guard lock{databases_mutex}; + auto res = getDatabase(database_name, local_context); //FIXME locks order + databases.erase(database_name); + return res; +} + +DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Context & /*local_context*/) const +{ + //String db = local_context.resolveDatabase(database_name); + //local_context.checkDatabaseAccessRights(db); //FIXME non-atomic + std::lock_guard lock{databases_mutex}; + assertDatabaseExistsUnlocked(database_name); + return databases.find(database_name)->second; +} + +DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name, const Context & /*local_context*/) const +{ + //String db = local_context.resolveDatabase(database_name); + std::lock_guard lock{databases_mutex}; + auto it = databases.find(database_name); + if (it == databases.end()) + return {}; + return it->second; +} + +bool DatabaseCatalog::isDatabaseExist(const String & database_name) const +{ + std::lock_guard lock{databases_mutex}; + return databases.end() != databases.find(database_name); +} + +Databases DatabaseCatalog::getDatabases() const +{ + std::lock_guard lock{databases_mutex}; + return databases; +} + +bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, const DB::Context & context) const +{ + //if (table_id.hasUUID()) + // return tryGetByUUID(table_id.uuid).second != nullptr; + //else + //{ + std::lock_guard lock{databases_mutex}; + auto db = databases.find(table_id.database_name); + return db != databases.end() && db->second->isTableExist(context, table_id.table_name); + //} +} + +void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id, const Context & context) const +{ + if (!isTableExist(table_id, context)) + throw Exception("Table " + table_id.getNameForLogs() + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); + +} + +DatabasePtr DatabaseCatalog::getDatabaseForTemporaryTables() const +{ + return getDatabase(TEMPORARY_DATABASE, global_context); +} + +void DatabaseCatalog::addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) +{ + assert(uuid != UUIDHelpers::Nil && 0 <= getFirstLevelIdx(uuid) && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + auto [_, inserted] = map_part.map.try_emplace(uuid, std::move(database), std::move(table)); + if (!inserted) + throw Exception("Mapping for table with UUID=" + toString(uuid) + " already exists", ErrorCodes::LOGICAL_ERROR); +} + +void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) +{ + assert(uuid != UUIDHelpers::Nil && 0 <= getFirstLevelIdx(uuid) && getFirstLevelIdx(uuid) < uuid_map.size()); + UUIDToStorageMapPart & map_part = uuid_map[getFirstLevelIdx(uuid)]; + std::lock_guard lock{map_part.mutex}; + if (!map_part.map.erase(uuid)) + throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); +} + +} + + diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h new file mode 100644 index 00000000000..beba4f404b0 --- /dev/null +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -0,0 +1,91 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +class Context; +class IDatabase; +struct StorageID; +class Exception; +using DatabasePtr = std::shared_ptr; +using DatabaseAndTable = std::pair; + +//TODO make singleton? +class DatabaseCatalog : boost::noncopyable +{ +public: + using Databases = std::map>; + static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables"; + static constexpr const char * SYSTEM_DATABASE = "system"; + + + DatabaseCatalog(Context & global_context_/*, String default_database_*/) + : global_context(global_context_)/*, default_database(std::move(default_database_))*/ {} + + void loadDatabases(); + void shutdown(); + + + //static String resolveDatabase(const String & database_name, const String & current_database); + void assertDatabaseExists(const String & database_name) const; + void assertDatabaseDoesntExist(const String & database_name) const; + + DatabasePtr getDatabaseForTemporaryTables() const; + + void attachDatabase(const String & database_name, const DatabasePtr & database, const Context & local_context); // ca, a + DatabasePtr detachDatabase(const String & database_name, const Context & local_context); // (sr), ca, a + + DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; // sr, ca, a + DatabasePtr tryGetDatabase(const String & database_name, const Context & local_context) const; // sr + bool isDatabaseExist(const String & database_name) const; // sr, ca + Databases getDatabases() const; + + DatabaseAndTable tryGetByUUID(const UUID & uuid) const; + + void assertTableDoesntExist(const StorageID & table_id, const Context & context) const; // sr, ca + bool isTableExist(const StorageID & table_id, const Context & context) const; // sr, ca + + void addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table); + void removeUUIDMapping(const UUID & uuid); + + StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const; + +private: + void assertDatabaseExistsUnlocked(const String & database_name) const; + void assertDatabaseDoesntExistUnlocked(const String & database_name) const; + + struct UUIDToStorageMapPart + { + std::unordered_map map; + mutable std::mutex mutex; + }; + + static constexpr UInt64 bits_for_first_level = 8; + using UUIDToStorageMap = std::array; + + inline size_t getFirstLevelIdx(const UUID & uuid) const + { + return uuid.toUnderType().low >> (64 - bits_for_first_level); + } + +private: + [[maybe_unused]] Context & global_context; + mutable std::mutex databases_mutex; + //const String default_database; + Databases databases; + UUIDToStorageMap uuid_map; + + +}; + + +} diff --git a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp index 28d06358327..4e5a84d30d2 100644 --- a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,7 @@ String InterpreterShowTablesQuery::getRewrittenQuery() throw Exception("The `FROM` and `TEMPORARY` cannot be used together in `SHOW TABLES`", ErrorCodes::SYNTAX_ERROR); String database = query.from.empty() ? context.getCurrentDatabase() : query.from; - context.assertDatabaseExists(database); + context.getDatabaseCatalog().assertDatabaseExists(database); std::stringstream rewritten_query; rewritten_query << "SELECT name FROM system."; From 869e20d2078738089f9abb887ed66094c30d0381 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 16:10:17 +0300 Subject: [PATCH 0232/2007] move databases from Context to DatabaseCatalog --- dbms/programs/copier/ClusterCopier.cpp | 2 +- dbms/programs/local/LocalServer.cpp | 10 +-- .../programs/server/ReplicasStatusHandler.cpp | 2 +- dbms/programs/server/Server.cpp | 6 +- dbms/programs/server/TCPHandler.cpp | 2 +- dbms/src/Interpreters/ActionLocksManager.cpp | 2 +- dbms/src/Interpreters/AsynchronousMetrics.cpp | 3 +- dbms/src/Interpreters/Context.cpp | 90 ++----------------- dbms/src/Interpreters/Context.h | 16 +--- dbms/src/Interpreters/DatabaseCatalog.cpp | 35 ++++++-- dbms/src/Interpreters/DatabaseCatalog.h | 18 ++-- .../Interpreters/InterpreterCreateQuery.cpp | 25 +++--- .../src/Interpreters/InterpreterDropQuery.cpp | 14 +-- .../Interpreters/InterpreterRenameQuery.cpp | 7 +- .../InterpreterShowCreateQuery.cpp | 6 +- .../InterpreterShowTablesQuery.cpp | 4 +- .../Interpreters/InterpreterSystemQuery.cpp | 7 +- dbms/src/Interpreters/loadMetadata.cpp | 2 +- dbms/src/Interpreters/tests/create_query.cpp | 2 +- .../tests/expression_analyzer.cpp | 2 +- .../tests/in_join_subqueries_preprocessor.cpp | 6 +- dbms/src/Interpreters/tests/select_query.cpp | 4 +- dbms/src/Storages/IStorage.cpp | 2 +- dbms/src/Storages/StorageBuffer.cpp | 2 +- dbms/src/Storages/StorageDistributed.cpp | 2 +- dbms/src/Storages/StorageMaterializedView.cpp | 2 +- dbms/src/Storages/StorageMerge.cpp | 4 +- dbms/src/Storages/StorageMergeTree.cpp | 4 +- dbms/src/Storages/StorageNull.cpp | 2 +- .../Storages/StorageReplicatedMergeTree.cpp | 6 +- .../Storages/System/StorageSystemColumns.cpp | 2 +- .../System/StorageSystemDatabases.cpp | 2 +- .../Storages/System/StorageSystemGraphite.cpp | 2 +- .../System/StorageSystemMutations.cpp | 2 +- .../System/StorageSystemPartsBase.cpp | 2 +- .../Storages/System/StorageSystemReplicas.cpp | 2 +- .../System/StorageSystemReplicationQueue.cpp | 2 +- .../Storages/System/StorageSystemTables.cpp | 5 +- ..._transform_query_for_external_database.cpp | 3 +- .../src/TableFunctions/TableFunctionMerge.cpp | 2 +- 40 files changed, 129 insertions(+), 184 deletions(-) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 2c6b16a7ae4..4d5a7788e92 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -2467,7 +2467,7 @@ void ClusterCopierApp::mainImpl() registerDisks(); static const std::string default_database = "_local"; - context->addDatabase(default_database, std::make_shared(default_database)); + DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); /// Initialize query scope just in case. diff --git a/dbms/programs/local/LocalServer.cpp b/dbms/programs/local/LocalServer.cpp index e9fca03d7e8..d5e08382424 100644 --- a/dbms/programs/local/LocalServer.cpp +++ b/dbms/programs/local/LocalServer.cpp @@ -188,7 +188,7 @@ try * if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons. */ std::string default_database = config().getString("default_database", "_local"); - context->addDatabase(default_database, std::make_shared(default_database)); + DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared(default_database)); context->setCurrentDatabase(default_database); applyCmdOptions(); @@ -201,7 +201,7 @@ try loadMetadataSystem(*context); attachSystemTables(); loadMetadata(*context); - context->getDatabaseCatalog().loadDatabases(); + DatabaseCatalog::instance().loadDatabases(); LOG_DEBUG(log, "Loaded metadata."); } else @@ -250,12 +250,12 @@ std::string LocalServer::getInitialCreateTableQuery() void LocalServer::attachSystemTables() { - DatabasePtr system_database = context->tryGetDatabase("system"); + DatabasePtr system_database = DatabaseCatalog::instance().tryGetDatabase(DatabaseCatalog::SYSTEM_DATABASE); if (!system_database) { /// TODO: add attachTableDelayed into DatabaseMemory to speedup loading - system_database = std::make_shared("system"); - context->addDatabase("system", system_database); + system_database = std::make_shared(DatabaseCatalog::SYSTEM_DATABASE); + DatabaseCatalog::instance().attachDatabase(DatabaseCatalog::SYSTEM_DATABASE, system_database); } attachSystemTablesLocal(*system_database); diff --git a/dbms/programs/server/ReplicasStatusHandler.cpp b/dbms/programs/server/ReplicasStatusHandler.cpp index 4d72c6da3cf..512b0889869 100644 --- a/dbms/programs/server/ReplicasStatusHandler.cpp +++ b/dbms/programs/server/ReplicasStatusHandler.cpp @@ -35,7 +35,7 @@ void ReplicasStatusHandler::handleRequest(Poco::Net::HTTPServerRequest & request bool ok = true; std::stringstream message; - auto databases = context.getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(); /// Iterate through all the replicated tables. for (const auto & db : databases) diff --git a/dbms/programs/server/Server.cpp b/dbms/programs/server/Server.cpp index fc53bf1c462..bde1d5c3cd3 100644 --- a/dbms/programs/server/Server.cpp +++ b/dbms/programs/server/Server.cpp @@ -549,10 +549,10 @@ int Server::main(const std::vector & /*args*/) /// After attaching system databases we can initialize system log. global_context->initializeSystemLogs(); /// After the system database is created, attach virtual system tables (in addition to query_log and part_log) - attachSystemTablesServer(*global_context->getDatabase("system"), has_zookeeper); + attachSystemTablesServer(*DatabaseCatalog::instance().getSystemDatabase(), has_zookeeper); /// Then, load remaining databases loadMetadata(*global_context); - global_context->getDatabaseCatalog().loadDatabases(); + DatabaseCatalog::instance().loadDatabases(); } catch (...) { @@ -693,7 +693,7 @@ int Server::main(const std::vector & /*args*/) /// This object will periodically calculate some metrics. AsynchronousMetrics async_metrics(*global_context); - attachSystemTablesAsync(*global_context->getDatabase("system"), async_metrics); + attachSystemTablesAsync(*DatabaseCatalog::instance().getSystemDatabase(), async_metrics); for (const auto & listen_host : listen_hosts) { diff --git a/dbms/programs/server/TCPHandler.cpp b/dbms/programs/server/TCPHandler.cpp index b645118494b..3cb00320b10 100644 --- a/dbms/programs/server/TCPHandler.cpp +++ b/dbms/programs/server/TCPHandler.cpp @@ -108,7 +108,7 @@ void TCPHandler::runImpl() /// When connecting, the default database can be specified. if (!default_database.empty()) { - if (!connection_context.isDatabaseExist(default_database)) + if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) { Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); LOG_ERROR(log, "Code: " << e.code() << ", e.displayText() = " << e.displayText() diff --git a/dbms/src/Interpreters/ActionLocksManager.cpp b/dbms/src/Interpreters/ActionLocksManager.cpp index cc5d01863a2..023129f7cdc 100644 --- a/dbms/src/Interpreters/ActionLocksManager.cpp +++ b/dbms/src/Interpreters/ActionLocksManager.cpp @@ -22,7 +22,7 @@ namespace ActionLocks template inline void forEachTable(Context & context, F && f) { - for (auto & elem : context.getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases()) for (auto iterator = elem.second->getTablesIterator(context); iterator->isValid(); iterator->next()) f(iterator->table()); diff --git a/dbms/src/Interpreters/AsynchronousMetrics.cpp b/dbms/src/Interpreters/AsynchronousMetrics.cpp index 661325b22c2..16d00036c24 100644 --- a/dbms/src/Interpreters/AsynchronousMetrics.cpp +++ b/dbms/src/Interpreters/AsynchronousMetrics.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -131,7 +132,7 @@ void AsynchronousMetrics::update() set("Uptime", context.getUptimeSeconds()); { - auto databases = context.getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(); size_t max_queue_size = 0; size_t max_inserts_in_queue = 0; diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 07d6c001074..ba7de8f877d 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -185,8 +185,6 @@ struct ContextShared String tmp_path; /// Path to the temporary files that occur when processing the request. mutable VolumePtr tmp_volume; /// Volume for the the temporary files that occur when processing the request. - std::shared_ptr database_catalog; /// Manages databases and tables in them. - mutable std::optional embedded_dictionaries; /// Metrica's dictionaries. Have lazy initialization. mutable std::optional external_dictionaries_loader; mutable std::optional external_models_loader; @@ -317,7 +315,7 @@ struct ContextShared if (system_logs) system_logs->shutdown(); - database_catalog->shutdown(); + DatabaseCatalog::instance().shutdown(); /// Preemptive destruction is important, because these objects may have a refcount to ContextShared (cyclic reference). /// TODO: Get rid of this. @@ -362,7 +360,6 @@ Context Context::createGlobal() res.row_policy = std::make_shared(); res.access_rights = std::make_shared(); res.shared = std::make_shared(); - res.shared->database_catalog = std::make_shared(res); return res; } @@ -491,16 +488,6 @@ std::chrono::steady_clock::duration Context::closeSessions() const return shared->close_interval; } -DatabaseCatalog & Context::getDatabaseCatalog() const -{ - return *shared->database_catalog; -} - -Databases Context::getDatabases() const -{ - return shared->database_catalog->getDatabases(); -} - String Context::resolveDatabase(const String & database_name) const { String res = database_name.empty() ? getCurrentDatabase() : database_name; @@ -509,50 +496,6 @@ String Context::resolveDatabase(const String & database_name) const return res; } -String Context::resolveDatabaseAndCheckAccess(const String & database_name) const -{ - auto lock = getLock(); - String res = database_name.empty() ? current_database : database_name; - if (res.empty()) - throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); - return res; -} - -//StorageID Context::resolveDatabase(StorageID table_id) const -//{ -// table_id.database_name = resolveDatabase(table_id.database_name); -// return table_id; -//} - -DatabasePtr Context::getDatabase(const String & database_name) const -{ - auto db = resolveDatabaseAndCheckAccess(database_name); - return shared->database_catalog->getDatabase(db, *this); -} - -DatabasePtr Context::tryGetDatabase(const String & database_name) const -{ - String db = resolveDatabase(database_name); - return shared->database_catalog->tryGetDatabase(database_name, *this); -} - -bool Context::isDatabaseExist(const String & database_name) const -{ - String db = resolveDatabaseAndCheckAccess(database_name); - return shared->database_catalog->isDatabaseExist(database_name); -} - -void Context::addDatabase(const String & database_name, const DatabasePtr & database) -{ - shared->database_catalog->attachDatabase(database_name, database, *this); -} - - -DatabasePtr Context::detachDatabase(const String & database_name) -{ - return shared->database_catalog->detachDatabase(database_name, *this); -} - String Context::getPath() const { auto lock = getLock(); @@ -833,14 +776,14 @@ bool Context::isTableExist(const String & database_name, const String & table_na { //FIXME do we need resolve temporary tables here? auto table_id = resolveStorageID({database_name, table_name}); - return shared->database_catalog->isTableExist(table_id, *this); + return DatabaseCatalog::instance().isTableExist(table_id, *this); } bool Context::isDictionaryExists(const String & database_name, const String & dictionary_name) const { auto lock = getLock(); - String db = resolveDatabaseAndCheckAccess(database_name); - auto db_ptr = shared->database_catalog->tryGetDatabase(database_name, *this); + String db = resolveDatabase(database_name); + auto db_ptr = DatabaseCatalog::instance().tryGetDatabase(database_name); return db_ptr && db_ptr->isDictionaryExist(*this, dictionary_name); } @@ -849,21 +792,6 @@ bool Context::isExternalTableExist(const String & table_name) const return external_tables_mapping.count(table_name); } - -void Context::assertTableDoesntExist(const String & database_name, const String & table_name) const -{ - //FIXME do we need resolve temporary tables here? (and do we need this method?) - auto table_id = resolveStorageID({database_name, table_name}); - shared->database_catalog->assertTableDoesntExist(table_id, *this); -} - - -void Context::assertDatabaseExists(const String & database_name) const -{ - String db = resolveDatabaseAndCheckAccess(database_name); - shared->database_catalog->assertDatabaseExists(db); -} - const Scalars & Context::getScalars() const { return scalars; @@ -938,7 +866,7 @@ StoragePtr Context::tryGetTable(const StorageID & table_id) const StoragePtr Context::getTableImpl(const StorageID & table_id, std::optional * exception) const { auto resolved_id = resolveStorageID(table_id); - return shared->database_catalog->getTable(resolved_id, *this, exception); + return DatabaseCatalog::instance().getTable(resolved_id, *this, exception); } @@ -948,7 +876,7 @@ void Context::addExternalTable(const String & table_name, const StoragePtr & sto if (external_tables_mapping.end() != external_tables_mapping.find(table_name)) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); - auto holder = std::make_shared(*this, *shared->database_catalog->getDatabaseForTemporaryTables(), storage, ast); + auto holder = std::make_shared(*this, *DatabaseCatalog::instance().getDatabaseForTemporaryTables(), storage, ast); external_tables_mapping.emplace(table_name, std::move(holder)); } @@ -1038,7 +966,7 @@ ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const if (external_tables_mapping.end() == it) throw Exception("Temporary table " + backQuoteIfNeed(table_name) + " doesn't exist", ErrorCodes::UNKNOWN_TABLE); - return shared->database_catalog->getDatabaseForTemporaryTables()->getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); + return DatabaseCatalog::instance().getDatabaseForTemporaryTables()->getCreateTableQuery(*this, it->second->getGlobalTableID().table_name); } Settings Context::getSettings() const @@ -1140,10 +1068,10 @@ String Context::getInitialQueryId() const void Context::setCurrentDatabase(const String & name) { + DatabaseCatalog::instance().assertDatabaseExists(name); auto lock = getLock(); - assertDatabaseExists(name); - current_database = name; calculateAccessRights(); + current_database = name; } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index da316dc4e4a..3ee96829e53 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -181,8 +182,6 @@ private: String default_format; /// Format, used when server formats data by itself and if query does not have FORMAT specification. /// Thus, used in HTTP interface. If not specified - then some globally default format is used. - // TODO maybe replace with DatabaseMemory? - //TableAndCreateASTs external_tables; /// Temporary tables. using TemporaryTablesMapping = std::map>; TemporaryTablesMapping external_tables_mapping; Scalars scalars; @@ -302,15 +301,10 @@ public: void addDependencyUnsafe(const StorageID & from, const StorageID & where); void removeDependencyUnsafe(const StorageID & from, const StorageID & where); - DatabaseCatalog & getDatabaseCatalog() const; - /// Checking the existence of the table/database. Database can be empty - in this case the current database is used. bool isTableExist(const String & database_name, const String & table_name) const; - bool isDatabaseExist(const String & database_name) const; bool isDictionaryExists(const String & database_name, const String & dictionary_name) const; bool isExternalTableExist(const String & table_name) const; - void assertTableDoesntExist(const String & database_name, const String & table_name) const; - void assertDatabaseExists(const String & database_name) const; String resolveDatabase(const String & database_name) const; String resolveDatabaseAndCheckAccess(const String & database_name) const; @@ -336,9 +330,6 @@ public: void addViewSource(const StoragePtr & storage); StoragePtr getViewSource(); - void addDatabase(const String & database_name, const DatabasePtr & database); - DatabasePtr detachDatabase(const String & database_name); - /// Get an object that protects the table from concurrently executing multiple DDL operations. std::unique_ptr getDDLGuard(const String & database, const String & table) const; @@ -418,11 +409,6 @@ public: /// Get query for the CREATE table. ASTPtr getCreateExternalTableQuery(const String & table_name) const; - DatabasePtr getDatabase(const String & database_name) const; - DatabasePtr tryGetDatabase(const String & database_name) const; - - Databases getDatabases() const; - std::shared_ptr acquireSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) const; void releaseSession(const String & session_id, std::chrono::steady_clock::duration timeout); diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index e53a3298e36..59e4368c381 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -17,11 +17,12 @@ namespace ErrorCodes extern const int DATABASE_ALREADY_EXISTS; } + void DatabaseCatalog::loadDatabases() { auto db_for_temporary_and_external_tables = std::make_shared(TEMPORARY_DATABASE); - attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables, global_context); + attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables); } void DatabaseCatalog::shutdown() @@ -131,7 +132,7 @@ void DatabaseCatalog::assertDatabaseDoesntExistUnlocked(const String & database_ throw Exception("Database " + backQuoteIfNeed(database_name) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); } -void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database, const Context & /*local_context*/) +void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database) { //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; @@ -140,17 +141,18 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas } -DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name, const Context & local_context) +DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name) { //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; - auto res = getDatabase(database_name, local_context); //FIXME locks order + auto res = getDatabase(database_name); //FIXME locks order databases.erase(database_name); return res; } -DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Context & /*local_context*/) const +DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const { + assert(!database_name.empty()); //String db = local_context.resolveDatabase(database_name); //local_context.checkDatabaseAccessRights(db); //FIXME non-atomic std::lock_guard lock{databases_mutex}; @@ -158,8 +160,9 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Con return databases.find(database_name)->second; } -DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name, const Context & /*local_context*/) const +DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const { + assert(!database_name.empty()); //String db = local_context.resolveDatabase(database_name); std::lock_guard lock{databases_mutex}; auto it = databases.find(database_name); @@ -170,6 +173,7 @@ DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name, const bool DatabaseCatalog::isDatabaseExist(const String & database_name) const { + assert(!database_name.empty()); std::lock_guard lock{databases_mutex}; return databases.end() != databases.find(database_name); } @@ -201,7 +205,12 @@ void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id, const C DatabasePtr DatabaseCatalog::getDatabaseForTemporaryTables() const { - return getDatabase(TEMPORARY_DATABASE, global_context); + return getDatabase(TEMPORARY_DATABASE); +} + +DatabasePtr DatabaseCatalog::getSystemDatabase() const +{ + return getDatabase(SYSTEM_DATABASE); } void DatabaseCatalog::addUUIDMapping(const UUID & uuid, DatabasePtr database, StoragePtr table) @@ -223,6 +232,18 @@ void DatabaseCatalog::removeUUIDMapping(const UUID & uuid) throw Exception("Mapping for table with UUID=" + toString(uuid) + " doesn't exist", ErrorCodes::LOGICAL_ERROR); } +DatabaseCatalog & DatabaseCatalog::instance() +{ + static DatabaseCatalog database_catalog; + return database_catalog; +} + +DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Context & local_context) const +{ + String resolved_database = local_context.resolveDatabase(database_name); + return getDatabase(resolved_database); +} + } diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index beba4f404b0..b5cae14e699 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -27,9 +27,10 @@ public: static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables"; static constexpr const char * SYSTEM_DATABASE = "system"; + static DatabaseCatalog & instance(); - DatabaseCatalog(Context & global_context_/*, String default_database_*/) - : global_context(global_context_)/*, default_database(std::move(default_database_))*/ {} + //DatabaseCatalog(/*Context & global_context_, String default_database_*/) {} + //: global_context(global_context_)/*, default_database(std::move(default_database_))*/ {} void loadDatabases(); void shutdown(); @@ -40,12 +41,14 @@ public: void assertDatabaseDoesntExist(const String & database_name) const; DatabasePtr getDatabaseForTemporaryTables() const; + DatabasePtr getSystemDatabase() const; - void attachDatabase(const String & database_name, const DatabasePtr & database, const Context & local_context); // ca, a - DatabasePtr detachDatabase(const String & database_name, const Context & local_context); // (sr), ca, a + void attachDatabase(const String & database_name, const DatabasePtr & database); // ca, a + DatabasePtr detachDatabase(const String & database_name); // (sr), ca, a - DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; // sr, ca, a - DatabasePtr tryGetDatabase(const String & database_name, const Context & local_context) const; // sr + DatabasePtr getDatabase(const String & database_name, const Context & local_context) const; + DatabasePtr getDatabase(const String & database_name) const; // sr, ca, a + DatabasePtr tryGetDatabase(const String & database_name) const; // sr bool isDatabaseExist(const String & database_name) const; // sr, ca Databases getDatabases() const; @@ -60,6 +63,7 @@ public: StoragePtr getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const; private: + DatabaseCatalog() = default; void assertDatabaseExistsUnlocked(const String & database_name) const; void assertDatabaseDoesntExistUnlocked(const String & database_name) const; @@ -78,7 +82,7 @@ private: } private: - [[maybe_unused]] Context & global_context; + //[[maybe_unused]] Context & global_context; mutable std::mutex databases_mutex; //const String default_database; Databases databases; diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index 4ca606d5a98..ab828fc7306 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -86,7 +86,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) auto guard = context.getDDLGuard(database_name, ""); /// Database can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard - if (context.isDatabaseExist(database_name)) + if (DatabaseCatalog::instance().isDatabaseExist(database_name)) { if (create.if_not_exists) return {}; @@ -147,7 +147,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) bool renamed = false; try { - context.addDatabase(database_name, database); + DatabaseCatalog::instance().attachDatabase(database_name, database); added = true; if (need_write_metadata) @@ -163,7 +163,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (renamed) Poco::File(metadata_file_tmp_path).remove(); if (added) - context.detachDatabase(database_name); + DatabaseCatalog::instance().detachDatabase(database_name); throw; } @@ -507,10 +507,10 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { /// NOTE Getting the structure from the table specified in the AS is done not atomically with the creation of the table. - String as_database_name = create.as_database.empty() ? context.getCurrentDatabase() : create.as_database; + String as_database_name = context.resolveDatabase(create.as_database); String as_table_name = create.as_table; - ASTPtr as_create_ptr = context.getDatabase(as_database_name)->getCreateTableQuery(context, as_table_name); + ASTPtr as_create_ptr = DatabaseCatalog::instance().getDatabase(as_database_name)->getCreateTableQuery(context, as_table_name); const auto & as_create = as_create_ptr->as(); const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name); @@ -542,18 +542,20 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) throw Exception("Temporary tables cannot be inside a database. You should not specify a database for a temporary table.", ErrorCodes::BAD_DATABASE_FOR_TEMPORARY_TABLE); + String current_database = context.getCurrentDatabase(); + // If this is a stub ATTACH query, read the query definition from the database if (create.attach && !create.storage && !create.columns_list) { bool if_not_exists = create.if_not_exists; // Table SQL definition is available even if the table is detached - auto query = context.getDatabase(create.database)->getCreateTableQuery(context, create.table); + auto database_name = create.database.empty() ? current_database : create.database; + auto query = DatabaseCatalog::instance().getDatabase(database_name)->getCreateTableQuery(context, create.table); create = query->as(); // Copy the saved create query, but use ATTACH instead of CREATE create.attach = true; create.if_not_exists = if_not_exists; } - String current_database = context.getCurrentDatabase(); if (!create.temporary && create.database.empty()) create.database = current_database; if (!create.to_table.empty() && create.to_database.empty()) @@ -588,7 +590,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, bool need_add_to_database = !create.temporary || create.is_live_view; if (need_add_to_database) { - database = context.getDatabase(create.database); + database = DatabaseCatalog::instance().getDatabase(create.database); /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). * If table doesnt exist, one thread is creating table, while others wait in DDLGuard. @@ -684,12 +686,11 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) { String dictionary_name = create.table; - if (create.database.empty()) - create.database = context.getCurrentDatabase(); + create.database = context.resolveDatabase(create.database); const String & database_name = create.database; auto guard = context.getDDLGuard(database_name, dictionary_name); - DatabasePtr database = context.getDatabase(database_name); + DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); if (database->isDictionaryExist(context, dictionary_name)) { @@ -703,7 +704,7 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) if (create.attach) { - auto query = context.getDatabase(database_name)->getCreateDictionaryQuery(context, dictionary_name); + auto query = DatabaseCatalog::instance().getDatabase(database_name)->getCreateDictionaryQuery(context, dictionary_name); create = query->as(); create.attach = true; } diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 94d077abe8c..9f7ea306f10 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -76,7 +76,7 @@ BlockIO InterpreterDropQuery::executeToTable( ErrorCodes::UNKNOWN_TABLE); } - String database_name = database_name_.empty() ? context.getCurrentDatabase() : database_name_; + String database_name = context.resolveDatabase(database_name_); auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, table_name) : nullptr); @@ -168,7 +168,7 @@ BlockIO InterpreterDropQuery::executeToDictionary( if (is_temporary) throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); - String database_name = database_name_.empty() ? context.getCurrentDatabase() : database_name_; + String database_name = context.resolveDatabase(database_name_); auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, dictionary_name) : nullptr); @@ -248,7 +248,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS else if (kind == ASTDropQuery::Kind::Detach) { context.checkAccess(AccessType::DETACH_DATABASE, database_name); - context.detachDatabase(database_name); + DatabaseCatalog::instance().detachDatabase(database_name); database->shutdown(); } else if (kind == ASTDropQuery::Kind::Drop) @@ -270,14 +270,14 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS auto context_lock = context.getLock(); /// Someone could have time to delete the database before us. - context.assertDatabaseExists(database_name); + DatabaseCatalog::instance().assertDatabaseExists(database_name); /// Someone could have time to create a table in the database to be deleted while we deleted the tables without the context lock. - if (!context.getDatabase(database_name)->empty(context)) + if (!DatabaseCatalog::instance().getDatabase(database_name)->empty(context)) throw Exception("New table appeared in database being dropped. Try dropping it again.", ErrorCodes::DATABASE_NOT_EMPTY); /// Delete database information from the RAM - context.detachDatabase(database_name); + DatabaseCatalog::instance().detachDatabase(database_name); database->shutdown(); @@ -296,7 +296,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS DatabasePtr InterpreterDropQuery::tryGetDatabase(const String & database_name, bool if_exists) { - return if_exists ? context.tryGetDatabase(database_name) : context.getDatabase(database_name); + return if_exists ? DatabaseCatalog::instance().tryGetDatabase(database_name) : DatabaseCatalog::instance().getDatabase(database_name); } DatabaseAndTable InterpreterDropQuery::tryGetDatabaseAndTable(const String & database_name, const String & table_name, bool if_exists) diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 7965291daaa..150fca16b53 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -88,16 +88,17 @@ BlockIO InterpreterRenameQuery::execute() for (auto & table_guard : table_guards) table_guard.second = context.getDDLGuard(table_guard.first.database_name, table_guard.first.table_name); + auto & database_catalog = DatabaseCatalog::instance(); for (auto & elem : descriptions) { - context.assertTableDoesntExist(elem.to_database_name, elem.to_table_name); + database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name), context); auto from_table = context.getTable(elem.from_database_name, elem.from_table_name); auto from_table_lock = from_table->lockExclusively(context.getCurrentQueryId()); - context.getDatabase(elem.from_database_name)->renameTable( + database_catalog.getDatabase(elem.from_database_name)->renameTable( context, elem.from_table_name, - *context.getDatabase(elem.to_database_name), + *database_catalog.getDatabase(elem.to_database_name), elem.to_table_name, from_table_lock); } diff --git a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp index 8d5e03b4a7d..4f59b6665d5 100644 --- a/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowCreateQuery.cpp @@ -52,7 +52,7 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() else { context.checkAccess(AccessType::SHOW, show_query->database, show_query->table); - create_query = context.getDatabase(show_query->database)->getCreateTableQuery(context, show_query->table); + create_query = DatabaseCatalog::instance().getDatabase(show_query->database, context)->getCreateTableQuery(context, show_query->table); } } else if ((show_query = query_ptr->as())) @@ -60,14 +60,14 @@ BlockInputStreamPtr InterpreterShowCreateQuery::executeImpl() if (show_query->temporary) throw Exception("Temporary databases are not possible.", ErrorCodes::SYNTAX_ERROR); context.checkAccess(AccessType::SHOW, show_query->database); - create_query = context.getDatabase(show_query->database)->getCreateDatabaseQuery(context); + create_query = DatabaseCatalog::instance().getDatabase(show_query->database)->getCreateDatabaseQuery(context); } else if ((show_query = query_ptr->as())) { if (show_query->temporary) throw Exception("Temporary dictionaries are not possible.", ErrorCodes::SYNTAX_ERROR); context.checkAccess(AccessType::SHOW, show_query->database, show_query->table); - create_query = context.getDatabase(show_query->database)->getCreateDictionaryQuery(context, show_query->table); + create_query = DatabaseCatalog::instance().getDatabase(show_query->database, context)->getCreateDictionaryQuery(context, show_query->table); } if (!create_query && show_query && show_query->temporary) diff --git a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp index 4e5a84d30d2..23d386a53f4 100644 --- a/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp +++ b/dbms/src/Interpreters/InterpreterShowTablesQuery.cpp @@ -36,8 +36,8 @@ String InterpreterShowTablesQuery::getRewrittenQuery() if (query.temporary && !query.from.empty()) throw Exception("The `FROM` and `TEMPORARY` cannot be used together in `SHOW TABLES`", ErrorCodes::SYNTAX_ERROR); - String database = query.from.empty() ? context.getCurrentDatabase() : query.from; - context.getDatabaseCatalog().assertDatabaseExists(database); + String database = context.resolveDatabase(query.from); + DatabaseCatalog::instance().assertDatabaseExists(database); std::stringstream rewritten_query; rewritten_query << "SELECT name FROM system."; diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 7c5043154af..6ac9da8f3d2 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -135,7 +136,7 @@ void startStopAction(Context & context, Poco::Logger * log, ASTSystemQuery & que } else { - for (auto & elem : context.getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases()) { for (auto iterator = elem.second->getTablesIterator(context); iterator->isValid(); iterator->next()) { @@ -312,7 +313,7 @@ BlockIO InterpreterSystemQuery::execute() StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_name, const String & table_name, Context & system_context) { context.checkAccess(AccessType::RESTART_REPLICA, database_name, table_name); - auto database = system_context.getDatabase(database_name); + auto database = DatabaseCatalog::instance().getDatabase(database_name, system_context); auto table_ddl_guard = system_context.getDDLGuard(database_name, table_name); ASTPtr create_ast; @@ -361,7 +362,7 @@ void InterpreterSystemQuery::restartReplicas(Context & system_context) { std::vector> replica_names; - for (auto & elem : system_context.getDatabases()) + for (auto & elem : DatabaseCatalog::instance().getDatabases()) { DatabasePtr & database = elem.second; const String & database_name = elem.first; diff --git a/dbms/src/Interpreters/loadMetadata.cpp b/dbms/src/Interpreters/loadMetadata.cpp index 241385f97ec..53954faa2c0 100644 --- a/dbms/src/Interpreters/loadMetadata.cpp +++ b/dbms/src/Interpreters/loadMetadata.cpp @@ -140,7 +140,7 @@ void loadMetadataSystem(Context & context) Poco::File(global_path + "metadata/" SYSTEM_DATABASE).createDirectories(); auto system_database = std::make_shared(SYSTEM_DATABASE, global_path + "metadata/" SYSTEM_DATABASE "/", context); - context.addDatabase(SYSTEM_DATABASE, system_database); + DatabaseCatalog::instance().attachDatabase(SYSTEM_DATABASE, system_database); } } diff --git a/dbms/src/Interpreters/tests/create_query.cpp b/dbms/src/Interpreters/tests/create_query.cpp index e159afced58..20a0bfcb062 100644 --- a/dbms/src/Interpreters/tests/create_query.cpp +++ b/dbms/src/Interpreters/tests/create_query.cpp @@ -83,7 +83,7 @@ try context.setPath("./"); auto database = std::make_shared("test", "./metadata/test/", context); - context.addDatabase("test", database); + DatabaseCatalog::instance().attachDatabase("test", database); database->loadStoredObjects(context, false); context.setCurrentDatabase("test"); diff --git a/dbms/src/Interpreters/tests/expression_analyzer.cpp b/dbms/src/Interpreters/tests/expression_analyzer.cpp index 079a22620bc..ff7579506f4 100644 --- a/dbms/src/Interpreters/tests/expression_analyzer.cpp +++ b/dbms/src/Interpreters/tests/expression_analyzer.cpp @@ -100,7 +100,7 @@ int main() context.makeGlobalContext(); auto system_database = std::make_shared("system"); - context.addDatabase("system", system_database); + DatabaseCatalog::instance().attachDatabase("system", system_database); //context.setCurrentDatabase("system"); system_database->attachTable("one", StorageSystemOne::create("one")); system_database->attachTable("numbers", StorageSystemNumbers::create("numbers", false)); diff --git a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp index a46afb2c8c3..017ca2e1739 100644 --- a/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp +++ b/dbms/src/Interpreters/tests/in_join_subqueries_preprocessor.cpp @@ -1166,7 +1166,7 @@ TestResult check(const TestEntry & entry) auto storage_distributed_hits = StorageDistributedFake::create("distant_db", "distant_hits", entry.shard_count); DB::DatabasePtr database = std::make_shared("test", "./metadata/test/", context); - context.addDatabase("test", database); + DB::DatabaseCatalog::instance().attachDatabase("test", database); database->attachTable("visits_all", storage_distributed_visits); database->attachTable("hits_all", storage_distributed_hits); context.setCurrentDatabase("test"); @@ -1209,12 +1209,12 @@ TestResult check(const TestEntry & entry) bool res = equals(ast_input, ast_expected); std::string output = DB::queryToString(ast_input); - context.detachDatabase("test"); + DB::DatabaseCatalog::instance().detachDatabase("test"); return TestResult(res, output); } catch (DB::Exception & e) { - context.detachDatabase("test"); + DB::DatabaseCatalog::instance().detachDatabase("test"); return TestResult(false, e.displayText()); } } diff --git a/dbms/src/Interpreters/tests/select_query.cpp b/dbms/src/Interpreters/tests/select_query.cpp index 197d1c55faf..2eb9ee92da2 100644 --- a/dbms/src/Interpreters/tests/select_query.cpp +++ b/dbms/src/Interpreters/tests/select_query.cpp @@ -38,9 +38,9 @@ try loadMetadata(context); DatabasePtr system = std::make_shared("system", "./metadata/system/", context); - context.addDatabase("system", system); + DatabaseCatalog::instance().attachDatabase("system", system); system->loadStoredObjects(context, false); - attachSystemTablesLocal(*context.getDatabase("system")); + attachSystemTablesLocal(*DatabaseCatalog::instance().getSystemDatabase()); context.setCurrentDatabase("default"); ReadBufferFromFileDescriptor in(STDIN_FILENO); diff --git a/dbms/src/Storages/IStorage.cpp b/dbms/src/Storages/IStorage.cpp index 332ecbc8681..4c833601ca9 100644 --- a/dbms/src/Storages/IStorage.cpp +++ b/dbms/src/Storages/IStorage.cpp @@ -387,7 +387,7 @@ void IStorage::alter( auto table_id = getStorageID(); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index 2f1b633413d..58cd66e4ee3 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -765,7 +765,7 @@ void StorageBuffer::alter(const AlterCommands & params, const Context & context, StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageDistributed.cpp b/dbms/src/Storages/StorageDistributed.cpp index 3dabc4aa701..0782176f158 100644 --- a/dbms/src/Storages/StorageDistributed.cpp +++ b/dbms/src/Storages/StorageDistributed.cpp @@ -469,7 +469,7 @@ void StorageDistributed::alter(const AlterCommands & params, const Context & con checkAlterIsPossible(params, context.getSettingsRef()); StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageMaterializedView.cpp b/dbms/src/Storages/StorageMaterializedView.cpp index 2ace03e154b..4c47a1d4691 100644 --- a/dbms/src/Storages/StorageMaterializedView.cpp +++ b/dbms/src/Storages/StorageMaterializedView.cpp @@ -294,7 +294,7 @@ void StorageMaterializedView::alter( } /// end modify query - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index 1455556f366..15c90106824 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -399,7 +399,7 @@ StorageMerge::StorageListWithLocks StorageMerge::getSelectedTables(const ASTPtr DatabaseTablesIteratorPtr StorageMerge::getDatabaseIterator(const Context & context) const { checkStackSize(); - auto database = context.getDatabase(source_database); + auto database = DatabaseCatalog::instance().getDatabase(source_database, context); auto table_name_match = [this](const String & table_name_) { return table_name_regexp.match(table_name_); }; return database->getTablesIterator(global_context, table_name_match); } @@ -425,7 +425,7 @@ void StorageMerge::alter( StorageInMemoryMetadata storage_metadata = getInMemoryMetadata(); params.apply(storage_metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, storage_metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, storage_metadata); setColumns(storage_metadata.columns); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index d8b25627a7e..f9131ac3a23 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -272,7 +272,7 @@ void StorageMergeTree::alter( { lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); update_metadata(); } @@ -289,7 +289,7 @@ void StorageMergeTree::alter( lockStructureExclusively(table_lock_holder, context.getCurrentQueryId()); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); update_metadata(); diff --git a/dbms/src/Storages/StorageNull.cpp b/dbms/src/Storages/StorageNull.cpp index bbb620132c8..3de86079b9b 100644 --- a/dbms/src/Storages/StorageNull.cpp +++ b/dbms/src/Storages/StorageNull.cpp @@ -51,7 +51,7 @@ void StorageNull::alter( StorageInMemoryMetadata metadata = getInMemoryMetadata(); params.apply(metadata); - context.getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(context, table_id.table_name, metadata); setColumns(std::move(metadata.columns)); } diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index e3a38f54219..50041982943 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -541,7 +541,7 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column } auto table_id = getStorageID(); - global_context.getDatabase(table_id.database_name)->alterTable(global_context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(global_context, table_id.table_name, metadata); /// Even if the primary/sorting keys didn't change we must reinitialize it /// because primary key column types might have changed. @@ -3231,7 +3231,7 @@ void StorageReplicatedMergeTree::alter( changeSettings(metadata.settings_ast, table_lock_holder); - global_context.getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, metadata); return; } @@ -3309,7 +3309,7 @@ void StorageReplicatedMergeTree::alter( auto old_metadata = getInMemoryMetadata(); old_metadata.settings_ast = metadata.settings_ast; changeSettings(metadata.settings_ast, table_lock_holder); - global_context.getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, old_metadata); + DatabaseCatalog::instance().getDatabase(table_id.database_name)->alterTable(query_context, table_id.table_name, old_metadata); } /// Modify shared metadata nodes in ZooKeeper. diff --git a/dbms/src/Storages/System/StorageSystemColumns.cpp b/dbms/src/Storages/System/StorageSystemColumns.cpp index 0fc85898264..6c568e1cba3 100644 --- a/dbms/src/Storages/System/StorageSystemColumns.cpp +++ b/dbms/src/Storages/System/StorageSystemColumns.cpp @@ -267,7 +267,7 @@ Pipes StorageSystemColumns::readWithProcessors( Pipes pipes; { - Databases databases = context.getDatabases(); + Databases databases = DatabaseCatalog::instance().getDatabases(); /// Add `database` column. MutableColumnPtr database_column_mut = ColumnString::create(); diff --git a/dbms/src/Storages/System/StorageSystemDatabases.cpp b/dbms/src/Storages/System/StorageSystemDatabases.cpp index ee193548921..4588fd28482 100644 --- a/dbms/src/Storages/System/StorageSystemDatabases.cpp +++ b/dbms/src/Storages/System/StorageSystemDatabases.cpp @@ -23,7 +23,7 @@ void StorageSystemDatabases::fillData(MutableColumns & res_columns, const Contex const auto access_rights = context.getAccessRights(); const bool check_access_for_databases = !access_rights->isGranted(AccessType::SHOW); - auto databases = context.getDatabases(); + auto databases = DatabaseCatalog::instance().getDatabases(); for (const auto & database : databases) { if (check_access_for_databases && !access_rights->isGranted(AccessType::SHOW, database.first)) diff --git a/dbms/src/Storages/System/StorageSystemGraphite.cpp b/dbms/src/Storages/System/StorageSystemGraphite.cpp index b58762b3860..b5023cff7c3 100644 --- a/dbms/src/Storages/System/StorageSystemGraphite.cpp +++ b/dbms/src/Storages/System/StorageSystemGraphite.cpp @@ -27,7 +27,7 @@ NamesAndTypesList StorageSystemGraphite::getNamesAndTypes() */ StorageSystemGraphite::Configs StorageSystemGraphite::getConfigs(const Context & context) const { - const Databases databases = context.getDatabases(); + const Databases databases = DatabaseCatalog::instance().getDatabases(); Configs graphite_configs; for (const auto & db : databases) diff --git a/dbms/src/Storages/System/StorageSystemMutations.cpp b/dbms/src/Storages/System/StorageSystemMutations.cpp index 279a553d5f9..51c5bd47c6d 100644 --- a/dbms/src/Storages/System/StorageSystemMutations.cpp +++ b/dbms/src/Storages/System/StorageSystemMutations.cpp @@ -42,7 +42,7 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, const Contex /// Collect a set of *MergeTree tables. std::map> merge_tree_tables; - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) { /// Lazy database can not contain MergeTree tables if (db.second->getEngineName() == "Lazy") diff --git a/dbms/src/Storages/System/StorageSystemPartsBase.cpp b/dbms/src/Storages/System/StorageSystemPartsBase.cpp index 2fdb28d62e1..a393d862b54 100644 --- a/dbms/src/Storages/System/StorageSystemPartsBase.cpp +++ b/dbms/src/Storages/System/StorageSystemPartsBase.cpp @@ -76,7 +76,7 @@ StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, const const bool check_access_for_tables = !access_rights->isGranted(AccessType::SHOW); { - Databases databases = context.getDatabases(); + Databases databases = DatabaseCatalog::instance().getDatabases(); /// Add column 'database'. MutableColumnPtr database_column_mut = ColumnString::create(); diff --git a/dbms/src/Storages/System/StorageSystemReplicas.cpp b/dbms/src/Storages/System/StorageSystemReplicas.cpp index fb3f117cfca..5e86dbc4e1d 100644 --- a/dbms/src/Storages/System/StorageSystemReplicas.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicas.cpp @@ -69,7 +69,7 @@ Pipes StorageSystemReplicas::readWithProcessors( /// We collect a set of replicated tables. std::map> replicated_tables; - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) { /// Lazy database can not contain replicated tables if (db.second->getEngineName() == "Lazy") diff --git a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp index 68627987bf4..50bc9f8cb62 100644 --- a/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp +++ b/dbms/src/Storages/System/StorageSystemReplicationQueue.cpp @@ -53,7 +53,7 @@ void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, const const bool check_access_for_databases = !access_rights->isGranted(AccessType::SHOW); std::map> replicated_tables; - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) { /// Lazy database can not contain replicated tables if (db.second->getEngineName() == "Lazy") diff --git a/dbms/src/Storages/System/StorageSystemTables.cpp b/dbms/src/Storages/System/StorageSystemTables.cpp index 5b39a0f0bf7..5b2eca1ec01 100644 --- a/dbms/src/Storages/System/StorageSystemTables.cpp +++ b/dbms/src/Storages/System/StorageSystemTables.cpp @@ -57,7 +57,7 @@ StorageSystemTables::StorageSystemTables(const std::string & name_) static ColumnPtr getFilteredDatabases(const ASTPtr & query, const Context & context) { MutableColumnPtr column = ColumnString::create(); - for (const auto & db : context.getDatabases()) + for (const auto & db : DatabaseCatalog::instance().getDatabases()) column->insert(db.first); Block block { ColumnWithTypeAndName(std::move(column), std::make_shared(), "database") }; @@ -118,7 +118,8 @@ protected: while (database_idx < databases->size() && (!tables_it || !tables_it->isValid())) { database_name = databases->getDataAt(database_idx).toString(); - database = context.tryGetDatabase(database_name); + //FIXME access is not checked + database = DatabaseCatalog::instance().tryGetDatabase(database_name); if (!database) { diff --git a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp index 57729218a40..856de3d89ca 100644 --- a/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/dbms/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -32,7 +32,8 @@ struct State registerFunctions(); DatabasePtr database = std::make_shared("test"); database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); - context.addDatabase("test", database); + context.makeGlobalContext(); + DatabaseCatalog::instance().attachDatabase("test", database); context.setCurrentDatabase("test"); } }; diff --git a/dbms/src/TableFunctions/TableFunctionMerge.cpp b/dbms/src/TableFunctions/TableFunctionMerge.cpp index b0c22c96117..eedc6dda10a 100644 --- a/dbms/src/TableFunctions/TableFunctionMerge.cpp +++ b/dbms/src/TableFunctions/TableFunctionMerge.cpp @@ -31,7 +31,7 @@ static NamesAndTypesList chooseColumns(const String & source_database, const Str StoragePtr any_table; { - auto database = context.getDatabase(source_database); + auto database = DatabaseCatalog::instance().getDatabase(source_database); auto iterator = database->getTablesIterator(context, table_name_match); if (iterator->isValid()) From 260a4687f05030a6f7505752705288496c6b7191 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 10 Feb 2020 19:55:09 +0300 Subject: [PATCH 0233/2007] Something working --- .../ReplicatedMergeTreeBlockOutputStream.cpp | 1 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 33 ++-- .../MergeTree/ReplicatedQueueAlterState.h | 150 ++++++------------ .../Storages/StorageReplicatedMergeTree.cpp | 28 ++-- 4 files changed, 88 insertions(+), 124 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index 0442d964e5e..bddf8434e1c 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -256,6 +256,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/columns", storage.getMetadataVersion())); + ops.emplace_back(zkutil::makeSetRequest(storage.zookeeper_path + "/alter_intention_counter", "", -1)); /// Deletes the information that the block number is used for writing. block_number_lock->getUnlockOps(ops); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 7c7ab2644f4..8bfa974ce45 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -167,7 +167,6 @@ void ReplicatedMergeTreeQueue::insertUnlocked( LOG_DEBUG(log, "ADDING DATA ENTRY WITH ALTER VERSION:" << entry->alter_version << " FOR PART:" << entry->source_parts[0] << " to " << entry->getVirtualPartNames()[0]); //LOG_DEBUG(log, "ADDING DATA ENTRY WITH ALTER VERSION:" << entry->alter_version); //std::cerr << "INSERT MUTATE PART:" << entry->alter_version << std::endl; - alter_sequence.addDataAlterIfEmpty(entry->alter_version, state_lock); } } @@ -246,10 +245,10 @@ void ReplicatedMergeTreeQueue::updateStateOnQueueEntryRemoval( if (entry->type == LogEntry::ALTER_METADATA) { + LOG_DEBUG(log, "FIN ALTER FOR PART with ALTER VERSION:" << entry->alter_version); //std::cerr << "Alter have mutation:" << entry->have_mutation << std::endl; alter_sequence.finishMetadataAlter(entry->alter_version, entry->have_mutation, state_lock); - LOG_DEBUG(log, "FIN ALTER FOR PART with ALTER VERSION:" << entry->alter_version); } if (entry->type == LogEntry::MUTATE_PART) { @@ -726,6 +725,9 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C if (mutation.parts_to_do.size() == 0) some_mutations_are_probably_done = true; + + if (entry->alter_version != -1) + alter_sequence.addMutationForAlter(entry->alter_version, entry->block_numbers, state_lock); } } @@ -987,6 +989,12 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( MergeTreeData & data, std::lock_guard & state_lock) const { + if (entry.type == LogEntry::GET_PART) + { + if (!alter_sequence.canExecuteGetEntry(entry.new_part_name, format_version, state_lock)) + return false; + } + if (entry.type == LogEntry::MERGE_PARTS || entry.type == LogEntry::GET_PART || entry.type == LogEntry::MUTATE_PART) @@ -1074,20 +1082,23 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { //std::cerr << "Should we execute alter:"; - LOG_DEBUG(log, "Should we execute alter entry:" << entry.toString()); + LOG_DEBUG(log, "Should we execute alter entry:" << entry.znode_name << "\n"<< entry.toString()); LOG_DEBUG(log, "We are in front:" << (entry.znode_name == *entries_in_queue.begin())); + for (auto & log_entry : entries_in_queue) { LOG_DEBUG(log, "LogEntry:" << log_entry); } + for (auto & log_entry : queue) + { + LOG_DEBUG(log, "LogEntryData:" << log_entry->znode_name << "\n" << log_entry->toString()); + } //std::cerr << alter_sequence.canExecuteMetadataAlter(entry.alter_version, state_lock) << std::endl; - if (!alter_sequence.canExecuteMetadataAlter(entry.alter_version, state_lock) || *entries_in_queue.begin() != entry.znode_name) + if (*entries_in_queue.begin() != entry.znode_name || !alter_sequence.canExecuteMetaAlter(entry.alter_version, state_lock)) { - LOG_DEBUG(log, "No we shouldn't"); - out_postpone_reason = "Cannot execute alter metadata with version: " + std::to_string(entry.alter_version) - + " because current head is " + std::to_string(alter_sequence.queue.front().alter_version) - + " with state: " + std::to_string(alter_sequence.queue.front().state); + out_postpone_reason + = "Cannot execute alter metadata with because head smallest node is " + *entries_in_queue.begin() + " but we are " + entry.znode_name; return false; } else @@ -1101,13 +1112,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( //std::cerr << "Should we execute mutation:"; //std::cerr << alter_sequence.canExecuteDataAlter(entry.alter_version, state_lock) << std::endl; - LOG_DEBUG(log, "Should we execute mutation entry:" << entry.toString()); + LOG_DEBUG(log, "Should we execute mutation entry:" << entry.znode_name << "\n" << entry.toString()); if (!alter_sequence.canExecuteDataAlter(entry.alter_version, state_lock)) { LOG_DEBUG(log, "NOOOO"); - out_postpone_reason = "Cannot execute alter data with version: " + std::to_string(entry.alter_version) - + " because current head is " + std::to_string(alter_sequence.queue.front().alter_version) - + " with state: " + std::to_string(alter_sequence.queue.front().state); + out_postpone_reason = "Cannot execute alter data with version: " + std::to_string(entry.alter_version); return false; } else diff --git a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h index a6a4b44da83..b359ce0dc14 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h +++ b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h @@ -4,6 +4,7 @@ #include #include #include +#include namespace DB { @@ -16,22 +17,16 @@ namespace ErrorCodes class AlterSequence { private: - enum AlterState - { - APPLY_METADATA_CHANGES, - APPLY_DATA_CHANGES, - DATA_CHANGES_NOT_NEEDED, - }; - struct AlterInQueue { - int alter_version; - AlterState state; std::map block_numbers; + bool metadata_finished = false; - AlterInQueue(int alter_version_, AlterState state_) - : alter_version(alter_version_) - , state(state_) + AlterInQueue() = default; + + AlterInQueue(const std::map & block_numbers_, bool metadata_finished_) + : block_numbers(block_numbers_) + , metadata_finished(metadata_finished_) { } }; @@ -43,65 +38,60 @@ public: : log(log_) { } - std::deque queue; + std::map queue_state; bool empty() const { - return queue.empty(); + return queue_state.empty(); + } + + void addMutationForAlter(int alter_version, const std::map & block_numbers, std::lock_guard & /*state_lock*/) + { + LOG_DEBUG(log, "Adding mutation with alter version:" << alter_version); + queue_state.emplace(alter_version, AlterInQueue(block_numbers, true)); } void addMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) { - if (!queue.empty() && queue.front().alter_version > alter_version) - { - throw Exception("Alter not in order " + std::to_string(alter_version), ErrorCodes::LOGICAL_ERROR); - } - queue.emplace_back(alter_version, AlterState::APPLY_METADATA_CHANGES); + LOG_DEBUG(log, "Adding meta with alter version:" << alter_version); + if (!queue_state.count(alter_version)) + queue_state.emplace(alter_version, AlterInQueue({}, false)); + else + queue_state[alter_version].metadata_finished = false; } - void addDataAlterIfEmpty(int alter_version, std::lock_guard & /*state_lock*/) + bool canExecuteGetEntry(const String & part_name, MergeTreeDataFormatVersion format_version, std::lock_guard & /*state_lock*/) const { - if (queue.empty()) - queue.emplace_back(alter_version, AlterState::APPLY_DATA_CHANGES); + if (empty()) + return true; + + MergeTreePartInfo info = MergeTreePartInfo::fromPartName(part_name, format_version); + if (queue_state.begin()->second.block_numbers.count(info.partition_id)) + return info.getDataVersion() < queue_state.begin()->second.block_numbers.at(info.partition_id); + return true; - //if (queue.front().alter_version != alter_version) - //{ - // throw Exception( - // "Alter head has another version number " - // + std::to_string(queue.front().alter_version) + " than ours " + std::to_string(alter_version), - // ErrorCodes::LOGICAL_ERROR); - //} } void finishMetadataAlter(int alter_version, bool have_data_alter, std::unique_lock & /*state_lock*/) { - if (queue.empty()) + if (queue_state.empty()) { throw Exception("Queue shouldn't be empty on metadata alter", ErrorCodes::LOGICAL_ERROR); } - LOG_DEBUG( - log, - "FINISHING METADATA ALTER WITH VERSION:" << alter_version << " AND HAVE DATA ALTER: " << have_data_alter - << " QUEUE HEAD:" << queue.front().alter_version << " state:" << queue.front().state); - if (queue.front().alter_version != alter_version) - throw Exception("Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue.front().alter_version), ErrorCodes::LOGICAL_ERROR); - - if (have_data_alter && queue.front().state == AlterState::APPLY_METADATA_CHANGES) + if (queue_state.begin()->first != alter_version) { - LOG_DEBUG( - log, - "FINISHING METADATA ALTER WITH VERSION:" << alter_version << " AND SWITCHING QUEUE STATE"); + LOG_DEBUG(log, "Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue_state.begin()->first)); + throw Exception("Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue_state.begin()->first), ErrorCodes::LOGICAL_ERROR); + } - //std::cerr << "Switching head state:" << AlterState::APPLY_DATA_CHANGES << std::endl; - queue.front().state = AlterState::APPLY_DATA_CHANGES; + if (!have_data_alter) + { + queue_state.erase(alter_version); } else { - LOG_DEBUG(log, "FINISHING METADATA ALTER WITH VERSION:" << alter_version << " AND DOING POP"); - - //std::cerr << "JUST POP FRONT\n"; - queue.pop_front(); + queue_state[alter_version].metadata_finished = true; } } @@ -109,69 +99,29 @@ public: { /// queue can be empty after load of finished mutation without move of mutation pointer - if (queue.empty()) + if (queue_state.empty()) { LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE EMPTY"); return; } - //std::cerr << "Finishing data alter:" << alter_version << std::endl; - if (queue.front().alter_version != alter_version) - { - for (auto & state : queue) - { - if (state.alter_version == alter_version) - { - LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT HEAD IS NOT SAME SO MAKE DATA_CHANGED_NOT_NEEDED"); - state.state = AlterState::DATA_CHANGES_NOT_NEEDED; - return; - } - } - } - //if (queue.front().alter_version != alter_version) - //{ - // LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE VERSION IS " << queue.front().alter_version << " state " << queue.front().state); - // throw Exception( - // "Finished data alter with version " + std::to_string(alter_version) + " but current alter in queue is " - // + std::to_string(queue.front().alter_version), - // ErrorCodes::LOGICAL_ERROR); - //} - if (queue.front().state != AlterState::APPLY_DATA_CHANGES) - { - LOG_DEBUG( - log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT STATE IS METADATA"); - queue.front().state = AlterState::DATA_CHANGES_NOT_NEEDED; - return; - } - - LOG_DEBUG( - log, - "FINISHING DATA ALTER WITH VERSION:" << alter_version << " QUEUE VERSION IS " << queue.front().alter_version << " STATE " - << queue.front().state); - queue.pop_front(); - } - - bool canExecuteMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) const - { - if (queue.empty()) - throw Exception("QUEUE EMPTY ON METADATA", ErrorCodes::LOGICAL_ERROR); - LOG_DEBUG(log, "CHECK METADATADATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE HEAD IS " << queue.front().alter_version); - - //std::cerr << "Alter queue front:" << queue.front().alter_version << " state:" << queue.front().state << std::endl; - return queue.front().alter_version == alter_version; + LOG_DEBUG(log, "FINISH DATA ALTER: " << alter_version); + queue_state.erase(alter_version); } bool canExecuteDataAlter(int alter_version, std::lock_guard & /*state_lock*/) const { - if (queue.empty()) - throw Exception("QUEUE EMPTY ON DATA", ErrorCodes::LOGICAL_ERROR); - //std::cerr << "Alter queue front:" << queue.front().alter_version << " state:" << queue.front().state << std::endl; - //std::cerr << "CAn execute:" << alter_version << std::endl; - //std::cerr << "FRont version:" << queue.front().alter_version << std::endl; - //std::cerr << "State:" << queue.front().state << std::endl; - LOG_DEBUG(log, "CHECK DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE HEAD IS " << queue.front().alter_version << " state:" << queue.front().state); - return queue.front().alter_version == alter_version && queue.front().state == AlterState::APPLY_DATA_CHANGES; + LOG_DEBUG(log, "Can execute data alter:" << alter_version); + for (auto [key, value] : queue_state) + { + LOG_DEBUG(log, "Key:" << key << " is metadata finished:" << value.metadata_finished); + } + return queue_state.at(alter_version).metadata_finished; + } + bool canExecuteMetaAlter(int alter_version, std::lock_guard & /*state_lock*/) const + { + return queue_state.empty() || queue_state.begin()->first == alter_version; } }; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index d732d569b99..cc1311a8760 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -397,6 +397,7 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() zookeeper->createIfNotExists(zookeeper_path + "/quorum", String()); zookeeper->createIfNotExists(zookeeper_path + "/quorum/last_part", String()); zookeeper->createIfNotExists(zookeeper_path + "/quorum/failed_parts", String()); + zookeeper->createIfNotExists(zookeeper_path + "/alter_intention_counter", String()); /// Tracking lag of replicas. zookeeper->createIfNotExists(replica_path + "/min_unprocessed_insert_time", String()); @@ -3290,7 +3291,7 @@ void StorageReplicatedMergeTree::alter( { assertNotReadonly(); - LOG_DEBUG(log, "Doing ALTER"); + LOG_DEBUG(log, "Doing ALTER FROM " << metadata_version); auto maybe_mutation_commands = params.getMutationCommands(getInMemoryMetadata()); auto table_id = getStorageID(); @@ -3394,7 +3395,6 @@ void StorageReplicatedMergeTree::alter( bool have_mutation = false; std::optional lock_holder; - size_t partitions_count = 0; if (!maybe_mutation_commands.empty()) { String mutations_path = zookeeper_path + "/mutations"; @@ -3406,24 +3406,20 @@ void StorageReplicatedMergeTree::alter( Coordination::Stat mutations_stat; zookeeper->get(mutations_path, &mutations_stat); + Coordination::Stat intention_counter_stat; + zookeeper->get(zookeeper_path + "/alter_intention_counter", &intention_counter_stat); lock_holder.emplace( - zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); + zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); - Coordination::Stat block_numbers_version; - zookeeper->get(zookeeper_path + "/block_numbers", &block_numbers_version); + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/alter_intention_counter", intention_counter_stat.version)); for (const auto & lock : lock_holder->getLocks()) { - Coordination::Stat partition_stat; mutation_entry.block_numbers[lock.partition_id] = lock.number; - zookeeper->get(zookeeper_path + "/block_numbers/" + lock.partition_id, &partition_stat); - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/block_numbers/" + lock.partition_id, partition_stat.version)); - partitions_count++; + LOG_DEBUG(log, "ALLOCATED:" << lock.number << " FOR VERSION:" << metadata_version + 1); } mutation_entry.create_time = time(nullptr); - /// We have to be sure, that no inserts happened - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/block_numbers", block_numbers_version.version)); ops.emplace_back(zkutil::makeSetRequest(mutations_path, String(), mutations_stat.version)); ops.emplace_back( @@ -3435,6 +3431,8 @@ void StorageReplicatedMergeTree::alter( Coordination::Responses results; int32_t rc = zookeeper->tryMulti(ops, results); + LOG_DEBUG(log, "ALTER REQUESTED TO" << entry.alter_version); + //std::cerr << "Results size:" << results.size() << std::endl; //std::cerr << "Have mutation:" << have_mutation << std::endl; @@ -3445,7 +3443,7 @@ void StorageReplicatedMergeTree::alter( { //std::cerr << "In have mutation\n"; //std::cerr << "INDEX:" << results.size() - 2 << std::endl; - String alter_path = dynamic_cast(*results[results.size() - 4 - partitions_count]).path_created; + String alter_path = dynamic_cast(*results[results.size() - 4]).path_created; //std::cerr << "Alter path:" << alter_path << std::endl; entry.znode_name = alter_path.substr(alter_path.find_last_of('/') + 1); @@ -3474,7 +3472,11 @@ void StorageReplicatedMergeTree::alter( std::vector unwaited; //std::cerr << "Started wait for alter\n"; if (query_context.getSettingsRef().replication_alter_partitions_sync == 2) + { + LOG_DEBUG(log, "Start waiting for metadata alter"); unwaited = waitForAllReplicasToProcessLogEntry(entry, false); + LOG_DEBUG(log, "Finished waiting for metadata alter"); + } else if (query_context.getSettingsRef().replication_alter_partitions_sync == 1) waitForReplicaToProcessLogEntry(replica_name, entry); //std::cerr << "FInished wait for alter\n"; @@ -3487,7 +3489,9 @@ void StorageReplicatedMergeTree::alter( if (mutation_znode) { //std::cerr << "Started wait for mutation:" << *mutation_znode << std::endl; + LOG_DEBUG(log, "Start waiting for mutation"); waitMutation(*mutation_znode, query_context.getSettingsRef().replication_alter_partitions_sync); + LOG_DEBUG(log, "Finished waiting for mutation"); //std::cerr << "FInished wait for mutation\n"; } } From 45338ad9ca8fab5bad1e92de6d9717822cb4feb9 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 21:19:35 +0300 Subject: [PATCH 0234/2007] fixes --- dbms/src/Interpreters/Context.cpp | 39 ++++++++++++------- dbms/src/Interpreters/Context.h | 21 +++++----- dbms/src/Interpreters/DatabaseCatalog.cpp | 20 +++------- dbms/src/Interpreters/DatabaseCatalog.h | 3 +- .../src/Interpreters/InterpreterDropQuery.cpp | 1 + 5 files changed, 45 insertions(+), 39 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index ba7de8f877d..b57442d19e5 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -95,7 +95,6 @@ namespace ErrorCodes extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int SESSION_NOT_FOUND; extern const int SESSION_IS_LOCKED; - extern const int CANNOT_GET_CREATE_TABLE_QUERY; extern const int LOGICAL_ERROR; extern const int SCALAR_ALREADY_EXISTS; extern const int UNKNOWN_SCALAR; @@ -765,7 +764,7 @@ Dependencies Context::getDependencies(const StorageID & from) const { auto lock = getLock(); StorageID resolved = resolveStorageIDUnlocked(from); - ViewDependencies::const_iterator iter = shared->view_dependencies.find(resolved); + auto iter = shared->view_dependencies.find(resolved); if (iter == shared->view_dependencies.end()) return {}; @@ -774,8 +773,7 @@ Dependencies Context::getDependencies(const StorageID & from) const bool Context::isTableExist(const String & database_name, const String & table_name) const { - //FIXME do we need resolve temporary tables here? - auto table_id = resolveStorageID({database_name, table_name}); + auto table_id = resolveStorageID({database_name, table_name}, StorageNamespace::Ordinary); return DatabaseCatalog::instance().isTableExist(table_id, *this); } @@ -2046,30 +2044,45 @@ void Context::resetInputCallbacks() input_blocks_reader = {}; } -StorageID Context::resolveStorageID(StorageID storage_id) const +StorageID Context::resolveStorageID(StorageID storage_id, StorageNamespace where) const { auto lock = getLock(); - return resolveStorageIDUnlocked(std::move(storage_id)); + return resolveStorageIDUnlocked(std::move(storage_id), where); } -StorageID Context::resolveStorageIDUnlocked(StorageID storage_id) const +StorageID Context::resolveStorageIDUnlocked(StorageID storage_id, StorageNamespace where) const { if (storage_id.uuid != UUIDHelpers::Nil) - { - //TODO maybe update table and db name? - //TODO add flag `resolved` to StorageID and check access rights if it was not previously resolved return storage_id; + + bool look_for_external_table = where & StorageNamespace::External; + bool in_current_database = where & StorageNamespace::CurrentDatabase; + bool in_specified_database = where & StorageNamespace::Global; + + if (!storage_id.database_name.empty()) + { + if (in_specified_database) + return storage_id; + throw Exception("External and temporary tables have no database, but " + + storage_id.database_name + " is specified", ErrorCodes::UNKNOWN_TABLE); } - if (storage_id.database_name.empty()) + + if (look_for_external_table) { auto it = external_tables_mapping.find(storage_id.getTableName()); if (it != external_tables_mapping.end()) - return it->second->getGlobalTableID(); /// Do not check access rights for session-local table + return it->second->getGlobalTableID(); + } + + if (in_current_database) + { if (current_database.empty()) throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); storage_id.database_name = current_database; + return storage_id; } - return storage_id; + + throw Exception("Cannot resolve database name for table " + storage_id.getNameForLogs(), ErrorCodes::UNKNOWN_TABLE); } diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 3ee96829e53..5d4a37bfdb6 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -145,8 +145,6 @@ struct SubscriptionForUserChange }; struct TemporaryTableHolder; -class DatabaseCatalog; -using DatabaseCatalogPtr = std::shared_ptr; /** A set of known objects that can be used in the query. * Consists of a shared part (always common to all sessions and queries) @@ -197,9 +195,6 @@ private: using SampleBlockCache = std::unordered_map; mutable SampleBlockCache sample_block_cache; - using DatabasePtr = std::shared_ptr; - using Databases = std::map>; - NameToNameMap query_parameters; /// Dictionary with query parameters for prepared statements. /// (key=name, value) @@ -306,11 +301,19 @@ public: bool isDictionaryExists(const String & database_name, const String & dictionary_name) const; bool isExternalTableExist(const String & table_name) const; + enum StorageNamespace + { + Global = 1u, /// Database name must be specified + CurrentDatabase = 2u, /// Use current database + Ordinary = Global | CurrentDatabase, /// If database name is not specified, use current database + External = 4u, /// Try get external table + All = External | Ordinary /// If database name is not specified, try get external table, + /// if external table not found use current database. + }; + String resolveDatabase(const String & database_name) const; - String resolveDatabaseAndCheckAccess(const String & database_name) const; - //StorageID resolveDatabase(StorageID table_id) const; - StorageID resolveStorageID(StorageID storage_id) const; - StorageID resolveStorageIDUnlocked(StorageID storage_id) const; + StorageID resolveStorageID(StorageID storage_id, StorageNamespace where = StorageNamespace::All) const; + StorageID resolveStorageIDUnlocked(StorageID storage_id, StorageNamespace where = StorageNamespace::All) const; const Scalars & getScalars() const; const Block & getScalar(const String & name) const; diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 59e4368c381..e1127efb965 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -65,13 +65,6 @@ DatabaseAndTable DatabaseCatalog::tryGetByUUID(const UUID & uuid) const return it->second; } -//String DatabaseCatalog::resolveDatabase(const String & database_name, const String & current_database) -//{ -// String res = database_name.empty() ? current_database : database_name; -// if (res.empty()) -// throw Exception("Default database is not selected", ErrorCodes::UNKNOWN_DATABASE); -// return res; -//} StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & local_context, std::optional * exception) const { @@ -121,6 +114,7 @@ void DatabaseCatalog::assertDatabaseDoesntExist(const String & database_name) co void DatabaseCatalog::assertDatabaseExistsUnlocked(const String & database_name) const { + assert(!database_name.empty()); if (databases.end() == databases.find(database_name)) throw Exception("Database " + backQuoteIfNeed(database_name) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); } @@ -128,13 +122,13 @@ void DatabaseCatalog::assertDatabaseExistsUnlocked(const String & database_name) void DatabaseCatalog::assertDatabaseDoesntExistUnlocked(const String & database_name) const { + assert(!database_name.empty()); if (databases.end() != databases.find(database_name)) throw Exception("Database " + backQuoteIfNeed(database_name) + " already exists.", ErrorCodes::DATABASE_ALREADY_EXISTS); } void DatabaseCatalog::attachDatabase(const String & database_name, const DatabasePtr & database) { - //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; assertDatabaseDoesntExistUnlocked(database_name); databases[database_name] = database; @@ -143,18 +137,15 @@ void DatabaseCatalog::attachDatabase(const String & database_name, const Databas DatabasePtr DatabaseCatalog::detachDatabase(const String & database_name) { - //local_context.checkDatabaseAccessRights(database_name); std::lock_guard lock{databases_mutex}; - auto res = getDatabase(database_name); //FIXME locks order + assertDatabaseExistsUnlocked(database_name); + auto res = databases.find(database_name)->second; databases.erase(database_name); return res; } DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const { - assert(!database_name.empty()); - //String db = local_context.resolveDatabase(database_name); - //local_context.checkDatabaseAccessRights(db); //FIXME non-atomic std::lock_guard lock{databases_mutex}; assertDatabaseExistsUnlocked(database_name); return databases.find(database_name)->second; @@ -163,7 +154,6 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name) const DatabasePtr DatabaseCatalog::tryGetDatabase(const String & database_name) const { assert(!database_name.empty()); - //String db = local_context.resolveDatabase(database_name); std::lock_guard lock{databases_mutex}; auto it = databases.find(database_name); if (it == databases.end()) @@ -198,7 +188,7 @@ bool DatabaseCatalog::isTableExist(const DB::StorageID & table_id, const DB::Con void DatabaseCatalog::assertTableDoesntExist(const StorageID & table_id, const Context & context) const { - if (!isTableExist(table_id, context)) + if (isTableExist(table_id, context)) throw Exception("Table " + table_id.getNameForLogs() + " already exists.", ErrorCodes::TABLE_ALREADY_EXISTS); } diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index b5cae14e699..70cbb6b8050 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -18,12 +18,11 @@ struct StorageID; class Exception; using DatabasePtr = std::shared_ptr; using DatabaseAndTable = std::pair; +using Databases = std::map>; -//TODO make singleton? class DatabaseCatalog : boost::noncopyable { public: - using Databases = std::map>; static constexpr const char * TEMPORARY_DATABASE = "_temporary_and_external_tables"; static constexpr const char * SYSTEM_DATABASE = "system"; diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 9f7ea306f10..9a92c0eee47 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -250,6 +250,7 @@ BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, AS context.checkAccess(AccessType::DETACH_DATABASE, database_name); DatabaseCatalog::instance().detachDatabase(database_name); database->shutdown(); + //FIXME someone may still use tables from database } else if (kind == ASTDropQuery::Kind::Drop) { From 18dd0c8f8cc7fe774e222234419edc364664ec86 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 10 Feb 2020 21:31:52 +0300 Subject: [PATCH 0235/2007] move DDLGuard to DatabaseCatalog --- dbms/src/Interpreters/Context.cpp | 38 ----------------- dbms/src/Interpreters/Context.h | 32 --------------- dbms/src/Interpreters/DatabaseCatalog.cpp | 28 +++++++++++++ dbms/src/Interpreters/DatabaseCatalog.h | 41 ++++++++++++++++++- .../Interpreters/InterpreterCreateQuery.cpp | 6 +-- .../src/Interpreters/InterpreterDropQuery.cpp | 6 +-- .../Interpreters/InterpreterRenameQuery.cpp | 5 ++- .../Interpreters/InterpreterSystemQuery.cpp | 2 +- 8 files changed, 78 insertions(+), 80 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index b57442d19e5..80108fcbc15 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -90,7 +90,6 @@ namespace ErrorCodes extern const int THERE_IS_NO_SESSION; extern const int THERE_IS_NO_QUERY; extern const int NO_ELEMENTS_IN_CONFIG; - extern const int DDL_GUARD_IS_ACTIVE; extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int PARTITION_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; extern const int SESSION_NOT_FOUND; @@ -253,16 +252,6 @@ struct ContextShared bool shutdown_called = false; - /// Do not allow simultaneous execution of DDL requests on the same table. - /// database -> object -> (mutex, counter), counter: how many threads are running a query on the table at the same time - /// For the duration of the operation, an element is placed here, and an object is returned, - /// which deletes the element in the destructor when counter becomes zero. - /// In case the element already exists, waits, when query will be executed in other thread. See class DDLGuard below. - using DDLGuards = std::unordered_map; - DDLGuards ddl_guards; - /// If you capture mutex and ddl_guards_mutex, then you need to grab them strictly in this order. - mutable std::mutex ddl_guards_mutex; - Stopwatch uptime_watch; Context::ApplicationType application_type = Context::ApplicationType::SERVER; @@ -931,33 +920,6 @@ StoragePtr Context::getViewSource() return view_source; } - -DDLGuard::DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem) - : map(map_), guards_lock(std::move(guards_lock_)) -{ - it = map.emplace(elem, Entry{std::make_unique(), 0}).first; - ++it->second.counter; - guards_lock.unlock(); - table_lock = std::unique_lock(*it->second.mutex); -} - -DDLGuard::~DDLGuard() -{ - guards_lock.lock(); - --it->second.counter; - if (!it->second.counter) - { - table_lock.unlock(); - map.erase(it); - } -} - -std::unique_ptr Context::getDDLGuard(const String & database, const String & table) const -{ - std::unique_lock lock(shared->ddl_guards_mutex); - return std::make_unique(shared->ddl_guards[database], std::move(lock), table); -} - ASTPtr Context::getCreateExternalTableQuery(const String & table_name) const { auto it = external_tables_mapping.find(table_name); diff --git a/dbms/src/Interpreters/Context.h b/dbms/src/Interpreters/Context.h index 5d4a37bfdb6..ceea7ad267b 100644 --- a/dbms/src/Interpreters/Context.h +++ b/dbms/src/Interpreters/Context.h @@ -76,7 +76,6 @@ class TraceLog; class MetricLog; struct MergeTreeSettings; class IDatabase; -class DDLGuard; class DDLWorker; class ITableFunction; class Block; @@ -333,9 +332,6 @@ public: void addViewSource(const StoragePtr & storage); StoragePtr getViewSource(); - /// Get an object that protects the table from concurrently executing multiple DDL operations. - std::unique_ptr getDDLGuard(const String & database, const String & table) const; - String getCurrentDatabase() const; String getCurrentQueryId() const; @@ -629,34 +625,6 @@ private: }; -/// Allows executing DDL query only in one thread. -/// Puts an element into the map, locks tables's mutex, counts how much threads run parallel query on the table, -/// when counter is 0 erases element in the destructor. -/// If the element already exists in the map, waits, when ddl query will be finished in other thread. -class DDLGuard -{ -public: - struct Entry - { - std::unique_ptr mutex; - UInt32 counter; - }; - - /// Element name -> (mutex, counter). - /// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion. - using Map = std::map; - - DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem); - ~DDLGuard(); - -private: - Map & map; - Map::iterator it; - std::unique_lock guards_lock; - std::unique_lock table_lock; -}; - - class SessionCleaner { public: diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index e1127efb965..98f7145c392 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -15,6 +15,7 @@ namespace ErrorCodes extern const int UNKNOWN_TABLE; extern const int TABLE_ALREADY_EXISTS; extern const int DATABASE_ALREADY_EXISTS; + extern const int DDL_GUARD_IS_ACTIVE; } @@ -234,6 +235,33 @@ DatabasePtr DatabaseCatalog::getDatabase(const String & database_name, const Con return getDatabase(resolved_database); } +std::unique_ptr DatabaseCatalog::getDDLGuard(const String & database, const String & table) +{ + std::unique_lock lock(ddl_guards_mutex); + return std::make_unique(ddl_guards[database], std::move(lock), table); +} + + +DDLGuard::DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem) + : map(map_), guards_lock(std::move(guards_lock_)) +{ + it = map.emplace(elem, Entry{std::make_unique(), 0}).first; + ++it->second.counter; + guards_lock.unlock(); + table_lock = std::unique_lock(*it->second.mutex); +} + +DDLGuard::~DDLGuard() +{ + guards_lock.lock(); + --it->second.counter; + if (!it->second.counter) + { + table_lock.unlock(); + map.erase(it); + } +} + } diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 70cbb6b8050..1a25800737a 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -20,6 +20,35 @@ using DatabasePtr = std::shared_ptr; using DatabaseAndTable = std::pair; using Databases = std::map>; + +/// Allows executing DDL query only in one thread. +/// Puts an element into the map, locks tables's mutex, counts how much threads run parallel query on the table, +/// when counter is 0 erases element in the destructor. +/// If the element already exists in the map, waits, when ddl query will be finished in other thread. +class DDLGuard +{ +public: + struct Entry + { + std::unique_ptr mutex; + UInt32 counter; + }; + + /// Element name -> (mutex, counter). + /// NOTE: using std::map here (and not std::unordered_map) to avoid iterator invalidation on insertion. + using Map = std::map; + + DDLGuard(Map & map_, std::unique_lock guards_lock_, const String & elem); + ~DDLGuard(); + +private: + Map & map; + Map::iterator it; + std::unique_lock guards_lock; + std::unique_lock table_lock; +}; + + class DatabaseCatalog : boost::noncopyable { public: @@ -34,6 +63,8 @@ public: void loadDatabases(); void shutdown(); + /// Get an object that protects the table from concurrently executing multiple DDL operations. + std::unique_ptr getDDLGuard(const String & database, const String & table); //static String resolveDatabase(const String & database_name, const String & current_database); void assertDatabaseExists(const String & database_name) const; @@ -88,7 +119,15 @@ private: UUIDToStorageMap uuid_map; + /// Do not allow simultaneous execution of DDL requests on the same table. + /// database -> object -> (mutex, counter), counter: how many threads are running a query on the table at the same time + /// For the duration of the operation, an element is placed here, and an object is returned, + /// which deletes the element in the destructor when counter becomes zero. + /// In case the element already exists, waits, when query will be executed in other thread. See class DDLGuard below. + using DDLGuards = std::unordered_map; + DDLGuards ddl_guards; + /// If you capture mutex and ddl_guards_mutex, then you need to grab them strictly in this order. + mutable std::mutex ddl_guards_mutex; }; - } diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index ab828fc7306..4aee64b0c42 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -83,7 +83,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) { String database_name = create.database; - auto guard = context.getDDLGuard(database_name, ""); + auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); /// Database can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard if (DatabaseCatalog::instance().isDatabaseExist(database_name)) @@ -595,7 +595,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create, /** If the request specifies IF NOT EXISTS, we allow concurrent CREATE queries (which do nothing). * If table doesnt exist, one thread is creating table, while others wait in DDLGuard. */ - guard = context.getDDLGuard(create.database, table_name); + guard = DatabaseCatalog::instance().getDDLGuard(create.database, table_name); /// Table can be created before or it can be created concurrently in another thread, while we were waiting in DDLGuard. if (database->isTableExist(context, table_name)) @@ -689,7 +689,7 @@ BlockIO InterpreterCreateQuery::createDictionary(ASTCreateQuery & create) create.database = context.resolveDatabase(create.database); const String & database_name = create.database; - auto guard = context.getDDLGuard(database_name, dictionary_name); + auto guard = DatabaseCatalog::instance().getDDLGuard(database_name, dictionary_name); DatabasePtr database = DatabaseCatalog::instance().getDatabase(database_name); if (database->isDictionaryExist(context, dictionary_name)) diff --git a/dbms/src/Interpreters/InterpreterDropQuery.cpp b/dbms/src/Interpreters/InterpreterDropQuery.cpp index 9a92c0eee47..af32089c7a9 100644 --- a/dbms/src/Interpreters/InterpreterDropQuery.cpp +++ b/dbms/src/Interpreters/InterpreterDropQuery.cpp @@ -78,7 +78,7 @@ BlockIO InterpreterDropQuery::executeToTable( String database_name = context.resolveDatabase(database_name_); - auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, table_name) : nullptr); + auto ddl_guard = (!no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(database_name, table_name) : nullptr); auto [database, table] = tryGetDatabaseAndTable(database_name, table_name, if_exists); @@ -170,7 +170,7 @@ BlockIO InterpreterDropQuery::executeToDictionary( String database_name = context.resolveDatabase(database_name_); - auto ddl_guard = (!no_ddl_lock ? context.getDDLGuard(database_name, dictionary_name) : nullptr); + auto ddl_guard = (!no_ddl_lock ? DatabaseCatalog::instance().getDDLGuard(database_name, dictionary_name) : nullptr); DatabasePtr database = tryGetDatabase(database_name, if_exists); @@ -237,7 +237,7 @@ BlockIO InterpreterDropQuery::executeToTemporaryTable(const String & table_name, BlockIO InterpreterDropQuery::executeToDatabase(const String & database_name, ASTDropQuery::Kind kind, bool if_exists) { - auto ddl_guard = context.getDDLGuard(database_name, ""); + auto ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, ""); if (auto database = tryGetDatabase(database_name, if_exists)) { diff --git a/dbms/src/Interpreters/InterpreterRenameQuery.cpp b/dbms/src/Interpreters/InterpreterRenameQuery.cpp index 150fca16b53..820676fa2d8 100644 --- a/dbms/src/Interpreters/InterpreterRenameQuery.cpp +++ b/dbms/src/Interpreters/InterpreterRenameQuery.cpp @@ -84,11 +84,12 @@ BlockIO InterpreterRenameQuery::execute() table_guards[to]; } + auto & database_catalog = DatabaseCatalog::instance(); + /// Must do it in consistent order. for (auto & table_guard : table_guards) - table_guard.second = context.getDDLGuard(table_guard.first.database_name, table_guard.first.table_name); + table_guard.second = database_catalog.getDDLGuard(table_guard.first.database_name, table_guard.first.table_name); - auto & database_catalog = DatabaseCatalog::instance(); for (auto & elem : descriptions) { database_catalog.assertTableDoesntExist(StorageID(elem.to_database_name, elem.to_table_name), context); diff --git a/dbms/src/Interpreters/InterpreterSystemQuery.cpp b/dbms/src/Interpreters/InterpreterSystemQuery.cpp index 6ac9da8f3d2..ca6de13d0fd 100644 --- a/dbms/src/Interpreters/InterpreterSystemQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSystemQuery.cpp @@ -315,7 +315,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const String & database_nam context.checkAccess(AccessType::RESTART_REPLICA, database_name, table_name); auto database = DatabaseCatalog::instance().getDatabase(database_name, system_context); - auto table_ddl_guard = system_context.getDDLGuard(database_name, table_name); + auto table_ddl_guard = DatabaseCatalog::instance().getDDLGuard(database_name, table_name); ASTPtr create_ast; /// Detach actions From f9a7a328526e46888de2fe0a2399f70195075c77 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Mon, 10 Feb 2020 21:38:28 +0300 Subject: [PATCH 0236/2007] less flappy --- dbms/tests/performance/cpu_synthetic.xml | 98 ++++++++++++------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/dbms/tests/performance/cpu_synthetic.xml b/dbms/tests/performance/cpu_synthetic.xml index 16bef3fd42e..50c9d5b6e6d 100644 --- a/dbms/tests/performance/cpu_synthetic.xml +++ b/dbms/tests/performance/cpu_synthetic.xml @@ -14,7 +14,7 @@ - test.hits + hits_100m_single -SELECT count() FROM test.hits WHERE NOT ignore(cityHash64(SearchPhrase)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(cityHash64(SearchPhrase)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(SearchPhrase)) -SELECT count() FROM test.hits WHERE NOT ignore(farmHash64(SearchPhrase)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(farmHash64(SearchPhrase)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(farmHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(farmHash64(SearchPhrase)) -SELECT count() FROM test.hits WHERE NOT ignore(metroHash64(SearchPhrase)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(metroHash64(SearchPhrase)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(metroHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(metroHash64(SearchPhrase)) -SELECT count() FROM test.hits WHERE NOT ignore(sipHash64(SearchPhrase)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(sipHash64(SearchPhrase)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(SearchPhrase)) -SELECT count() FROM test.hits WHERE NOT ignore(MD5(SearchPhrase)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(MD5(SearchPhrase)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(SearchPhrase)) -SELECT count() FROM test.hits WHERE NOT ignore(MD5(URL)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(MD5(URL)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(URL)) -SELECT count() FROM test.hits WHERE NOT ignore(cityHash64(URL)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(cityHash64(URL)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(URL)) -SELECT count() FROM test.hits WHERE NOT ignore(sipHash64(URL)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(sipHash64(URL)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(URL)) -SELECT count() FROM test.hits WHERE NOT ignore(cityHash64(PageCharset)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(cityHash64(PageCharset)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(PageCharset)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(PageCharset)) -SELECT count() FROM test.hits WHERE URL LIKE '%metrika%' SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE URL LIKE '%metrika%' +SELECT count() FROM hits_100m_single WHERE URL LIKE '%metrika%' SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE URL LIKE '%metrika%' -SELECT count() FROM test.hits WHERE positionCaseInsensitiveUTF8(URL, 'новости') != 0 SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE positionCaseInsensitiveUTF8(URL, 'новости') != 0 +SELECT count() FROM hits_100m_single WHERE positionCaseInsensitiveUTF8(URL, 'новости') != 0 SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE positionCaseInsensitiveUTF8(URL, 'новости') != 0 -SELECT count() FROM test.hits WHERE match(URL, '^https?://(?:www\\.)?metri[kc]a\\.yandex\\.(?:ru|com|com\\.tr|ua|by|kz)/.+?2014') SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE match(URL, '^https?://(?:www\\.)?metri[kc]a\\.yandex\\.(?:ru|com|com\\.tr|ua|by|kz)/.+?2014') +SELECT count() FROM hits_100m_single WHERE match(URL, '^https?://(?:www\\.)?metri[kc]a\\.yandex\\.(?:ru|com|com\\.tr|ua|by|kz)/.+?2014') SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE match(URL, '^https?://(?:www\\.)?metri[kc]a\\.yandex\\.(?:ru|com|com\\.tr|ua|by|kz)/.+?2014') -SELECT SearchEngineID, SearchPhrase, RegionID FROM test.hits GROUP BY SearchEngineID, SearchPhrase, RegionID ORDER BY count() DESC LIMIT 10 SETTINGS max_threads = 1 -SELECT SearchEngineID, SearchPhrase, RegionID FROM test.hits GROUP BY SearchEngineID, SearchPhrase, RegionID ORDER BY count() DESC LIMIT 10 +SELECT SearchEngineID, SearchPhrase, RegionID FROM hits_100m_single GROUP BY SearchEngineID, SearchPhrase, RegionID ORDER BY count() DESC LIMIT 10 SETTINGS max_threads = 1 +SELECT SearchEngineID, SearchPhrase, RegionID FROM hits_100m_single GROUP BY SearchEngineID, SearchPhrase, RegionID ORDER BY count() DESC LIMIT 10 -SELECT count() FROM test.hits WHERE NOT ignore(toMonday(EventTime)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(toMonday(EventTime)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(toMonday(EventTime)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(toMonday(EventTime)) -SELECT count() FROM test.hits WHERE NOT ignore(cutQueryString(URL)) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(cutQueryString(URL)) +SELECT count() FROM hits_100m_single WHERE NOT ignore(cutQueryString(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(cutQueryString(URL)) -SELECT quantilesIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits SETTINGS max_threads = 1 -SELECT quantilesIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits +SELECT quantilesIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT quantilesTimingIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits SETTINGS max_threads = 1 -SELECT quantilesTimingIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits +SELECT quantilesTimingIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesTimingIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT quantilesExactIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits SETTINGS max_threads = 1 -SELECT quantilesExactIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits +SELECT quantilesExactIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesExactIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT quantilesTDigestIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits SETTINGS max_threads = 1 -SELECT quantilesTDigestIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM test.hits +SELECT quantilesTDigestIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesTDigestIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT uniq(UserID) FROM test.hits SETTINGS max_threads = 1 -SELECT uniq(UserID) FROM test.hits +SELECT uniq(UserID) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT uniq(UserID) FROM hits_100m_single -SELECT uniqCombined(17)(UserID) FROM test.hits SETTINGS max_threads = 1 -SELECT uniqCombined(17)(UserID) FROM test.hits +SELECT uniqCombined(17)(UserID) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT uniqCombined(17)(UserID) FROM hits_100m_single -SELECT uniqExact(UserID) FROM test.hits SETTINGS max_threads = 1 -SELECT uniqExact(UserID) FROM test.hits +SELECT uniqExact(UserID) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT uniqExact(UserID) FROM hits_100m_single -SELECT RegionID, uniq(UserID) FROM test.hits GROUP BY RegionID SETTINGS max_threads = 1 -SELECT RegionID, uniq(UserID) FROM test.hits GROUP BY RegionID +SELECT RegionID, uniq(UserID) FROM hits_100m_single GROUP BY RegionID SETTINGS max_threads = 1 +SELECT RegionID, uniq(UserID) FROM hits_100m_single GROUP BY RegionID -SELECT count() FROM test.hits WHERE NOT ignore(*) SETTINGS max_threads = 1 -SELECT count() FROM test.hits WHERE NOT ignore(*) +SELECT count() FROM hits_100m_single WHERE NOT ignore(*) SETTINGS max_threads = 1 +SELECT count() FROM hits_100m_single WHERE NOT ignore(*) From c72c38aea96257e747e10dd8a532526e251aa36e Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Mon, 10 Feb 2020 23:27:06 +0300 Subject: [PATCH 0237/2007] require strict part type in MergeTreeReaders --- .../Storages/MergeTree/IMergeTreeDataPart.h | 35 ++++++++++--------- .../src/Storages/MergeTree/IMergeTreeReader.h | 1 - .../MergeTree/MergeTreeDataPartCompact.cpp | 3 +- .../MergeTree/MergeTreeDataPartCompact.h | 6 ---- .../MergeTree/MergeTreeDataPartWide.cpp | 7 ++-- .../MergeTree/MergeTreeDataPartWide.h | 6 ---- .../Storages/MergeTree/MergeTreeIOSettings.h | 1 - .../MergeTree/MergeTreeReaderCompact.cpp | 15 +++++--- .../MergeTree/MergeTreeReaderCompact.h | 6 +++- .../MergeTree/MergeTreeReaderWide.cpp | 15 +++++--- .../Storages/MergeTree/MergeTreeReaderWide.h | 6 +++- 11 files changed, 58 insertions(+), 43 deletions(-) diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index e4a3621126b..fb91a46ca74 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -53,6 +53,24 @@ public: using ColumnSizeByName = std::unordered_map; using NameToPosition = std::unordered_map; + using Type = MergeTreeDataPartType; + + + IMergeTreeDataPart( + const MergeTreeData & storage_, + const String & name_, + const MergeTreePartInfo & info_, + const DiskPtr & disk, + const std::optional & relative_path, + Type part_type_); + + IMergeTreeDataPart( + MergeTreeData & storage_, + const String & name_, + const DiskPtr & disk, + const std::optional & relative_path, + Type part_type_); + virtual MergeTreeReaderPtr getReader( const NamesAndTypesList & columns_, const MarkRanges & mark_ranges, @@ -93,27 +111,11 @@ public: using ColumnToSize = std::map; virtual void accumulateColumnSizes(ColumnToSize & /* column_to_size */) const {} - using Type = MergeTreeDataPartType; Type getType() const { return part_type; } static String typeToString(Type type); String getTypeName() const { return typeToString(getType()); } - IMergeTreeDataPart( - const MergeTreeData & storage_, - const String & name_, - const MergeTreePartInfo & info_, - const DiskPtr & disk, - const std::optional & relative_path, - Type part_type_); - - IMergeTreeDataPart( - MergeTreeData & storage_, - const String & name_, - const DiskPtr & disk, - const std::optional & relative_path, - Type part_type_); - void setColumns(const NamesAndTypesList & new_columns); const NamesAndTypesList & getColumns() const { return columns; } @@ -132,6 +134,7 @@ public: /// This is useful when you want to change e.g. block numbers or the mutation version of the part. String getNewName(const MergeTreePartInfo & new_part_info) const; + /// Returns column position in part structure or std::nullopt if it's missing in part. std::optional getColumnPosition(const String & column_name) const; /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()). diff --git a/dbms/src/Storages/MergeTree/IMergeTreeReader.h b/dbms/src/Storages/MergeTree/IMergeTreeReader.h index ed2fd3e4d2c..61366d26072 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeReader.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeReader.h @@ -55,7 +55,6 @@ public: MergeTreeData::DataPartPtr data_part; protected: - /// avg_value_size_hints are used to reduce the number of reallocations when creating columns of variable size. ValueSizeMap avg_value_size_hints; /// Stores states for IDataType::deserializeBinaryBulk diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index abe7e34fd1f..8f36ce5e828 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -43,8 +43,9 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader( const ValueSizeMap & avg_value_size_hints, const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { + auto ptr = std::static_pointer_cast(shared_from_this()); return std::make_unique( - shared_from_this(), columns_to_read, uncompressed_cache, + ptr, columns_to_read, uncompressed_cache, mark_cache, mark_ranges, reader_settings, avg_value_size_hints, profile_callback); } diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h index ec366038df9..90c58f497b7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.h @@ -5,16 +5,10 @@ namespace DB { -struct ColumnSize; -class MergeTreeData; - /// Description of the data part. class MergeTreeDataPartCompact : public IMergeTreeDataPart { public: - using Checksums = MergeTreeDataPartChecksums; - using Checksum = MergeTreeDataPartChecksums::Checksum; - static constexpr auto DATA_FILE_NAME = "data"; static constexpr auto DATA_FILE_EXTENSION = ".bin"; static constexpr auto TEMP_FILE_SUFFIX = "_temp"; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp index 95efffc012d..740071d17a2 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.cpp @@ -43,8 +43,11 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader( const ValueSizeMap & avg_value_size_hints, const ReadBufferFromFileBase::ProfileCallback & profile_callback) const { - return std::make_unique(shared_from_this(), columns_to_read, uncompressed_cache, - mark_cache, mark_ranges, reader_settings, avg_value_size_hints, profile_callback); + auto ptr = std::static_pointer_cast(shared_from_this()); + return std::make_unique( + ptr, columns_to_read, uncompressed_cache, + mark_cache, mark_ranges, reader_settings, + avg_value_size_hints, profile_callback); } IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter( diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h index 7af6d9f7680..27ee4fc66e7 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartWide.h @@ -5,16 +5,10 @@ namespace DB { -struct ColumnSize; -class MergeTreeData; - /// Description of the data part. class MergeTreeDataPartWide : public IMergeTreeDataPart { public: - using Checksums = MergeTreeDataPartChecksums; - using Checksum = MergeTreeDataPartChecksums::Checksum; - MergeTreeDataPartWide( const MergeTreeData & storage_, const String & name_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h index c0e4384c135..5d3b2945d47 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h +++ b/dbms/src/Storages/MergeTree/MergeTreeIOSettings.h @@ -37,4 +37,3 @@ struct MergeTreeWriterSettings bool skip_offsets = false; }; } - diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index 0e896843d2e..533a66c0beb 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -15,10 +15,17 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; } -MergeTreeReaderCompact::MergeTreeReaderCompact(const MergeTreeData::DataPartPtr & data_part_, - const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, - const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) + +MergeTreeReaderCompact::MergeTreeReaderCompact( + const DataPartCompactPtr & data_part_, + const NamesAndTypesList & columns_, + UncompressedCache * uncompressed_cache_, + MarkCache * mark_cache_, + const MarkRanges & mark_ranges_, + const MergeTreeReaderSettings & settings_, + const ValueSizeMap & avg_value_size_hints_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_) : IMergeTreeReader(data_part_, columns_, uncompressed_cache_, mark_cache_, mark_ranges_, settings_, avg_value_size_hints_) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h index 29a97fe7c5a..4ec5b8489d6 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderCompact.h @@ -8,11 +8,15 @@ namespace DB { +class MergeTreeDataPartCompact; +using DataPartCompactPtr = std::shared_ptr; + /// Reader for compact parts class MergeTreeReaderCompact : public IMergeTreeReader { public: - MergeTreeReaderCompact(const MergeTreeData::DataPartPtr & data_part_, + MergeTreeReaderCompact( + const DataPartCompactPtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 1db7b53a799..da3be711f22 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include @@ -24,10 +25,16 @@ namespace ErrorCodes extern const int ARGUMENT_OUT_OF_BOUND; } -MergeTreeReaderWide::MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data_part_, - const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, - const MarkRanges & mark_ranges_, const MergeTreeReaderSettings & settings_, const ValueSizeMap & avg_value_size_hints_, - const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_) +MergeTreeReaderWide::MergeTreeReaderWide( + const DataPartWidePtr & data_part_, + const NamesAndTypesList & columns_, + UncompressedCache * uncompressed_cache_, + MarkCache * mark_cache_, + const MarkRanges & mark_ranges_, + const MergeTreeReaderSettings & settings_, + const ValueSizeMap & avg_value_size_hints_, + const ReadBufferFromFileBase::ProfileCallback & profile_callback_, + clockid_t clock_type_) : IMergeTreeReader(data_part_, columns_ , uncompressed_cache_, mark_cache_, mark_ranges_ , settings_, avg_value_size_hints_) diff --git a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h index 4ff62d89300..a43687d113f 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h +++ b/dbms/src/Storages/MergeTree/MergeTreeReaderWide.h @@ -8,11 +8,15 @@ namespace DB { +class MergeTreeDataPartWide; +using DataPartWidePtr = std::shared_ptr; + /// Reader for Wide parts. class MergeTreeReaderWide : public IMergeTreeReader { public: - MergeTreeReaderWide(const MergeTreeData::DataPartPtr & data_part_, + MergeTreeReaderWide( + const DataPartWidePtr & data_part_, const NamesAndTypesList & columns_, UncompressedCache * uncompressed_cache_, MarkCache * mark_cache_, From e2688e030d9a1f84f5caa4a14f0a8cd43ed0c2db Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 11 Feb 2020 12:53:54 +0300 Subject: [PATCH 0238/2007] fix indufficient arguments parsing --- dbms/src/TableFunctions/TableFunctionRandom.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index fff1ed83539..a08dbe6b691 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -409,6 +409,11 @@ StoragePtr TableFunctionRandom::executeImpl(const ASTPtr & ast_function, const C ASTs & args = args_func.at(0)->children; + if (args.size() < 2) + throw Exception("Table function '" + getName() + "' requires at least two arguments: "\ + " structure, limit(, max_array_length, max_string_length, random_seed).", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + if (args.size() > 5) throw Exception("Table function '" + getName() + "' requires at most five arguments: "\ " structure, limit, max_array_length, max_string_length, random_seed.", From b481682a1e067d44219bd47e752f685f1e704513 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 11 Feb 2020 14:19:19 +0300 Subject: [PATCH 0239/2007] just comments --- dbms/programs/copier/ClusterCopier.cpp | 872 ++++--------------------- dbms/programs/copier/ClusterCopier.h | 4 +- dbms/programs/copier/Internals.h | 750 ++++++++++++++++++++- 3 files changed, 854 insertions(+), 772 deletions(-) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp index 2c6b16a7ae4..e8b3018a861 100644 --- a/dbms/programs/copier/ClusterCopier.cpp +++ b/dbms/programs/copier/ClusterCopier.cpp @@ -1,87 +1,10 @@ #include "ClusterCopier.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - +#include "Internals.h" namespace DB { -namespace ErrorCodes -{ - extern const int NO_ZOOKEEPER; - extern const int BAD_ARGUMENTS; - extern const int UNKNOWN_TABLE; - extern const int UNFINISHED; - extern const int UNKNOWN_ELEMENT_IN_CONFIG; -} - using ConfigurationPtr = Poco::AutoPtr; @@ -92,661 +15,19 @@ static ConfigurationPtr getConfigurationFromXMLString(const std::string & xml_da return {new Poco::Util::XMLConfiguration{&input_source}}; } -namespace -{ - - -using DatabaseAndTableName = std::pair; - -String getQuotedTable(const String & database, const String & table) -{ - if (database.empty()) - { - return backQuoteIfNeed(table); - } - - return backQuoteIfNeed(database) + "." + backQuoteIfNeed(table); -} - -String getQuotedTable(const DatabaseAndTableName & db_and_table) -{ - return getQuotedTable(db_and_table.first, db_and_table.second); -} - - -enum class TaskState -{ - Started = 0, - Finished, - Unknown -}; - -/// Used to mark status of shard partition tasks -struct TaskStateWithOwner -{ - TaskStateWithOwner() = default; - TaskStateWithOwner(TaskState state_, const String & owner_) : state(state_), owner(owner_) {} - - TaskState state{TaskState::Unknown}; - String owner; - - static String getData(TaskState state, const String & owner) - { - return TaskStateWithOwner(state, owner).toString(); - } - - String toString() - { - WriteBufferFromOwnString wb; - wb << static_cast(state) << "\n" << escape << owner; - return wb.str(); - } - - static TaskStateWithOwner fromString(const String & data) - { - ReadBufferFromString rb(data); - TaskStateWithOwner res; - UInt32 state; - - rb >> state >> "\n" >> escape >> res.owner; - - if (state >= static_cast(TaskState::Unknown)) - throw Exception("Unknown state " + data, ErrorCodes::LOGICAL_ERROR); - - res.state = static_cast(state); - return res; - } -}; - - -/// Hierarchical description of the tasks -struct ShardPartition; -struct TaskShard; -struct TaskTable; -struct TaskCluster; -struct ClusterPartition; - -using TasksPartition = std::map>; -using ShardInfo = Cluster::ShardInfo; -using TaskShardPtr = std::shared_ptr; -using TasksShard = std::vector; -using TasksTable = std::list; -using ClusterPartitions = std::map>; - - -/// Just destination partition of a shard -struct ShardPartition -{ - ShardPartition(TaskShard & parent, const String & name_quoted_) : task_shard(parent), name(name_quoted_) {} - - String getPartitionPath() const; - String getPartitionCleanStartPath() const; - String getCommonPartitionIsDirtyPath() const; - String getCommonPartitionIsCleanedPath() const; - String getPartitionActiveWorkersPath() const; - String getActiveWorkerPath() const; - String getPartitionShardsPath() const; - String getShardStatusPath() const; - - TaskShard & task_shard; - String name; -}; - - -struct ShardPriority -{ - UInt8 is_remote = 1; - size_t hostname_difference = 0; - UInt8 random = 0; - - static bool greaterPriority(const ShardPriority & current, const ShardPriority & other) - { - return std::forward_as_tuple(current.is_remote, current.hostname_difference, current.random) - < std::forward_as_tuple(other.is_remote, other.hostname_difference, other.random); - } -}; - - -struct TaskShard -{ - TaskShard(TaskTable & parent, const ShardInfo & info_) : task_table(parent), info(info_) {} - - TaskTable & task_table; - - ShardInfo info; - UInt32 numberInCluster() const { return info.shard_num; } - UInt32 indexInCluster() const { return info.shard_num - 1; } - - String getDescription() const; - String getHostNameExample() const; - - /// Used to sort clusters by their proximity - ShardPriority priority; - - /// Column with unique destination partitions (computed from engine_push_partition_key expr.) in the shard - ColumnWithTypeAndName partition_key_column; - - /// There is a task for each destination partition - TasksPartition partition_tasks; - - /// Which partitions have been checked for existence - /// If some partition from this lists is exists, it is in partition_tasks - std::set checked_partitions; - - /// Last CREATE TABLE query of the table of the shard - ASTPtr current_pull_table_create_query; - - /// Internal distributed tables - DatabaseAndTableName table_read_shard; - DatabaseAndTableName table_split_shard; -}; - - -/// Contains info about all shards that contain a partition -struct ClusterPartition -{ - double elapsed_time_seconds = 0; - UInt64 bytes_copied = 0; - UInt64 rows_copied = 0; - UInt64 blocks_copied = 0; - - UInt64 total_tries = 0; -}; - - -struct TaskTable -{ - TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfiguration & config, const String & prefix, - const String & table_key); - - TaskCluster & task_cluster; - - String getPartitionPath(const String & partition_name) const; - String getPartitionIsDirtyPath(const String & partition_name) const; - String getPartitionIsCleanedPath(const String & partition_name) const; - String getPartitionTaskStatusPath(const String & partition_name) const; - - String name_in_config; - - /// Used as task ID - String table_id; - - /// Source cluster and table - String cluster_pull_name; - DatabaseAndTableName table_pull; - - /// Destination cluster and table - String cluster_push_name; - DatabaseAndTableName table_push; - - /// Storage of destination table - String engine_push_str; - ASTPtr engine_push_ast; - ASTPtr engine_push_partition_key_ast; - - /// A Distributed table definition used to split data - String sharding_key_str; - ASTPtr sharding_key_ast; - ASTPtr engine_split_ast; - - /// Additional WHERE expression to filter input data - String where_condition_str; - ASTPtr where_condition_ast; - - /// Resolved clusters - ClusterPtr cluster_pull; - ClusterPtr cluster_push; - - /// Filter partitions that should be copied - bool has_enabled_partitions = false; - Strings enabled_partitions; - NameSet enabled_partitions_set; - - /// Prioritized list of shards - TasksShard all_shards; - TasksShard local_shards; - - ClusterPartitions cluster_partitions; - NameSet finished_cluster_partitions; - - /// Parition names to process in user-specified order - Strings ordered_partition_names; - - ClusterPartition & getClusterPartition(const String & partition_name) - { - auto it = cluster_partitions.find(partition_name); - if (it == cluster_partitions.end()) - throw Exception("There are no cluster partition " + partition_name + " in " + table_id, ErrorCodes::LOGICAL_ERROR); - return it->second; - } - - Stopwatch watch; - UInt64 bytes_copied = 0; - UInt64 rows_copied = 0; - - template - void initShards(RandomEngine && random_engine); -}; - - -struct TaskCluster -{ - TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_) - : task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {} - - void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); - - /// Set (or update) settings and max_workers param - void reloadSettings(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); - - /// Base node for all tasks. Its structure: - /// workers/ - directory with active workers (amount of them is less or equal max_workers) - /// description - node with task configuration - /// table_table1/ - directories with per-partition copying status - String task_zookeeper_path; - - /// Database used to create temporary Distributed tables - String default_local_database; - - /// Limits number of simultaneous workers - UInt64 max_workers = 0; - - /// Base settings for pull and push - Settings settings_common; - /// Settings used to fetch data - Settings settings_pull; - /// Settings used to insert data - Settings settings_push; - - String clusters_prefix; - - /// Subtasks - TasksTable table_tasks; - - std::random_device random_device; - pcg64 random_engine; -}; - - -struct MultiTransactionInfo -{ - int32_t code; - Coordination::Requests requests; - Coordination::Responses responses; -}; - -// Creates AST representing 'ENGINE = Distributed(cluster, db, table, [sharding_key]) -std::shared_ptr createASTStorageDistributed( - const String & cluster_name, const String & database, const String & table, const ASTPtr & sharding_key_ast = nullptr) -{ - auto args = std::make_shared(); - args->children.emplace_back(std::make_shared(cluster_name)); - args->children.emplace_back(std::make_shared(database)); - args->children.emplace_back(std::make_shared(table)); - if (sharding_key_ast) - args->children.emplace_back(sharding_key_ast); - - auto engine = std::make_shared(); - engine->name = "Distributed"; - engine->arguments = args; - - auto storage = std::make_shared(); - storage->set(storage->engine, engine); - - return storage; -} - - -BlockInputStreamPtr squashStreamIntoOneBlock(const BlockInputStreamPtr & stream) -{ - return std::make_shared( - stream, - std::numeric_limits::max(), - std::numeric_limits::max()); -} - -Block getBlockWithAllStreamData(const BlockInputStreamPtr & stream) -{ - return squashStreamIntoOneBlock(stream)->read(); -} - - -/// Path getters - -String TaskTable::getPartitionPath(const String & partition_name) const -{ - return task_cluster.task_zookeeper_path // root - + "/tables/" + table_id // tables/dst_cluster.merge.hits - + "/" + escapeForFileName(partition_name); // 201701 -} - -String ShardPartition::getPartitionCleanStartPath() const -{ - return getPartitionPath() + "/clean_start"; -} - -String ShardPartition::getPartitionPath() const -{ - return task_shard.task_table.getPartitionPath(name); -} - -String ShardPartition::getShardStatusPath() const -{ - // schema: //tables///shards/ - // e.g. /root/table_test.hits/201701/shards/1 - return getPartitionShardsPath() + "/" + toString(task_shard.numberInCluster()); -} - -String ShardPartition::getPartitionShardsPath() const -{ - return getPartitionPath() + "/shards"; -} - -String ShardPartition::getPartitionActiveWorkersPath() const -{ - return getPartitionPath() + "/partition_active_workers"; -} - -String ShardPartition::getActiveWorkerPath() const -{ - return getPartitionActiveWorkersPath() + "/" + toString(task_shard.numberInCluster()); -} - -String ShardPartition::getCommonPartitionIsDirtyPath() const -{ - return getPartitionPath() + "/is_dirty"; -} - -String ShardPartition::getCommonPartitionIsCleanedPath() const -{ - return getCommonPartitionIsDirtyPath() + "/cleaned"; -} - -String TaskTable::getPartitionIsDirtyPath(const String & partition_name) const -{ - return getPartitionPath(partition_name) + "/is_dirty"; -} - -String TaskTable::getPartitionIsCleanedPath(const String & partition_name) const -{ - return getPartitionIsDirtyPath(partition_name) + "/cleaned"; -} - -String TaskTable::getPartitionTaskStatusPath(const String & partition_name) const -{ - return getPartitionPath(partition_name) + "/shards"; -} - -String DB::TaskShard::getDescription() const -{ - std::stringstream ss; - ss << "N" << numberInCluster() - << " (having a replica " << getHostNameExample() - << ", pull table " + getQuotedTable(task_table.table_pull) - << " of cluster " + task_table.cluster_pull_name << ")"; - return ss.str(); -} - -String DB::TaskShard::getHostNameExample() const -{ - auto & replicas = task_table.cluster_pull->getShardsAddresses().at(indexInCluster()); - return replicas.at(0).readableString(); -} - - -static bool isExtendedDefinitionStorage(const ASTPtr & storage_ast) -{ - const auto & storage = storage_ast->as(); - return storage.partition_by || storage.order_by || storage.sample_by; -} - -static ASTPtr extractPartitionKey(const ASTPtr & storage_ast) -{ - String storage_str = queryToString(storage_ast); - - const auto & storage = storage_ast->as(); - const auto & engine = storage.engine->as(); - - if (!endsWith(engine.name, "MergeTree")) - { - throw Exception("Unsupported engine was specified in " + storage_str + ", only *MergeTree engines are supported", - ErrorCodes::BAD_ARGUMENTS); - } - - if (isExtendedDefinitionStorage(storage_ast)) - { - if (storage.partition_by) - return storage.partition_by->clone(); - - static const char * all = "all"; - return std::make_shared(Field(all, strlen(all))); - } - else - { - bool is_replicated = startsWith(engine.name, "Replicated"); - size_t min_args = is_replicated ? 3 : 1; - - if (!engine.arguments) - throw Exception("Expected arguments in " + storage_str, ErrorCodes::BAD_ARGUMENTS); - - ASTPtr arguments_ast = engine.arguments->clone(); - ASTs & arguments = arguments_ast->children; - - if (arguments.size() < min_args) - throw Exception("Expected at least " + toString(min_args) + " arguments in " + storage_str, ErrorCodes::BAD_ARGUMENTS); - - ASTPtr & month_arg = is_replicated ? arguments[2] : arguments[1]; - return makeASTFunction("toYYYYMM", month_arg->clone()); - } -} - - -TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfiguration & config, const String & prefix_, - const String & table_key) -: task_cluster(parent) -{ - String table_prefix = prefix_ + "." + table_key + "."; - - name_in_config = table_key; - - cluster_pull_name = config.getString(table_prefix + "cluster_pull"); - cluster_push_name = config.getString(table_prefix + "cluster_push"); - - table_pull.first = config.getString(table_prefix + "database_pull"); - table_pull.second = config.getString(table_prefix + "table_pull"); - - table_push.first = config.getString(table_prefix + "database_push"); - table_push.second = config.getString(table_prefix + "table_push"); - - /// Used as node name in ZooKeeper - table_id = escapeForFileName(cluster_push_name) - + "." + escapeForFileName(table_push.first) - + "." + escapeForFileName(table_push.second); - - engine_push_str = config.getString(table_prefix + "engine"); - { - ParserStorage parser_storage; - engine_push_ast = parseQuery(parser_storage, engine_push_str, 0); - engine_push_partition_key_ast = extractPartitionKey(engine_push_ast); - } - - sharding_key_str = config.getString(table_prefix + "sharding_key"); - { - ParserExpressionWithOptionalAlias parser_expression(false); - sharding_key_ast = parseQuery(parser_expression, sharding_key_str, 0); - engine_split_ast = createASTStorageDistributed(cluster_push_name, table_push.first, table_push.second, sharding_key_ast); - } - - where_condition_str = config.getString(table_prefix + "where_condition", ""); - if (!where_condition_str.empty()) - { - ParserExpressionWithOptionalAlias parser_expression(false); - where_condition_ast = parseQuery(parser_expression, where_condition_str, 0); - - // Will use canonical expression form - where_condition_str = queryToString(where_condition_ast); - } - - String enabled_partitions_prefix = table_prefix + "enabled_partitions"; - has_enabled_partitions = config.has(enabled_partitions_prefix); - - if (has_enabled_partitions) - { - Strings keys; - config.keys(enabled_partitions_prefix, keys); - - if (keys.empty()) - { - /// Parse list of partition from space-separated string - String partitions_str = config.getString(table_prefix + "enabled_partitions"); - boost::trim_if(partitions_str, isWhitespaceASCII); - boost::split(enabled_partitions, partitions_str, isWhitespaceASCII, boost::token_compress_on); - } - else - { - /// Parse sequence of ... - for (const String & key : keys) - { - if (!startsWith(key, "partition")) - throw Exception("Unknown key " + key + " in " + enabled_partitions_prefix, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); - - enabled_partitions.emplace_back(config.getString(enabled_partitions_prefix + "." + key)); - } - } - - std::copy(enabled_partitions.begin(), enabled_partitions.end(), std::inserter(enabled_partitions_set, enabled_partitions_set.begin())); - } -} - - -static ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std::string & local_hostname, UInt8 random) -{ - ShardPriority res; - - if (replicas.empty()) - return res; - - res.is_remote = 1; - for (auto & replica : replicas) - { - if (isLocalAddress(DNSResolver::instance().resolveHost(replica.host_name))) - { - res.is_remote = 0; - break; - } - } - - res.hostname_difference = std::numeric_limits::max(); - for (auto & replica : replicas) - { - size_t difference = getHostNameDifference(local_hostname, replica.host_name); - res.hostname_difference = std::min(difference, res.hostname_difference); - } - - res.random = random; - return res; -} - -template -void TaskTable::initShards(RandomEngine && random_engine) -{ - const String & fqdn_name = getFQDNOrHostName(); - std::uniform_int_distribution get_urand(0, std::numeric_limits::max()); - - // Compute the priority - for (auto & shard_info : cluster_pull->getShardsInfo()) - { - TaskShardPtr task_shard = std::make_shared(*this, shard_info); - const auto & replicas = cluster_pull->getShardsAddresses().at(task_shard->indexInCluster()); - task_shard->priority = getReplicasPriority(replicas, fqdn_name, get_urand(random_engine)); - - all_shards.emplace_back(task_shard); - } - - // Sort by priority - std::sort(all_shards.begin(), all_shards.end(), - [] (const TaskShardPtr & lhs, const TaskShardPtr & rhs) - { - return ShardPriority::greaterPriority(lhs->priority, rhs->priority); - }); - - // Cut local shards - auto it_first_remote = std::lower_bound(all_shards.begin(), all_shards.end(), 1, - [] (const TaskShardPtr & lhs, UInt8 is_remote) - { - return lhs->priority.is_remote < is_remote; - }); - - local_shards.assign(all_shards.begin(), it_first_remote); -} - - -void DB::TaskCluster::loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key) -{ - String prefix = base_key.empty() ? "" : base_key + "."; - - clusters_prefix = prefix + "remote_servers"; - if (!config.has(clusters_prefix)) - throw Exception("You should specify list of clusters in " + clusters_prefix, ErrorCodes::BAD_ARGUMENTS); - - Poco::Util::AbstractConfiguration::Keys tables_keys; - config.keys(prefix + "tables", tables_keys); - - for (const auto & table_key : tables_keys) - { - table_tasks.emplace_back(*this, config, prefix + "tables", table_key); - } -} - -void DB::TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & config, const String & base_key) -{ - String prefix = base_key.empty() ? "" : base_key + "."; - - max_workers = config.getUInt64(prefix + "max_workers"); - - settings_common = Settings(); - if (config.has(prefix + "settings")) - settings_common.loadSettingsFromConfig(prefix + "settings", config); - - settings_pull = settings_common; - if (config.has(prefix + "settings_pull")) - settings_pull.loadSettingsFromConfig(prefix + "settings_pull", config); - - settings_push = settings_common; - if (config.has(prefix + "settings_push")) - settings_push.loadSettingsFromConfig(prefix + "settings_push", config); - - auto set_default_value = [] (auto && setting, auto && default_value) - { - setting = setting.changed ? setting.value : default_value; - }; - - /// Override important settings - settings_pull.readonly = 1; - settings_push.insert_distributed_sync = 1; - set_default_value(settings_pull.load_balancing, LoadBalancing::NEAREST_HOSTNAME); - set_default_value(settings_pull.max_threads, 1); - set_default_value(settings_pull.max_block_size, 8192UL); - set_default_value(settings_pull.preferred_block_size_bytes, 0); - set_default_value(settings_push.insert_distributed_timeout, 0); -} - - -} // end of an anonymous namespace - class ClusterCopier { public: - ClusterCopier(const String & task_path_, - const String & host_id_, - const String & proxy_database_name_, + ClusterCopier(String task_path_, + String host_id_, + String proxy_database_name_, Context & context_) : - task_zookeeper_path(task_path_), - host_id(host_id_), - working_database_name(proxy_database_name_), + task_zookeeper_path(std::move(task_path_)), + host_id(std::move(host_id_)), + working_database_name(std::move(proxy_database_name_)), context(context_), log(&Poco::Logger::get("ClusterCopier")) { @@ -940,14 +221,14 @@ public: task_description_watch_zookeeper = zookeeper; String task_config_str; - Coordination::Stat stat; + Coordination::Stat stat{}; int code; zookeeper->tryGetWatch(task_description_path, task_config_str, &stat, task_description_watch_callback, &code); if (code) throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS); - LOG_DEBUG(log, "Loading description, zxid=" << task_descprtion_current_stat.czxid); + LOG_DEBUG(log, "Loading description, zxid=" << task_description_current_stat.czxid); auto config = getConfigurationFromXMLString(task_config_str); /// Setup settings @@ -955,7 +236,7 @@ public: context.getSettingsRef() = task_cluster->settings_common; task_cluster_current_config = config; - task_descprtion_current_stat = stat; + task_description_current_stat = stat; } void updateConfigIfNeeded() @@ -1092,7 +373,7 @@ protected: { updateConfigIfNeeded(); - Coordination::Stat stat; + Coordination::Stat stat{}; zookeeper->get(workers_version_path, &stat); auto version = stat.version; zookeeper->get(workers_path, &stat); @@ -1186,7 +467,7 @@ protected: task_table.getPartitionIsDirtyPath(partition_name), task_table.getPartitionIsCleanedPath(partition_name) ); - Coordination::Stat stat; + Coordination::Stat stat{}; LogicalClock task_start_clock; if (zookeeper->exists(task_table.getPartitionTaskStatusPath(partition_name), &stat)) task_start_clock = LogicalClock(stat.mzxid); @@ -1264,7 +545,7 @@ protected: } /// Replaces ENGINE and table name in a create query - std::shared_ptr rewriteCreateQueryStorage(const ASTPtr & create_query_ast, const DatabaseAndTableName & new_table, const ASTPtr & new_storage_ast) + static std::shared_ptr rewriteCreateQueryStorage(const ASTPtr & create_query_ast, const DatabaseAndTableName & new_table, const ASTPtr & new_storage_ast) { const auto & create = create_query_ast->as(); auto res = std::make_shared(create); @@ -1291,7 +572,7 @@ protected: public: UInt32 value; - WrappingUInt32(UInt32 _value) + explicit WrappingUInt32(UInt32 _value) : value(_value) {} @@ -1315,13 +596,20 @@ protected: /** Conforming Zxid definition. * cf. https://github.com/apache/zookeeper/blob/631d1b284f0edb1c4f6b0fb221bf2428aec71aaa/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md#guarantees-properties-and-definitions + * + * But it is better to read this: https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html + * + * Actually here is the definition of Zxid. + * Every change to the ZooKeeper state receives a stamp in the form of a zxid (ZooKeeper Transaction Id). + * This exposes the total ordering of all changes to ZooKeeper. Each change will have a unique zxid + * and if zxid1 is smaller than zxid2 then zxid1 happened before zxid2. */ class Zxid { public: WrappingUInt32 epoch; WrappingUInt32 counter; - Zxid(UInt64 _zxid) + explicit Zxid(UInt64 _zxid) : epoch(_zxid >> 32) , counter(_zxid) {} @@ -1338,6 +626,51 @@ protected: } }; + /* When multiple ClusterCopiers discover that the target partition is not empty, + * they will attempt to clean up this partition before proceeding to copying. + * + * Instead of purging is_dirty, the history of cleaning work is preserved and partition hygiene is established + * based on a happens-before relation between the events. + * This relation is encoded by LogicalClock based on the mzxid of the is_dirty ZNode and is_dirty/cleaned. + * The fact of the partition hygiene is encoded by CleanStateClock. + * + * For you to know what mzxid means: + * + * ZooKeeper Stat Structure: + * The Stat structure for each znode in ZooKeeper is made up of the following fields: + * + * -- czxid + * The zxid of the change that caused this znode to be created. + * + * -- mzxid + * The zxid of the change that last modified this znode. + * + * -- ctime + * The time in milliseconds from epoch when this znode was created. + * + * -- mtime + * The time in milliseconds from epoch when this znode was last modified. + * + * -- version + * The number of changes to the data of this znode. + * + * -- cversion + * The number of changes to the children of this znode. + * + * -- aversion + * The number of changes to the ACL of this znode. + * + * -- ephemeralOwner + * The session id of the owner of this znode if the znode is an ephemeral node. + * If it is not an ephemeral node, it will be zero. + * + * -- dataLength + * The length of the data field of this znode. + * + * -- numChildren + * The number of children of this znode. + * */ + class LogicalClock { public: @@ -1345,7 +678,7 @@ protected: LogicalClock() = default; - LogicalClock(UInt64 _zxid) + explicit LogicalClock(UInt64 _zxid) : zxid(_zxid) {} @@ -1354,7 +687,7 @@ protected: return bool(zxid); } - // happens-before relation with a reasonable time bound + /// happens-before relation with a reasonable time bound bool happensBefore(const LogicalClock & other) const { return !zxid @@ -1366,13 +699,14 @@ protected: return happensBefore(other); } - // strict equality check + /// strict equality check bool operator==(const LogicalClock & other) const { return zxid == other.zxid; } }; + class CleanStateClock { public: @@ -1404,7 +738,7 @@ protected: const String & clean_state_path) : stale(std::make_shared(false)) { - Coordination::Stat stat; + Coordination::Stat stat{}; String _some_data; auto watch_callback = [stale = stale] (const Coordination::WatchResponse & rsp) @@ -1462,13 +796,13 @@ protected: const String current_shards_path = task_partition.getPartitionShardsPath(); const String current_partition_active_workers_dir = task_partition.getPartitionActiveWorkersPath(); const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); - const String dirt_cleaner_path = is_dirty_flag_path + "/cleaner"; - const String is_dirt_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); + const String dirty_cleaner_path = is_dirty_flag_path + "/cleaner"; + const String is_dirty_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); zkutil::EphemeralNodeHolder::Ptr cleaner_holder; try { - cleaner_holder = zkutil::EphemeralNodeHolder::create(dirt_cleaner_path, *zookeeper, host_id); + cleaner_holder = zkutil::EphemeralNodeHolder::create(dirty_cleaner_path, *zookeeper, host_id); } catch (const Coordination::Exception & e) { @@ -1482,7 +816,7 @@ protected: throw; } - Coordination::Stat stat; + Coordination::Stat stat{}; if (zookeeper->exists(current_partition_active_workers_dir, &stat)) { if (stat.numChildren != 0) @@ -1517,7 +851,7 @@ protected: // Lock the dirty flag zookeeper->set(is_dirty_flag_path, host_id, clean_state_clock.discovery_version.value()); zookeeper->tryRemove(task_partition.getPartitionCleanStartPath()); - CleanStateClock my_clock(zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + CleanStateClock my_clock(zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); /// Remove all status nodes { @@ -1556,9 +890,9 @@ protected: { zookeeper->set(is_dirty_flag_path, host_id, my_clock.discovery_version.value()); if (my_clock.clean_state_version) - zookeeper->set(is_dirt_cleaned_path, host_id, my_clock.clean_state_version.value()); + zookeeper->set(is_dirty_cleaned_path, host_id, my_clock.clean_state_version.value()); else - zookeeper->create(is_dirt_cleaned_path, host_id, zkutil::CreateMode::Persistent); + zookeeper->create(is_dirty_cleaned_path, host_id, zkutil::CreateMode::Persistent); } else { @@ -1582,7 +916,7 @@ protected: bool tryProcessTable(const ConnectionTimeouts & timeouts, TaskTable & task_table) { - /// An heuristic: if previous shard is already done, then check next one without sleeps due to max_workers constraint + /// A heuristic: if previous shard is already done, then check next one without sleeps due to max_workers constraint bool previous_shard_is_instantly_finished = false; /// Process each partition that is present in cluster @@ -1594,6 +928,7 @@ protected: ClusterPartition & cluster_partition = task_table.cluster_partitions[partition_name]; Stopwatch watch; + /// We will check all the shards of the table and check if they contain current partition. TasksShard expected_shards; UInt64 num_failed_shards = 0; @@ -1637,6 +972,8 @@ protected: } auto it_shard_partition = shard->partition_tasks.find(partition_name); + /// Previously when we discovered that shard does not contain current partition, we skipped it. + /// At this moment partition have to be present. if (it_shard_partition == shard->partition_tasks.end()) throw Exception("There are no such partition in a shard. This is a bug.", ErrorCodes::LOGICAL_ERROR); auto & partition = it_shard_partition->second; @@ -1738,6 +1075,7 @@ protected: Error, }; + /// Job for copying partition from particular shard. PartitionTaskStatus tryProcessPartitionTask(const ConnectionTimeouts & timeouts, ShardPartition & task_partition, bool is_unprioritized_task) { PartitionTaskStatus res; @@ -1770,6 +1108,8 @@ protected: TaskShard & task_shard = task_partition.task_shard; TaskTable & task_table = task_shard.task_table; ClusterPartition & cluster_partition = task_table.getClusterPartition(task_partition.name); + const size_t number_of_splits = task_table.number_of_splits; + UNUSED(number_of_splits); /// We need to update table definitions for each partition, it could be changed after ALTER createShardInternalTables(timeouts, task_shard); @@ -1777,7 +1117,7 @@ protected: auto zookeeper = context.getZooKeeper(); const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); - const String is_dirt_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); + const String is_dirty_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); const String current_task_is_active_path = task_partition.getActiveWorkerPath(); const String current_task_status_path = task_partition.getShardStatusPath(); @@ -1803,12 +1143,15 @@ protected: }; /// Returns SELECT query filtering current partition and applying user filter - auto get_select_query = [&] (const DatabaseAndTableName & from_table, const String & fields, String limit = "") + auto get_select_query = [&] (const DatabaseAndTableName & from_table, const String & fields, String limit = "", + bool enable_splitting = false, size_t current_piece_number = 0) { String query; query += "SELECT " + fields + " FROM " + getQuotedTable(from_table); /// TODO: Bad, it is better to rewrite with ASTLiteral(partition_key_field) query += " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + task_partition.name + " AS partition_key))"; + if (enable_splitting) + query += " AND ( cityHash64(*) = " + std::to_string(current_piece_number) + " )"; if (!task_table.where_condition_str.empty()) query += " AND (" + task_table.where_condition_str + ")"; if (!limit.empty()) @@ -1823,11 +1166,11 @@ protected: LOG_DEBUG(log, "Processing " << current_task_status_path); - CleanStateClock clean_state_clock (zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + CleanStateClock clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); LogicalClock task_start_clock; { - Coordination::Stat stat; + Coordination::Stat stat{}; if (zookeeper->exists(task_partition.getPartitionShardsPath(), &stat)) task_start_clock = LogicalClock(stat.mzxid); } @@ -1887,7 +1230,8 @@ protected: } // Task is abandoned, initialize DROP PARTITION - LOG_DEBUG(log, "Task " << current_task_status_path << " has not been successfully finished by " << status.owner << ". Partition will be dropped and refilled."); + LOG_DEBUG(log, "Task " << current_task_status_path << " has not been successfully finished by " << + status.owner << ". Partition will be dropped and refilled."); create_is_dirty_node(clean_state_clock); return PartitionTaskStatus::Error; @@ -1900,7 +1244,8 @@ protected: if (!zookeeper->tryGet(task_partition.getPartitionCleanStartPath(), clean_start_status) || clean_start_status != "ok") { zookeeper->createIfNotExists(task_partition.getPartitionCleanStartPath(), ""); - auto checker = zkutil::EphemeralNodeHolder::create(task_partition.getPartitionCleanStartPath() + "/checker", *zookeeper, host_id); + auto checker = zkutil::EphemeralNodeHolder::create(task_partition.getPartitionCleanStartPath() + "/checker", + *zookeeper, host_id); // Maybe we are the first worker ASTPtr query_select_ast = get_select_query(task_shard.table_split_shard, "count()"); UInt64 count; @@ -1916,7 +1261,7 @@ protected: if (count != 0) { - Coordination::Stat stat_shards; + Coordination::Stat stat_shards{}; zookeeper->get(task_partition.getPartitionShardsPath(), &stat_shards); /// NOTE: partition is still fresh if dirt discovery happens before cleaning @@ -1938,7 +1283,7 @@ protected: /// Try start processing, create node about it { String start_state = TaskStateWithOwner::getData(TaskState::Started, host_id); - CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); @@ -1955,15 +1300,17 @@ protected: /// Try create table (if not exists) on each shard { - auto create_query_push_ast = rewriteCreateQueryStorage(task_shard.current_pull_table_create_query, task_table.table_push, task_table.engine_push_ast); + auto create_query_push_ast = rewriteCreateQueryStorage(task_shard.current_pull_table_create_query, + task_table.table_push, task_table.engine_push_ast); create_query_push_ast->as().if_not_exists = true; String query = queryToString(create_query_push_ast); LOG_DEBUG(log, "Create destination tables. Query: " << query); - UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, create_query_push_ast, &task_cluster->settings_push, + UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, + create_query_push_ast, &task_cluster->settings_push, PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << " have been created on " << shards - << " shards of " << task_table.cluster_push->getShardCount()); + LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << + " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); } /// Do the copying @@ -2073,7 +1420,7 @@ protected: /// Finalize the processing, change state of current partition task (and also check is_dirty flag) { String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); - CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirt_cleaned_path); + CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); if (clean_state_clock != new_clean_state_clock) { LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); @@ -2265,7 +1612,7 @@ protected: UInt64 & num_successful_executions = per_shard_num_successful_replicas.at(shard_index); num_successful_executions = 0; - auto increment_and_check_exit = [&] () + auto increment_and_check_exit = [&] () -> bool { ++num_successful_executions; return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard; @@ -2348,7 +1695,7 @@ private: ConfigurationPtr task_cluster_initial_config; ConfigurationPtr task_cluster_current_config; - Coordination::Stat task_descprtion_current_stat{}; + Coordination::Stat task_description_current_stat{}; std::unique_ptr task_cluster; @@ -2373,12 +1720,11 @@ void ClusterCopierApp::initialize(Poco::Util::Application & self) config_xml_path = config().getString("config-file"); task_path = config().getString("task-path"); - log_level = config().getString("log-level", "debug"); + log_level = config().getString("log-level", "trace"); is_safe_mode = config().has("safe-mode"); if (config().has("copy-fault-probability")) copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0); base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current(); - // process_id is '#_' time_t timestamp = Poco::Timestamp().epochTime(); auto curr_pid = Poco::Process::id(); diff --git a/dbms/programs/copier/ClusterCopier.h b/dbms/programs/copier/ClusterCopier.h index 89f45df8686..fe228fd6194 100644 --- a/dbms/programs/copier/ClusterCopier.h +++ b/dbms/programs/copier/ClusterCopier.h @@ -71,11 +71,9 @@ private: void mainImpl(); - void setupLogging(); - std::string config_xml_path; std::string task_path; - std::string log_level = "debug"; + std::string log_level = "trace"; bool is_safe_mode = false; double copy_fault_probability = 0; bool is_help = false; diff --git a/dbms/programs/copier/Internals.h b/dbms/programs/copier/Internals.h index a25ae7b973c..5f14604fbf9 100644 --- a/dbms/programs/copier/Internals.h +++ b/dbms/programs/copier/Internals.h @@ -1,8 +1,746 @@ -// -// Created by jakalletti on 2/7/20. -// +#pragma once -#ifndef CLICKHOUSE_INTERNALS_H -#define CLICKHOUSE_INTERNALS_H +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NO_ZOOKEEPER; + extern const int BAD_ARGUMENTS; + extern const int UNKNOWN_TABLE; + extern const int UNFINISHED; + extern const int UNKNOWN_ELEMENT_IN_CONFIG; +} + +namespace +{ + +using DatabaseAndTableName = std::pair; + +String getQuotedTable(const String & database, const String & table) +{ + if (database.empty()) + { + return backQuoteIfNeed(table); + } + + return backQuoteIfNeed(database) + "." + backQuoteIfNeed(table); +} + +String getQuotedTable(const DatabaseAndTableName & db_and_table) +{ + return getQuotedTable(db_and_table.first, db_and_table.second); +} + + +enum class TaskState +{ + Started = 0, + Finished, + Unknown +}; + +/// Used to mark status of shard partition tasks +struct TaskStateWithOwner +{ + TaskStateWithOwner() = default; + TaskStateWithOwner(TaskState state_, const String & owner_) : state(state_), owner(owner_) {} + + TaskState state{TaskState::Unknown}; + String owner; + + static String getData(TaskState state, const String & owner) + { + return TaskStateWithOwner(state, owner).toString(); + } + + String toString() + { + WriteBufferFromOwnString wb; + wb << static_cast(state) << "\n" << escape << owner; + return wb.str(); + } + + static TaskStateWithOwner fromString(const String & data) + { + ReadBufferFromString rb(data); + TaskStateWithOwner res; + UInt32 state; + + rb >> state >> "\n" >> escape >> res.owner; + + if (state >= static_cast(TaskState::Unknown)) + throw Exception("Unknown state " + data, ErrorCodes::LOGICAL_ERROR); + + res.state = static_cast(state); + return res; + } +}; + + +/// Hierarchical description of the tasks +struct ShardPartition; +struct TaskShard; +struct TaskTable; +struct TaskCluster; +struct ClusterPartition; + +using TasksPartition = std::map>; +using ShardInfo = Cluster::ShardInfo; +using TaskShardPtr = std::shared_ptr; +using TasksShard = std::vector; +using TasksTable = std::list; +using ClusterPartitions = std::map>; + + +/// Just destination partition of a shard +/// I don't know what this comment means. +/// In short, when we discovered what shards contain currently processing partition, +/// This class describes a partition (name) that is stored on the shard (parent). +struct ShardPartition +{ + ShardPartition(TaskShard & parent, String name_quoted_) : task_shard(parent), name(std::move(name_quoted_)) {} + + String getPartitionPath() const; + String getPartitionCleanStartPath() const; + String getCommonPartitionIsDirtyPath() const; + String getCommonPartitionIsCleanedPath() const; + String getPartitionActiveWorkersPath() const; + String getActiveWorkerPath() const; + String getPartitionShardsPath() const; + String getShardStatusPath() const; + + TaskShard & task_shard; + String name; +}; + + +struct ShardPriority +{ + UInt8 is_remote = 1; + size_t hostname_difference = 0; + UInt8 random = 0; + + static bool greaterPriority(const ShardPriority & current, const ShardPriority & other) + { + return std::forward_as_tuple(current.is_remote, current.hostname_difference, current.random) + < std::forward_as_tuple(other.is_remote, other.hostname_difference, other.random); + } +}; + +/// Tables has many shards and table's partiton can be stored on different shards. +/// When we copy partition we have to discover it's shards (shards which store this partition) +/// For simplier retrieval of which partitions are stored in particular shard we created TaskShard. +struct TaskShard +{ + TaskShard(TaskTable & parent, const ShardInfo & info_) : task_table(parent), info(info_) {} + + TaskTable & task_table; + + ShardInfo info; + UInt32 numberInCluster() const { return info.shard_num; } + UInt32 indexInCluster() const { return info.shard_num - 1; } + + String getDescription() const; + String getHostNameExample() const; + + /// Used to sort clusters by their proximity + ShardPriority priority; + + /// Column with unique destination partitions (computed from engine_push_partition_key expr.) in the shard + ColumnWithTypeAndName partition_key_column; + + /// There is a task for each destination partition + TasksPartition partition_tasks; + + /// Which partitions have been checked for existence + /// If some partition from this lists is exists, it is in partition_tasks + std::set checked_partitions; + + /// Last CREATE TABLE query of the table of the shard + ASTPtr current_pull_table_create_query; + + /// Internal distributed tables + DatabaseAndTableName table_read_shard; + DatabaseAndTableName table_split_shard; +}; + + +/// Contains info about all shards that contain a partition +struct ClusterPartition +{ + double elapsed_time_seconds = 0; + UInt64 bytes_copied = 0; + UInt64 rows_copied = 0; + UInt64 blocks_copied = 0; + + UInt64 total_tries = 0; +}; + + +struct TaskTable +{ + TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfiguration & config, const String & prefix, + const String & table_key); + + TaskCluster & task_cluster; + + String getPartitionPath(const String & partition_name) const; + [[maybe_unused]] String getPartitionPathWithPieceNumber(const String & partition_name, size_t current_piece_number) const; + String getPartitionIsDirtyPath(const String & partition_name) const; + String getPartitionIsCleanedPath(const String & partition_name) const; + String getPartitionTaskStatusPath(const String & partition_name) const; + + /// Partitions will be splitted into number-of-splits pieces. + /// Each piece will be copied independently. (10 by default) + size_t number_of_splits; + + String name_in_config; + + /// Used as task ID + String table_id; + + /// Source cluster and table + String cluster_pull_name; + DatabaseAndTableName table_pull; + + /// Destination cluster and table + String cluster_push_name; + DatabaseAndTableName table_push; + + /// Storage of destination table + String engine_push_str; + ASTPtr engine_push_ast; + ASTPtr engine_push_partition_key_ast; + + /// A Distributed table definition used to split data + String sharding_key_str; + ASTPtr sharding_key_ast; + ASTPtr engine_split_ast; + + /// Additional WHERE expression to filter input data + String where_condition_str; + ASTPtr where_condition_ast; + + /// Resolved clusters + ClusterPtr cluster_pull; + ClusterPtr cluster_push; + + /// Filter partitions that should be copied + bool has_enabled_partitions = false; + Strings enabled_partitions; + NameSet enabled_partitions_set; + + /// Prioritized list of shards + /// all_shards contains information about all shards in the table. + /// So we have to check whether particular shard have current partiton or not while processing. + TasksShard all_shards; + TasksShard local_shards; + + /// All partitions of the current table. + ClusterPartitions cluster_partitions; + NameSet finished_cluster_partitions; + + /// Parition names to process in user-specified order + Strings ordered_partition_names; + + ClusterPartition & getClusterPartition(const String & partition_name) + { + auto it = cluster_partitions.find(partition_name); + if (it == cluster_partitions.end()) + throw Exception("There are no cluster partition " + partition_name + " in " + table_id, ErrorCodes::LOGICAL_ERROR); + return it->second; + } + + Stopwatch watch; + UInt64 bytes_copied = 0; + UInt64 rows_copied = 0; + + template + void initShards(RandomEngine && random_engine); +}; + + +struct TaskCluster +{ + TaskCluster(const String & task_zookeeper_path_, const String & default_local_database_) + : task_zookeeper_path(task_zookeeper_path_), default_local_database(default_local_database_) {} + + void loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); + + /// Set (or update) settings and max_workers param + void reloadSettings(const Poco::Util::AbstractConfiguration & config, const String & base_key = ""); + + /// Base node for all tasks. Its structure: + /// workers/ - directory with active workers (amount of them is less or equal max_workers) + /// description - node with task configuration + /// table_table1/ - directories with per-partition copying status + String task_zookeeper_path; + + /// Database used to create temporary Distributed tables + String default_local_database; + + /// Limits number of simultaneous workers + UInt64 max_workers = 0; + + /// Base settings for pull and push + Settings settings_common; + /// Settings used to fetch data + Settings settings_pull; + /// Settings used to insert data + Settings settings_push; + + String clusters_prefix; + + /// Subtasks + TasksTable table_tasks; + + std::random_device random_device; + pcg64 random_engine; +}; + + + +struct MultiTransactionInfo +{ + int32_t code; + Coordination::Requests requests; + Coordination::Responses responses; +}; + +// Creates AST representing 'ENGINE = Distributed(cluster, db, table, [sharding_key]) +std::shared_ptr createASTStorageDistributed( + const String & cluster_name, const String & database, const String & table, const ASTPtr & sharding_key_ast = nullptr) +{ + auto args = std::make_shared(); + args->children.emplace_back(std::make_shared(cluster_name)); + args->children.emplace_back(std::make_shared(database)); + args->children.emplace_back(std::make_shared(table)); + if (sharding_key_ast) + args->children.emplace_back(sharding_key_ast); + + auto engine = std::make_shared(); + engine->name = "Distributed"; + engine->arguments = args; + + auto storage = std::make_shared(); + storage->set(storage->engine, engine); + + return storage; +} + + +BlockInputStreamPtr squashStreamIntoOneBlock(const BlockInputStreamPtr & stream) +{ + return std::make_shared( + stream, + std::numeric_limits::max(), + std::numeric_limits::max()); +} + +Block getBlockWithAllStreamData(const BlockInputStreamPtr & stream) +{ + return squashStreamIntoOneBlock(stream)->read(); +} + + +/// Path getters + +String TaskTable::getPartitionPath(const String & partition_name) const +{ + return task_cluster.task_zookeeper_path // root + + "/tables/" + table_id // tables/dst_cluster.merge.hits + + "/" + escapeForFileName(partition_name); // 201701 +} + +String TaskTable::getPartitionPathWithPieceNumber(const String & partition_name, size_t current_piece_number) const +{ + return getPartitionPath(partition_name) + "/" + std::to_string(current_piece_number); // 1...number_of_splits +} + +String ShardPartition::getPartitionCleanStartPath() const +{ + return getPartitionPath() + "/clean_start"; +} + +String ShardPartition::getPartitionPath() const +{ + return task_shard.task_table.getPartitionPath(name); +} + +String ShardPartition::getShardStatusPath() const +{ + // schema: //tables/
//shards/ + // e.g. /root/table_test.hits/201701/shards/1 + return getPartitionShardsPath() + "/" + toString(task_shard.numberInCluster()); +} + +String ShardPartition::getPartitionShardsPath() const +{ + return getPartitionPath() + "/shards"; +} + +String ShardPartition::getPartitionActiveWorkersPath() const +{ + return getPartitionPath() + "/partition_active_workers"; +} + +String ShardPartition::getActiveWorkerPath() const +{ + return getPartitionActiveWorkersPath() + "/" + toString(task_shard.numberInCluster()); +} + +String ShardPartition::getCommonPartitionIsDirtyPath() const +{ + return getPartitionPath() + "/is_dirty"; +} + +String ShardPartition::getCommonPartitionIsCleanedPath() const +{ + return getCommonPartitionIsDirtyPath() + "/cleaned"; +} + +String TaskTable::getPartitionIsDirtyPath(const String & partition_name) const +{ + return getPartitionPath(partition_name) + "/is_dirty"; +} + +String TaskTable::getPartitionIsCleanedPath(const String & partition_name) const +{ + return getPartitionIsDirtyPath(partition_name) + "/cleaned"; +} + +String TaskTable::getPartitionTaskStatusPath(const String & partition_name) const +{ + return getPartitionPath(partition_name) + "/shards"; +} + +String TaskShard::getDescription() const +{ + std::stringstream ss; + ss << "N" << numberInCluster() + << " (having a replica " << getHostNameExample() + << ", pull table " + getQuotedTable(task_table.table_pull) + << " of cluster " + task_table.cluster_pull_name << ")"; + return ss.str(); +} + +String TaskShard::getHostNameExample() const +{ + auto & replicas = task_table.cluster_pull->getShardsAddresses().at(indexInCluster()); + return replicas.at(0).readableString(); +} + + +static bool isExtendedDefinitionStorage(const ASTPtr & storage_ast) +{ + const auto & storage = storage_ast->as(); + return storage.partition_by || storage.order_by || storage.sample_by; +} + +static ASTPtr extractPartitionKey(const ASTPtr & storage_ast) +{ + String storage_str = queryToString(storage_ast); + + const auto & storage = storage_ast->as(); + const auto & engine = storage.engine->as(); + + if (!endsWith(engine.name, "MergeTree")) + { + throw Exception("Unsupported engine was specified in " + storage_str + ", only *MergeTree engines are supported", + ErrorCodes::BAD_ARGUMENTS); + } + + if (isExtendedDefinitionStorage(storage_ast)) + { + if (storage.partition_by) + return storage.partition_by->clone(); + + static const char * all = "all"; + return std::make_shared(Field(all, strlen(all))); + } + else + { + bool is_replicated = startsWith(engine.name, "Replicated"); + size_t min_args = is_replicated ? 3 : 1; + + if (!engine.arguments) + throw Exception("Expected arguments in " + storage_str, ErrorCodes::BAD_ARGUMENTS); + + ASTPtr arguments_ast = engine.arguments->clone(); + ASTs & arguments = arguments_ast->children; + + if (arguments.size() < min_args) + throw Exception("Expected at least " + toString(min_args) + " arguments in " + storage_str, ErrorCodes::BAD_ARGUMENTS); + + ASTPtr & month_arg = is_replicated ? arguments[2] : arguments[1]; + return makeASTFunction("toYYYYMM", month_arg->clone()); + } +} + + +TaskTable::TaskTable(TaskCluster & parent, const Poco::Util::AbstractConfiguration & config, const String & prefix_, + const String & table_key) + : task_cluster(parent) +{ + String table_prefix = prefix_ + "." + table_key + "."; + + name_in_config = table_key; + + number_of_splits = config.getUInt64("number_of_splits", 10); + + cluster_pull_name = config.getString(table_prefix + "cluster_pull"); + cluster_push_name = config.getString(table_prefix + "cluster_push"); + + table_pull.first = config.getString(table_prefix + "database_pull"); + table_pull.second = config.getString(table_prefix + "table_pull"); + + table_push.first = config.getString(table_prefix + "database_push"); + table_push.second = config.getString(table_prefix + "table_push"); + + /// Used as node name in ZooKeeper + table_id = escapeForFileName(cluster_push_name) + + "." + escapeForFileName(table_push.first) + + "." + escapeForFileName(table_push.second); + + engine_push_str = config.getString(table_prefix + "engine"); + { + ParserStorage parser_storage; + engine_push_ast = parseQuery(parser_storage, engine_push_str, 0); + engine_push_partition_key_ast = extractPartitionKey(engine_push_ast); + } + + sharding_key_str = config.getString(table_prefix + "sharding_key"); + { + ParserExpressionWithOptionalAlias parser_expression(false); + sharding_key_ast = parseQuery(parser_expression, sharding_key_str, 0); + engine_split_ast = createASTStorageDistributed(cluster_push_name, table_push.first, table_push.second, sharding_key_ast); + } + + where_condition_str = config.getString(table_prefix + "where_condition", ""); + if (!where_condition_str.empty()) + { + ParserExpressionWithOptionalAlias parser_expression(false); + where_condition_ast = parseQuery(parser_expression, where_condition_str, 0); + + // Will use canonical expression form + where_condition_str = queryToString(where_condition_ast); + } + + String enabled_partitions_prefix = table_prefix + "enabled_partitions"; + has_enabled_partitions = config.has(enabled_partitions_prefix); + + if (has_enabled_partitions) + { + Strings keys; + config.keys(enabled_partitions_prefix, keys); + + if (keys.empty()) + { + /// Parse list of partition from space-separated string + String partitions_str = config.getString(table_prefix + "enabled_partitions"); + boost::trim_if(partitions_str, isWhitespaceASCII); + boost::split(enabled_partitions, partitions_str, isWhitespaceASCII, boost::token_compress_on); + } + else + { + /// Parse sequence of ... + for (const String & key : keys) + { + if (!startsWith(key, "partition")) + throw Exception("Unknown key " + key + " in " + enabled_partitions_prefix, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + + enabled_partitions.emplace_back(config.getString(enabled_partitions_prefix + "." + key)); + } + } + + std::copy(enabled_partitions.begin(), enabled_partitions.end(), std::inserter(enabled_partitions_set, enabled_partitions_set.begin())); + } +} + + +static ShardPriority getReplicasPriority(const Cluster::Addresses & replicas, const std::string & local_hostname, UInt8 random) +{ + ShardPriority res; + + if (replicas.empty()) + return res; + + res.is_remote = 1; + for (auto & replica : replicas) + { + if (isLocalAddress(DNSResolver::instance().resolveHost(replica.host_name))) + { + res.is_remote = 0; + break; + } + } + + res.hostname_difference = std::numeric_limits::max(); + for (auto & replica : replicas) + { + size_t difference = getHostNameDifference(local_hostname, replica.host_name); + res.hostname_difference = std::min(difference, res.hostname_difference); + } + + res.random = random; + return res; +} + +template +void TaskTable::initShards(RandomEngine && random_engine) +{ + const String & fqdn_name = getFQDNOrHostName(); + std::uniform_int_distribution get_urand(0, std::numeric_limits::max()); + + // Compute the priority + for (auto & shard_info : cluster_pull->getShardsInfo()) + { + TaskShardPtr task_shard = std::make_shared(*this, shard_info); + const auto & replicas = cluster_pull->getShardsAddresses().at(task_shard->indexInCluster()); + task_shard->priority = getReplicasPriority(replicas, fqdn_name, get_urand(random_engine)); + + all_shards.emplace_back(task_shard); + } + + // Sort by priority + std::sort(all_shards.begin(), all_shards.end(), + [] (const TaskShardPtr & lhs, const TaskShardPtr & rhs) + { + return ShardPriority::greaterPriority(lhs->priority, rhs->priority); + }); + + // Cut local shards + auto it_first_remote = std::lower_bound(all_shards.begin(), all_shards.end(), 1, + [] (const TaskShardPtr & lhs, UInt8 is_remote) + { + return lhs->priority.is_remote < is_remote; + }); + + local_shards.assign(all_shards.begin(), it_first_remote); +} + + +void TaskCluster::loadTasks(const Poco::Util::AbstractConfiguration & config, const String & base_key) +{ + String prefix = base_key.empty() ? "" : base_key + "."; + + clusters_prefix = prefix + "remote_servers"; + if (!config.has(clusters_prefix)) + throw Exception("You should specify list of clusters in " + clusters_prefix, ErrorCodes::BAD_ARGUMENTS); + + Poco::Util::AbstractConfiguration::Keys tables_keys; + config.keys(prefix + "tables", tables_keys); + + for (const auto & table_key : tables_keys) + { + table_tasks.emplace_back(*this, config, prefix + "tables", table_key); + } +} + +void TaskCluster::reloadSettings(const Poco::Util::AbstractConfiguration & config, const String & base_key) +{ + String prefix = base_key.empty() ? "" : base_key + "."; + + max_workers = config.getUInt64(prefix + "max_workers"); + + settings_common = Settings(); + if (config.has(prefix + "settings")) + settings_common.loadSettingsFromConfig(prefix + "settings", config); + + settings_pull = settings_common; + if (config.has(prefix + "settings_pull")) + settings_pull.loadSettingsFromConfig(prefix + "settings_pull", config); + + settings_push = settings_common; + if (config.has(prefix + "settings_push")) + settings_push.loadSettingsFromConfig(prefix + "settings_push", config); + + auto set_default_value = [] (auto && setting, auto && default_value) + { + setting = setting.changed ? setting.value : default_value; + }; + + /// Override important settings + settings_pull.readonly = 1; + settings_push.insert_distributed_sync = 1; + set_default_value(settings_pull.load_balancing, LoadBalancing::NEAREST_HOSTNAME); + set_default_value(settings_pull.max_threads, 1); + set_default_value(settings_pull.max_block_size, 8192UL); + set_default_value(settings_pull.preferred_block_size_bytes, 0); + set_default_value(settings_push.insert_distributed_timeout, 0); +} + + +} // end of an anonymous namespace +} -#endif //CLICKHOUSE_INTERNALS_H From b26a8b5622d6516c92a0d045ab6d1e63b488f8bb Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Tue, 11 Feb 2020 16:41:26 +0300 Subject: [PATCH 0240/2007] choose part type while selecting parts to merge --- .../Storages/MergeTree/IMergeTreeDataPart.cpp | 21 +-------- .../Storages/MergeTree/IMergeTreeDataPart.h | 3 +- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreeDataMergerMutator.cpp | 21 ++++----- .../MergeTree/MergeTreeDataMergerMutator.h | 1 + .../MergeTree/MergeTreeDataPartCompact.cpp | 2 +- .../MergeTree/MergeTreeDataPartType.cpp | 43 +++++++++++++++++++ .../MergeTree/MergeTreeDataPartType.h | 33 ++++++++++++-- .../MergeTree/ReplicatedMergeTreeLogEntry.cpp | 11 ++++- .../MergeTree/ReplicatedMergeTreeLogEntry.h | 2 + dbms/src/Storages/StorageMergeTree.cpp | 1 + .../Storages/StorageReplicatedMergeTree.cpp | 11 +++-- .../src/Storages/StorageReplicatedMergeTree.h | 1 + 13 files changed, 111 insertions(+), 41 deletions(-) create mode 100644 dbms/src/Storages/MergeTree/MergeTreeDataPartType.cpp diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp index c6dfe4a76cb..a2667712356 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -340,7 +340,7 @@ void IMergeTreeDataPart::assertOnDisk() const { if (!isStoredOnDisk()) throw Exception("Data part '" + name + "' with type '" - + typeToString(getType()) + "' is not stored on disk", ErrorCodes::LOGICAL_ERROR); + + getType().toString() + "' is not stored on disk", ErrorCodes::LOGICAL_ERROR); } @@ -416,7 +416,7 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks void IMergeTreeDataPart::loadIndexGranularity() { - throw Exception("Method 'loadIndexGranularity' is not implemented for part with type " + typeToString(getType()), ErrorCodes::NOT_IMPLEMENTED); + throw Exception("Method 'loadIndexGranularity' is not implemented for part with type " + getType().toString(), ErrorCodes::NOT_IMPLEMENTED); } void IMergeTreeDataPart::loadIndex() @@ -779,23 +779,6 @@ void IMergeTreeDataPart::remove() const } } -String IMergeTreeDataPart::typeToString(Type type) -{ - switch (type) - { - case Type::WIDE: - return "Wide"; - case Type::COMPACT: - return "Compact"; - case Type::IN_MEMORY: - return "InMemory"; - case Type::UNKNOWN: - return "Unknown"; - } - - __builtin_unreachable(); -} - String IMergeTreeDataPart::getRelativePathForDetachedPart(const String & prefix) const { /// Do not allow underscores in the prefix because they are used as separators. diff --git a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h index fb91a46ca74..65b22d90734 100644 --- a/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/dbms/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -113,8 +113,7 @@ public: Type getType() const { return part_type; } - static String typeToString(Type type); - String getTypeName() const { return typeToString(getType()); } + String getTypeName() const { return getType().toString(); } void setColumns(const NamesAndTypesList & new_columns); diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 7ef141327fc..2673a9826f5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1704,7 +1704,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::createPart( const String & name, const MergeTreePartInfo & part_info, const DiskPtr & disk, const String & relative_path) const { - auto type = MergeTreeDataPartType::UNKNOWN; + MergeTreeDataPartType type; auto full_path = getFullPathOnDisk(disk) + relative_path + "/"; auto mrk_ext = MergeTreeIndexGranularityInfo::getMrkExtensionFromFS(full_path); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index 6ad04e4e0f2..6d93809817e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -89,19 +89,25 @@ void FutureMergedMutatedPart::assign(MergeTreeData::DataPartsVector parts_) UInt32 max_level = 0; Int64 max_mutation = 0; + size_t sum_rows = 0; + size_t sum_bytes_uncompressed = 0; for (const auto & part : parts) { max_level = std::max(max_level, part->info.level); max_mutation = std::max(max_mutation, part->info.mutation); + sum_rows += part->rows_count; + sum_bytes_uncompressed += part->getTotalColumnsSize().data_uncompressed; } + const auto & storage = parts.front()->storage; + type = storage.choosePartType(sum_bytes_uncompressed, sum_rows); part_info.partition_id = parts.front()->info.partition_id; part_info.min_block = parts.front()->info.min_block; part_info.max_block = parts.back()->info.max_block; part_info.level = max_level + 1; part_info.mutation = max_mutation; - if (parts.front()->storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) + if (storage.format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { DayNum min_date = DayNum(std::numeric_limits::max()); DayNum max_date = DayNum(std::numeric_limits::min()); @@ -576,20 +582,14 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor all_columns, data.sorting_key_expr, data.skip_indices, data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names); - size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; - size_t estimated_bytes_uncompressed = 0; - for (const auto & part : parts) - estimated_bytes_uncompressed += part->getTotalColumnsSize().data_uncompressed; - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart( future_part.name, + future_part.type, future_part.part_info, space_reservation->getDisk(), - all_columns, - estimated_bytes_uncompressed, - sum_input_rows_upper_bound, TMP_PREFIX + future_part.name); + new_data_part->setColumns(all_columns); new_data_part->partition.assign(future_part.getPartition()); new_data_part->is_temp = true; @@ -607,6 +607,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor need_remove_expired_values = false; } + size_t sum_input_rows_upper_bound = merge_entry->total_rows_count; MergeAlgorithm merge_alg = chooseMergeAlgorithm(parts, sum_input_rows_upper_bound, gathering_columns, deduplicate, need_remove_expired_values); LOG_DEBUG(log, "Selected MergeAlgorithm: " << ((merge_alg == MergeAlgorithm::Vertical) ? "Vertical" : "Horizontal")); @@ -990,7 +991,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mutatePartToTempor auto new_data_part = data.createPart( future_part.name, - source_part->getType(), + future_part.type, future_part.part_info, space_reservation->getDisk(), "tmp_mut_" + future_part.name); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index e475037f32b..36e0687a19b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -18,6 +18,7 @@ struct FutureMergedMutatedPart { String name; String path; + MergeTreeDataPartType type; MergeTreePartInfo part_info; MergeTreeData::DataPartsVector parts; diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp index 8f36ce5e828..aab80365fe1 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartCompact.cpp @@ -78,7 +78,7 @@ ColumnSize MergeTreeDataPartCompact::getTotalColumnsSize() const if (bin_checksum != checksums.files.end()) { total_size.data_compressed += bin_checksum->second.file_size; - total_size.data_compressed += bin_checksum->second.uncompressed_size; + total_size.data_uncompressed += bin_checksum->second.uncompressed_size; } auto mrk_checksum = checksums.files.find(DATA_FILE_NAME + index_granularity_info.marks_file_extension); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.cpp new file mode 100644 index 00000000000..f8a38af5496 --- /dev/null +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.cpp @@ -0,0 +1,43 @@ +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_PART_TYPE; +} + +void MergeTreeDataPartType::fromString(const String & str) +{ + if (str == "Wide") + value = WIDE; + else if (str == "Compact") + value = COMPACT; + else if (str == "InMemory") + value = IN_MEMORY; + else if (str == "Unknown") + value = UNKNOWN; + else + throw DB::Exception("Unexpected string for part type: " + str, ErrorCodes::UNKNOWN_PART_TYPE); +} + +String MergeTreeDataPartType::toString() const +{ + switch (value) + { + case WIDE: + return "Wide"; + case COMPACT: + return "Compact"; + case IN_MEMORY: + return "InMemory"; + case UNKNOWN: + return "Unknown"; + } + + __builtin_unreachable(); +} + +} diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h index ed229414e3b..a7d176f7baa 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -1,11 +1,17 @@ #pragma once +#include + namespace DB { - /// Types of data part format. - enum class MergeTreeDataPartType + +/// Types of data part format. +class MergeTreeDataPartType +{ +public: + enum Value { - /// Data of each is stored in one or several (for complex types) files. + /// Data of each column is stored in one or several (for complex types) files. /// Every data file is followed by marks file. WIDE, @@ -17,4 +23,25 @@ namespace DB UNKNOWN, }; + + MergeTreeDataPartType() : value(UNKNOWN) {} + MergeTreeDataPartType(Value value_) : value(value_) {} + + bool operator==(const MergeTreeDataPartType & other) const + { + return value == other.value; + } + + bool operator!=(const MergeTreeDataPartType & other) const + { + return !(*this == other); + } + + void fromString(const String & str); + String toString() const; + +private: + Value value; +}; + } diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp index 29878cc064e..b871f75130b 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp @@ -12,7 +12,7 @@ namespace DB void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const { - out << "format version: 4\n" + out << "format version: 5\n" << "create_time: " << LocalDateTime(create_time ? create_time : time(nullptr)) << "\n" << "source replica: " << source_replica << '\n' << "block_id: " << escape << block_id << '\n'; @@ -29,6 +29,7 @@ void ReplicatedMergeTreeLogEntryData::writeText(WriteBuffer & out) const out << s << '\n'; out << "into\n" << new_part_name; out << "\ndeduplicate: " << deduplicate; + out << "\npart_type: " << new_part_type.toString(); break; case DROP_RANGE: @@ -82,7 +83,7 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) in >> "format version: " >> format_version >> "\n"; - if (format_version < 1 || format_version > 4) + if (format_version < 1 || format_version > 5) throw Exception("Unknown ReplicatedMergeTreeLogEntry format version: " + DB::toString(format_version), ErrorCodes::UNKNOWN_FORMAT_VERSION); if (format_version >= 2) @@ -120,6 +121,12 @@ void ReplicatedMergeTreeLogEntryData::readText(ReadBuffer & in) in >> new_part_name; if (format_version >= 4) in >> "\ndeduplicate: " >> deduplicate; + if (format_version >= 5) + { + String part_type_str; + in >> "\npart_type: " >> type_str; + new_part_type.fromString(type_str); + } } else if (type_str == "drop" || type_str == "detach") { diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h index ca8c9315fa9..d761e958828 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -72,6 +73,7 @@ struct ReplicatedMergeTreeLogEntryData /// The name of resulting part for GET_PART and MERGE_PARTS /// Part range for DROP_RANGE and CLEAR_COLUMN String new_part_name; + MergeTreeDataPartType new_part_type; String block_id; /// For parts of level zero, the block identifier for deduplication (node name in /blocks/). mutable String actual_new_part_name; /// GET_PART could actually fetch a part covering 'new_part_name'. diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 199b6bfc149..1e5f72f9307 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -745,6 +745,7 @@ bool StorageMergeTree::tryMutatePart() future_part.parts.push_back(part); future_part.part_info = new_part_info; future_part.name = part->getNewName(new_part_info); + future_part.type = part->getType(); tagger.emplace(future_part, MergeTreeDataMergerMutator::estimateNeededDiskSpace({part}), *this, true); break; diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 439849fd8e5..5267f19b7df 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1220,6 +1220,7 @@ bool StorageReplicatedMergeTree::tryExecutePartMutation(const StorageReplicatedM future_mutated_part.part_info = new_part_info; future_mutated_part.name = entry.new_part_name; future_mutated_part.updatePath(*this, reserved_space); + future_mutated_part.type = source_part->getType(); auto table_id = getStorageID(); MergeList::EntryPtr merge_entry = global_context.getMergeList().insert( @@ -2279,7 +2280,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, merge_pred)) { success = createLogEntryToMergeParts(zookeeper, future_merged_part.parts, - future_merged_part.name, deduplicate, force_ttl); + future_merged_part.name, future_merged_part.type, deduplicate, force_ttl); } /// If there are many mutations in queue it may happen, that we cannot enqueue enough merges to merge all new parts else if (max_source_part_size_for_mutation > 0 && queue.countMutations() > 0 @@ -2344,6 +2345,7 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( zkutil::ZooKeeperPtr & zookeeper, const DataPartsVector & parts, const String & merged_name, + const MergeTreeDataPartType & merged_part_type, bool deduplicate, bool force_ttl, ReplicatedMergeTreeLogEntryData * out_log_entry) @@ -2380,12 +2382,15 @@ bool StorageReplicatedMergeTree::createLogEntryToMergeParts( entry.type = LogEntry::MERGE_PARTS; entry.source_replica = replica_name; entry.new_part_name = merged_name; + entry.new_part_type = merged_part_type; entry.deduplicate = deduplicate; entry.force_ttl = force_ttl; entry.create_time = time(nullptr); for (const auto & part : parts) + { entry.source_parts.push_back(part->name); + } String path_created = zookeeper->create(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential); entry.znode_name = path_created.substr(path_created.find_last_of('/') + 1); @@ -3152,7 +3157,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p future_merged_part, disk_space, can_merge, partition_id, true, nullptr); ReplicatedMergeTreeLogEntryData merge_entry; if (selected && !createLogEntryToMergeParts(zookeeper, future_merged_part.parts, - future_merged_part.name, deduplicate, force_ttl, &merge_entry)) + future_merged_part.name, future_merged_part.type, deduplicate, force_ttl, &merge_entry)) return handle_noop("Can't create merge queue node in ZooKeeper"); if (merge_entry.type != ReplicatedMergeTreeLogEntryData::Type::EMPTY) merge_entries.push_back(std::move(merge_entry)); @@ -3189,7 +3194,7 @@ bool StorageReplicatedMergeTree::optimize(const ASTPtr & query, const ASTPtr & p ReplicatedMergeTreeLogEntryData merge_entry; if (!createLogEntryToMergeParts(zookeeper, future_merged_part.parts, - future_merged_part.name, deduplicate, force_ttl, &merge_entry)) + future_merged_part.name, future_merged_part.type, deduplicate, force_ttl, &merge_entry)) return handle_noop("Can't create merge queue node in ZooKeeper"); if (merge_entry.type != ReplicatedMergeTreeLogEntryData::Type::EMPTY) merge_entries.push_back(std::move(merge_entry)); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.h b/dbms/src/Storages/StorageReplicatedMergeTree.h index 9c7c36a19b8..080a19b66d5 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.h +++ b/dbms/src/Storages/StorageReplicatedMergeTree.h @@ -438,6 +438,7 @@ private: zkutil::ZooKeeperPtr & zookeeper, const DataPartsVector & parts, const String & merged_name, + const MergeTreeDataPartType & merged_part_type, bool deduplicate, bool force_ttl, ReplicatedMergeTreeLogEntryData * out_log_entry = nullptr); From 68eea2ba18a43532e3fb11684157744593525a6e Mon Sep 17 00:00:00 2001 From: Yuriy Date: Tue, 11 Feb 2020 16:58:49 +0300 Subject: [PATCH 0241/2007] send string columns to mysql client as utf8 --- dbms/src/Core/MySQLProtocol.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/src/Core/MySQLProtocol.cpp b/dbms/src/Core/MySQLProtocol.cpp index 82af8f290a1..c303101d7cc 100644 --- a/dbms/src/Core/MySQLProtocol.cpp +++ b/dbms/src/Core/MySQLProtocol.cpp @@ -103,6 +103,7 @@ size_t getLengthEncodedStringSize(const String & s) ColumnDefinition getColumnDefinition(const String & column_name, const TypeIndex type_index) { ColumnType column_type; + CharacterSet charset = CharacterSet::binary; int flags = 0; switch (type_index) { @@ -157,12 +158,14 @@ ColumnDefinition getColumnDefinition(const String & column_name, const TypeIndex case TypeIndex::String: case TypeIndex::FixedString: column_type = ColumnType::MYSQL_TYPE_STRING; + charset = CharacterSet::utf8_general_ci; break; default: column_type = ColumnType::MYSQL_TYPE_STRING; + charset = CharacterSet::utf8_general_ci; break; } - return ColumnDefinition(column_name, CharacterSet::binary, 0, column_type, flags, 0); + return ColumnDefinition(column_name, charset, 0, column_type, flags, 0); } } From a7ec7a6c4ddfde0a2be304bbdd097df434aa6446 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 11 Feb 2020 17:05:51 +0300 Subject: [PATCH 0242/2007] pcg64 -> pcg64_oneseq --- .../TableFunctions/TableFunctionRandom.cpp | 13 +- .../01072_random_table_function.reference | 146 +++++++++--------- .../01072_random_table_function.sql | 84 +++++----- 3 files changed, 122 insertions(+), 121 deletions(-) diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index a08dbe6b691..f23b98bfbaa 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -92,7 +92,7 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64 generator(random_seed); + pcg64_oneseq generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { data[i] = static_cast(generator()); @@ -138,7 +138,7 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64 generator(random_seed); + pcg64_oneseq generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { data[i] = static_cast(generator()); @@ -164,7 +164,7 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64 generator(random_seed); + pcg64_oneseq generator(random_seed); double d = 1.0; for (UInt64 i = 0; i < limit; ++i) { @@ -306,7 +306,7 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64 generator(random_seed); + pcg64_oneseq generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { data[i] = static_cast(generator()); @@ -317,7 +317,7 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64 generator(random_seed); + pcg64_oneseq generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { Int128 x = static_cast(generator()) << 64 | static_cast(generator()); @@ -329,7 +329,7 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64 generator(random_seed); + pcg64_oneseq generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { auto x = UInt128(generator(), generator()); @@ -467,3 +467,4 @@ void registerTableFunctionRandom(TableFunctionFactory & factory) } } + diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.reference b/dbms/tests/queries/0_stateless/01072_random_table_function.reference index 3906b417524..2770d1fcaf3 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.reference +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.reference @@ -1,3 +1,15 @@ +UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 +2254772619926532955 2254772619926532955 1234817989 1234817989 54213 -11323 197 -59 +9120028858397505560 9120028858397505560 1171957426 1171957426 42674 -22862 178 -78 +4555697903102013946 4555697903102013946 275100647 275100647 46055 -19481 231 -25 +5784362079052877875 5784362079052877875 1033685688 1033685688 51896 -13640 184 -72 +11035971995277520997 -7410772078432030619 180895192 180895192 15832 15832 216 -40 +7901646768096461004 7901646768096461004 135557292 135557292 28844 28844 172 -84 +6733841386518201279 6733841386518201279 716914271 716914271 15967 15967 95 95 +7736560050027905187 7736560050027905187 1012211222 1012211222 7702 7702 22 22 +2199287578947862030 2199287578947862030 2185722662 -2109244634 31526 31526 38 38 +3019483913099890467 3019483913099890467 2647224658 -1647742638 29010 29010 82 82 +- Enum8(\'hello\' = 1, \'world\' = 5) world hello @@ -34,18 +46,6 @@ h w o - -UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 -2254772619926532955 2254772619926532955 1234817989 1234817989 54213 -11323 197 -59 -9120028858397505560 9120028858397505560 1171957426 1171957426 42674 -22862 178 -78 -4555697903102013946 4555697903102013946 275100647 275100647 46055 -19481 231 -25 -5784362079052877875 5784362079052877875 1033685688 1033685688 51896 -13640 184 -72 -11035971995277520997 -7410772078432030619 180895192 180895192 15832 15832 216 -40 -7901646768096461004 7901646768096461004 135557292 135557292 28844 28844 172 -84 -6733841386518201279 6733841386518201279 716914271 716914271 15967 15967 95 95 -7736560050027905187 7736560050027905187 1012211222 1012211222 7702 7702 22 22 -2199287578947862030 2199287578947862030 2185722662 -2109244634 31526 31526 38 38 -3019483913099890467 3019483913099890467 2647224658 -1647742638 29010 29010 82 82 -- Date DateTime DateTime(\'Europe/Moscow\') 2106-02-07 2009-02-16 23:59:49 2009-02-16 23:59:49 2086-11-02 2007-02-20 10:43:46 2007-02-20 10:43:46 @@ -94,18 +94,6 @@ Decimal32(4) Decimal64(8) Decimal64(8) -210924.4634 21992875789.47862030 13765369952377767241248.9441272127848016 -164774.2638 30194839130.99890467 -13890064946313418575619.0315227826809939 - -UUID -7e90d9ed-3e18-3818-1f4a-8fc063ff735b -50462ea9-fe2f-d033-3f39-171b126331fa -6da849de-f100-c0cc-9927-a60f01acf065 -6b5dc860-1d64-40a3-5d73-6910493dc3bf -29e75c9e-fd53-4f23-1e85-7066961dbe0e -b60faa27-fe38-ff30-bda6-6d4f737b3622 -297f7b2f-ec54-178b-623d-6d8244222885 -556e7945-df65-729e-dcb0-e0ca3a435f2e -8d6df922-9588-7e50-678f-236036acd439 -bdb39798-af19-13ad-9780-b53edc0f4a21 -- Tuple(Int32, Int64) (1234817989,2254772619926532955) (1171957426,9120028858397505560) @@ -142,18 +130,6 @@ Array(Nullable(Int32)) [733724312,-23652950] [371735004,462118779,148602156,-1055384004,-1041274619,247762201,522289659,822210177] - -Array(Nullable(UUID)) -['7e90d9ed-3e18-3818-1f4a-8fc063ff735b','50462ea9-fe2f-d033-3f39-171b126331fa','6da849de-f100-c0cc-9927-a60f01acf065','6b5dc860-1d64-40a3-5d73-6910493dc3bf','29e75c9e-fd53-4f23-1e85-7066961dbe0e','b60faa27-fe38-ff30-bda6-6d4f737b3622','297f7b2f-ec54-178b-623d-6d8244222885','556e7945-df65-729e-dcb0-e0ca3a435f2e','8d6df922-9588-7e50-678f-236036acd439'] -['bdb39798-af19-13ad-9780-b53edc0f4a21','5ca17a81-ab30-2b25-c798-10de3635d333','a766c8bc-be3b-a400-1c75-6bca44383f17','167b3c3c-66d3-5a18-d907-2738ac937ed6','aaf2f78f-d92d-f3ce-b1e8-dec2de293c9f','5a460194-f0be-04dd-9cd2-5f9f3c0df43d'] -['c78566cc-f112-f7d5-10a4-718dab8c49c6','9a1805ce-8d1f-b583-02ac-2bf55634a5a8','f831fa23-87f5-c29a-8037-a13d200408f2','ecbb216a-dbd7-48c7-a99c-437311212691','2f211de3-ea53-af08-ef09-86ff50310353','c77b4d76-9b12-b917-7780-64a7653bef7b','b0b3e454-dae7-bef7-a160-7e6f691a0ff0'] -['eecd18dc-5852-84fd-71c1-b47ac0eb42b5','f339b5da-c55c-037b-72bb-f2729ec509ec','84c6ef50-9f4c-45eb-26e5-bce543f759b0','a8c5d648-f236-e754-305f-cbffc3662033','9e879501-a713-e63d-3a0d-329ff89784e9','c0783c4f-d81f-4f55-54bd-a20cd5cda08a','ed32b485-0648-bdc8-43f5-49d13e5bd5bf','63303c7a-fa5f-2644-7eb6-ac4f06e0ff48'] -['5af43b6e-67ca-62c9-17b9-a4a5fef8a3f9','c210e199-2065-50db-3f52-4d8e320d00dc'] -['1fcb5a9e-82f3-9f97-005c-592e50819f3d','3effe804-94a7-9260-29cf-228db3254a34'] -['f0878b54-f5e7-2d0e-6c08-b54b8cf8b96d'] -['6b71c084-6cf0-204d-7122-e162ab8ba84a','39ede902-3205-610c-51c1-de1a24c718d6'] -['e7c99a0c-ad54-4afe-f09d-67791106d667','5da402c8-f4d5-0dc9-6206-0fecee137c66'] -['2070622f-c9d8-2203-df1d-0d54d6399c9b','b8ac67ea-31b9-9c3e-f23e-f5b937979b0e','afedd53b-6ea2-04f4-e48a-fe739e227439','283efb3f-16ae-beba-d7f1-ab4749287623','add23ae2-d111-3ccb-ea27-0407d32fa407','f1bbff94-d3cf-fbc2-c43e-9fff2980a1d1','88ad4f5d-29c1-5b06-a0cd-54e60a2d07ec','17abe221-5b8b-19e0-5e93-413f2eb95363'] -- Tuple(Int32, Array(Int64)) (1234817989,[2254772619926532955,9120028858397505560,4555697903102013946,5784362079052877875,-7410772078432030619,7901646768096461004,6733841386518201279,7736560050027905187,2199287578947862030]) (1171957426,[3019483913099890467,-4781013766399904222,-5327852745410412752,7078934595552553093,2990244123355912075,-2544286630298820818]) @@ -166,6 +142,54 @@ Tuple(Int32, Array(Int64)) (-2109244634,[-1388479317275096889,-1222297392734207149]) (-1647742638,[3396028458740199176,8610993157653131131,-4072576266223306473,-6818310818869145616,-5713972449102020873,8197031236106666677,-1239306987803343619,8267468115072584172]) - +Nullable(String) +)/VC)%f9 +\0ih|;B +\0J"Z,kd +\0m"m]$35 +\00 +\0( +\0 +\0g +\0> +\0XjbW:s< +- +Array(String) +['(|ZVAg2F','\0GXjbW','\0<^guT(','\0y M$lZ0','\03','\0p','\0','\0i','\0P'] +['\0"}YRG%B','\0T3(E^> p','\0JTaj','\0)*3','\0k%=p','\0Yub$81`X'] +['','\0\\p]|]','\05','\0k$C/pnA'] +['\0ryz{*p',''] +['\07`mjt*G',''] +['\0~g'] +['\0k','\0 '] +['\0F','\0&h \0XjbW:s< - -Nullable(String) -)/VC)%f9 -\0ih|;B -\0J"Z,kd -\0m"m]$35 -\00 -\0( -\0 -\0g -\0> -\0XjbW:s< +[77] -124167.6723 ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') +[32,110] -141397.7312 ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') +[68] -67417.0770 ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') - -Array(String) -['(|ZVAg2F','\0GXjbW','\0<^guT(','\0y M$lZ0','\03','\0p','\0','\0i','\0P'] -['\0"}YRG%B','\0T3(E^> p','\0JTaj','\0)*3','\0k%=p','\0Yub$81`X'] -['','\0\\p]|]','\05','\0k$C/pnA'] -['\0ryz{*p',''] -['\07`mjt*G',''] -['\0~g'] -['\0k','\0 '] -['\0F','\0&h -210924.4634 w 2.143274805821858e307 ('2056-04-25','2039-04-06 20:11:02','2063-07-18 01:46:10.215','8d6df922-9588-7e50-678f-236036acd439') w -[-36,123,44,60,5,25,-5,-127] 2647224658 \0XjbW:s< -164774.2638 o 2.9425818885529257e307 ('2049-06-05','2053-11-20 07:10:58','1996-11-02 14:35:41.110','bdb39798-af19-13ad-9780-b53edc0f4a21') \r +[-59,-78,-25,-72,-40,-84,95,22,38] 1234817989 )/VC)%f9 123481.7989 o 2.1973467205491123e307 ('2106-02-07','2009-02-16 23:59:49','2007-02-20 10:43:46.989','1f4a8fc0-63ff-735b-7e90-d9ed3e183818') Ų +[82,65,35,-110,-57,-69] 1171957426 \0ih|;B 117195.7426 w 8.887754501811354e307 ('2086-11-02','2007-02-20 10:43:46','2002-10-04 02:54:48.647','3f39171b-1263-31fa-5046-2ea9fe2fd033') +[72,119,-78,-58,13,39,-71] 275100647 \0J"Z,kd 27510.0647 w 4.4396706606805647e307 ('2096-02-04','1978-09-20 03:50:47','1974-04-19 01:48:12.192','9927a60f-01ac-f065-6da8-49def100c0cc') ج +[81,107,-11,-63,-59,69,-80,-122] 1033685688 \0m"m]$35 103368.5688 w 5.637042481600483e307 ('2106-02-07','2002-10-04 02:54:48','2002-01-28 12:47:02.271','5d736910-493d-c3bf-6b5d-c8601d6440a3') _ +[87,-76] 180895192 \00 18089.5192 h 1.07549012514996e308 ('2013-05-07','1975-09-25 19:39:52','2053-11-20 07:10:58.662','1e857066-961d-be0e-29e7-5c9efd534f23') &R +[22,-84] 135557292 \0( 13555.7292 w 7.700402896226395e307 ('2048-12-21','1974-04-19 01:48:12','1986-04-08 19:07:15.849','bda66d4f-737b-3622-b60f-aa27fe38ff30') A# +[-45] 716914271 \0 71691.4271 h 6.562339881458101e307 ('2013-09-19','1992-09-19 18:51:11','2081-03-06 04:00:55.914','623d6d82-4422-2885-297f-7b2fec54178b') +[-40,84] 1012211222 \0g 101221.1222 h 7.539520705557441e307 ('1991-02-02','2002-01-28 12:47:02','1979-01-20 20:39:20.939','dcb0e0ca-3a43-5f2e-556e-7945df65729e') H +[-104,-86] 2185722662 \0> -210924.4634 w 2.143274805821858e307 ('2056-04-25','2039-04-06 20:11:02','2063-07-18 01:46:10.215','678f2360-36ac-d439-8d6d-f92295887e50') w +[-36,123,44,60,5,25,-5,-127] 2647224658 \0XjbW:s< -164774.2638 o 2.9425818885529257e307 ('2049-06-05','2053-11-20 07:10:58','1996-11-02 14:35:41.110','9780b53e-dc0f-4a21-bdb3-9798af1913ad') \r - diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.sql b/dbms/tests/queries/0_stateless/01072_random_table_function.sql index ea80c38df66..dc212a7b8ff 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.sql +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.sql @@ -1,3 +1,16 @@ +SELECT + toTypeName(ui64), toTypeName(i64), + toTypeName(ui32), toTypeName(i32), + toTypeName(ui16), toTypeName(i16), + toTypeName(ui8), toTypeName(i8) +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 1); +SELECT + ui64, i64, + ui32, i32, + ui16, i16, + ui8, i8 +FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 10, 1); +SELECT '-'; SELECT toTypeName(i) FROM generate('i Enum8(\'hello\' = 1, \'world\' = 5)', 1); @@ -20,19 +33,6 @@ SELECT FROM generate('i Nullable(Enum16(\'h\' = 1, \'w\' = 5 , \'o\' = -200)))', 10, 10, 10, 1); SELECT '-'; SELECT -toTypeName(ui64), toTypeName(i64), -toTypeName(ui32), toTypeName(i32), -toTypeName(ui16), toTypeName(i16), -toTypeName(ui8), toTypeName(i8) -FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 1); -SELECT -ui64, i64, -ui32, i32, -ui16, i16, -ui8, i8 -FROM generate('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 10, 10, 10, 1); -SELECT '-'; -SELECT toTypeName(d), toTypeName(dt), toTypeName(dtm) FROM generate('d Date, dt DateTime, dtm DateTime(\'Europe/Moscow\')', 1); SELECT @@ -60,13 +60,6 @@ SELECT d32, d64, d128 FROM generate('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 10, 10, 10, 1); SELECT '-'; -SELECT - toTypeName(i) -FROM generate('i UUID', 1); -SELECT - i -FROM generate('i UUID', 10, 10, 10, 1); -SELECT '-'; SELECT toTypeName(i) FROM generate('i Tuple(Int32, Int64)', 1); @@ -88,13 +81,6 @@ SELECT i FROM generate('i Array(Nullable(Int32))', 10, 10, 10, 1); SELECT '-'; -SELECT - toTypeName(i) -FROM generate('i Array(Nullable(UUID))', 1); -SELECT - i -FROM generate('i Array(Nullable(UUID))', 10, 10, 10, 1); -SELECT '-'; SELECT toTypeName(i) FROM generate('i Tuple(Int32, Array(Int64))', 1); @@ -102,20 +88,6 @@ SELECT i FROM generate('i Tuple(Int32, Array(Int64))', 10, 10, 10, 1); SELECT '-'; -SELECT - toTypeName(i) -FROM generate('i FixedString(4)', 1); -SELECT - i -FROM generate('i FixedString(4)', 10, 10, 10, 1); -SELECT '-'; -SELECT - toTypeName(i) -FROM generate('i String', 10); -SELECT - i -FROM generate('i String', 10, 10, 10, 1); -SELECT '-'; SELECT toTypeName(i) FROM generate('i Nullable(String)', 1); @@ -131,7 +103,34 @@ SELECT FROM generate('i Array(String)', 10, 10, 10, 1); SELECT '-'; - +SELECT + toTypeName(i) +FROM generate('i UUID', 1); +SELECT + i +FROM generate('i UUID', 10, 10, 10, 1); +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i Array(Nullable(UUID))', 1); +SELECT + i +FROM generate('i Array(Nullable(UUID))', 10, 10, 10, 1); +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i FixedString(4)', 1); +SELECT + i +FROM generate('i FixedString(4)', 10, 10, 10, 1); +SELECT '-'; +SELECT + toTypeName(i) +FROM generate('i String', 10); +SELECT + i +FROM generate('i String', 10, 10, 10, 1); +SELECT '-'; DROP TABLE IF EXISTS test_table; CREATE TABLE test_table(a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)) ENGINE=Memory; INSERT INTO test_table SELECT * FROM generate('a Array(Int8), d Decimal32(4), c Tuple(DateTime64(3), UUID)', 3, 2, 10, 1); @@ -150,3 +149,4 @@ SELECT * FROM test_table_2; SELECT '-'; DROP TABLE IF EXISTS test_table_2; + From 9e0d4b0bd4cc88e23a8f1e8e1113d597a425702f Mon Sep 17 00:00:00 2001 From: alesapin Date: Tue, 11 Feb 2020 17:54:46 +0300 Subject: [PATCH 0243/2007] Seems to work --- .../ReplicatedMergeTreeBlockOutputStream.cpp | 2 +- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 11 +- .../MergeTree/ReplicatedQueueAlterState.h | 22 +++- .../Storages/StorageReplicatedMergeTree.cpp | 40 +++++-- ..._parallel_alter_modify_zookeeper.reference | 17 +++ .../01079_parallel_alter_modify_zookeeper.sh | 109 ++++++++++++++++++ 6 files changed, 189 insertions(+), 12 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference create mode 100755 dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp index bddf8434e1c..bf410011fef 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp @@ -256,7 +256,7 @@ void ReplicatedMergeTreeBlockOutputStream::commitPart(zkutil::ZooKeeperPtr & zoo zkutil::CreateMode::PersistentSequential)); ops.emplace_back(zkutil::makeCheckRequest(storage.zookeeper_path + "/columns", storage.getMetadataVersion())); - ops.emplace_back(zkutil::makeSetRequest(storage.zookeeper_path + "/alter_intention_counter", "", -1)); + ops.emplace_back(zkutil::makeSetRequest(storage.zookeeper_path + "/block_numbers/" + part->info.partition_id, "", -1)); /// Deletes the information that the block number is used for writing. block_number_lock->getUnlockOps(ops); diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 8bfa974ce45..6b0322d1e7e 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -287,6 +287,8 @@ void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) for (auto it = from_it; it != in_partition->second.end(); ++it) { MutationStatus & status = *it->second; + + LOG_DEBUG(log, "Removing part name:" << part_name << " from mutation:" << status.entry->znode_name); status.parts_to_do.removePartAndCoveredParts(part_name); if (status.parts_to_do.size() == 0) some_mutations_are_probably_done = true; @@ -695,6 +697,7 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C for (const ReplicatedMergeTreeMutationEntryPtr & entry : new_mutations) { + LOG_DEBUG(log, "PROCESSING MUTATION:" << entry->znode_name); auto & mutation = mutations_by_znode.emplace(entry->znode_name, MutationStatus(entry, format_version)) .first->second; @@ -972,6 +975,9 @@ bool ReplicatedMergeTreeQueue::addFuturePartIfNotCoveredByThem(const String & pa { std::lock_guard lock(state_mutex); + if (!alter_sequence.canExecuteGetEntry(part_name, format_version, lock)) + return false; + if (isNotCoveredByFuturePartsImpl(part_name, reject_reason, lock)) { CurrentlyExecuting::setActualPartName(entry, part_name, *this); @@ -991,7 +997,10 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { if (entry.type == LogEntry::GET_PART) { - if (!alter_sequence.canExecuteGetEntry(entry.new_part_name, format_version, state_lock)) + if (!entry.actual_new_part_name.empty() && !alter_sequence.canExecuteGetEntry(entry.actual_new_part_name, format_version, state_lock)) + return false; + + if (!entry.new_part_name.empty() && !alter_sequence.canExecuteGetEntry(entry.new_part_name, format_version, state_lock)) return false; } diff --git a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h index b359ce0dc14..7e10a81247c 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h +++ b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h @@ -47,7 +47,10 @@ public: void addMutationForAlter(int alter_version, const std::map & block_numbers, std::lock_guard & /*state_lock*/) { LOG_DEBUG(log, "Adding mutation with alter version:" << alter_version); - queue_state.emplace(alter_version, AlterInQueue(block_numbers, true)); + if (queue_state.count(alter_version)) + queue_state[alter_version].block_numbers = block_numbers; + else + queue_state.emplace(alter_version, AlterInQueue(block_numbers, true)); } void addMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) @@ -65,8 +68,19 @@ public: return true; MergeTreePartInfo info = MergeTreePartInfo::fromPartName(part_name, format_version); - if (queue_state.begin()->second.block_numbers.count(info.partition_id)) - return info.getDataVersion() < queue_state.begin()->second.block_numbers.at(info.partition_id); + LOG_DEBUG(log, "Checking can fetch:" << part_name); + + for (const auto & [block_number, state] : queue_state) + { + + LOG_DEBUG(log, "Looking at block:" << block_number << " with part name:" << part_name); + if (state.block_numbers.count(info.partition_id)) + { + LOG_DEBUG(log, "Block number:" << block_number << " has part name " << part_name << " version " << state.block_numbers.at(info.partition_id)); + return info.getDataVersion() < state.block_numbers.at(info.partition_id); + } + } + LOG_DEBUG(log, "Nobody has block number for part " << part_name); return true; } @@ -117,6 +131,8 @@ public: { LOG_DEBUG(log, "Key:" << key << " is metadata finished:" << value.metadata_finished); } + if (alter_version < queue_state.begin()->first) + return true; return queue_state.at(alter_version).metadata_finished; } bool canExecuteMetaAlter(int alter_version, std::lock_guard & /*state_lock*/) const diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index cc1311a8760..9bfefeb5e00 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2265,6 +2265,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() LOG_DEBUG(log, "I'm not leader, I don't want to assign anything"); return; } + LOG_DEBUG(log, "Merge selecting started"); const auto storage_settings_ptr = getSettings(); const bool deduplicate = false; /// TODO: read deduplicate option from table config @@ -2304,6 +2305,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (max_source_parts_size_for_merge > 0 && merger_mutator.selectPartsToMerge(future_merged_part, false, max_source_parts_size_for_merge, merge_pred)) { + LOG_DEBUG(log, "ASSIGNING MERGE"); success = createLogEntryToMergeParts(zookeeper, future_merged_part.parts, future_merged_part.name, deduplicate, force_ttl); } @@ -2337,6 +2339,10 @@ void StorageReplicatedMergeTree::mergeSelectingTask() } } } + else + { + LOG_DEBUG(log, "TOO MANY MUTATIONS IN QUEUE"); + } } } catch (...) @@ -3395,6 +3401,7 @@ void StorageReplicatedMergeTree::alter( bool have_mutation = false; std::optional lock_holder; + size_t partitions_count = 0; if (!maybe_mutation_commands.empty()) { String mutations_path = zookeeper_path + "/mutations"; @@ -3406,17 +3413,36 @@ void StorageReplicatedMergeTree::alter( Coordination::Stat mutations_stat; zookeeper->get(mutations_path, &mutations_stat); - Coordination::Stat intention_counter_stat; - zookeeper->get(zookeeper_path + "/alter_intention_counter", &intention_counter_stat); + //Coordination::Stat intention_counter_stat; + //zookeeper->get(zookeeper_path + "/alter_intention_counter", &intention_counter_stat); + Strings partitions = zookeeper->getChildren(zookeeper_path + "/block_numbers"); + + std::vector> partition_futures; + for (const String & partition : partitions) + partition_futures.push_back(zookeeper->asyncGet(zookeeper_path + "/block_numbers/" + partition)); + + std::unordered_map partition_versions; + for (size_t i = 0; i < partition_futures.size(); ++i) + { + auto stat = partition_futures[i].get().stat; + auto partition = partitions[i]; + partition_versions[partition] = stat.version; + LOG_DEBUG(log, "Partition version:" << partition << " stat version " << stat.version); + } + lock_holder.emplace( zookeeper_path + "/block_numbers", "block-", zookeeper_path + "/temp", *zookeeper); - ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/alter_intention_counter", intention_counter_stat.version)); - for (const auto & lock : lock_holder->getLocks()) { mutation_entry.block_numbers[lock.partition_id] = lock.number; - LOG_DEBUG(log, "ALLOCATED:" << lock.number << " FOR VERSION:" << metadata_version + 1); + //ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/alter_intention_counter", intention_counter_stat.version)); + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/block_numbers/" + lock.partition_id, partition_versions[lock.partition_id])); + partitions_count++; + LOG_DEBUG( + log, + "ALLOCATED:" << lock.number << " FOR VERSION:" << metadata_version + 1 + << " Partition version:" << partition_versions[lock.partition_id] + 1 << " for partition " << lock.partition_id); } mutation_entry.create_time = time(nullptr); @@ -3431,7 +3457,7 @@ void StorageReplicatedMergeTree::alter( Coordination::Responses results; int32_t rc = zookeeper->tryMulti(ops, results); - LOG_DEBUG(log, "ALTER REQUESTED TO" << entry.alter_version); + LOG_DEBUG(log, "ALTER REQUESTED TO:" << entry.alter_version); //std::cerr << "Results size:" << results.size() << std::endl; //std::cerr << "Have mutation:" << have_mutation << std::endl; @@ -3443,7 +3469,7 @@ void StorageReplicatedMergeTree::alter( { //std::cerr << "In have mutation\n"; //std::cerr << "INDEX:" << results.size() - 2 << std::endl; - String alter_path = dynamic_cast(*results[results.size() - 4]).path_created; + String alter_path = dynamic_cast(*results[results.size() - 3 - partitions_count]).path_created; //std::cerr << "Alter path:" << alter_path << std::endl; entry.znode_name = alter_path.substr(alter_path.find_last_of('/') + 1); diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference new file mode 100644 index 00000000000..ff9c6824f00 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.reference @@ -0,0 +1,17 @@ +1725 +1725 +1725 +1725 +1725 +Starting alters +Finishing alters +1 +0 +1 +0 +1 +0 +1 +0 +1 +0 diff --git a/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh new file mode 100755 index 00000000000..572beb88389 --- /dev/null +++ b/dbms/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +REPLICAS=5 + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "CREATE TABLE concurrent_alter_mt_$i (key UInt64, value1 UInt64, value2 String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/concurrent_alter_mt', '$i') ORDER BY key SETTINGS max_replicated_mutations_in_queue=1000, number_of_free_entries_in_pool_to_execute_mutation=0" +done + +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, toString(number) from numbers(10)" +$CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_1 SELECT number, number + 10, toString(number) from numbers(10, 40)" + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_alter_mt_$i" +done + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_mt_$i" +done + +INITIAL_SUM=`$CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_alter_mt_1"` + +# This is just garbage thread with conflictings alter +# it additionally loads alters "queue". +function garbage_alter_thread() +{ + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + $CLICKHOUSE_CLIENT -n --query "ALTER TABLE concurrent_alter_mt_$REPLICA ADD COLUMN h String DEFAULT '0'; ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN h UInt64; ALTER TABLE concurrent_alter_mt_$REPLICA DROP COLUMN h;"; + done +} + + +# This alters mostly requires not only metadata change +# but also conversion of data. Also they are all compatible +# between each other, so can be executed concurrently. +function correct_alter_thread() +{ + TYPES=(Float64 String UInt8 UInt32) + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + TYPE=${TYPES[$RANDOM % ${#TYPES[@]} ]} + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_$REPLICA MODIFY COLUMN value1 $TYPE SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency + sleep 0.$RANDOM + done +} + +# This thread add some data to table. After we finish we can check, that +# all our data have same types. +# insert queries will fail sometime because of wrong types. +function insert_thread() +{ + + VALUES=(7.0 7 '7') + while true; do + REPLICA=$(($RANDOM % 5 + 1)) + VALUE=${VALUES[$RANDOM % ${#VALUES[@]} ]} + $CLICKHOUSE_CLIENT --query "INSERT INTO concurrent_alter_mt_$REPLICA VALUES($RANDOM, $VALUE, toString($VALUE))" + sleep 0.$RANDOM + done +} + + + +echo "Starting alters" +export -f garbage_alter_thread; +export -f correct_alter_thread; +export -f insert_thread; + + +TIMEOUT=120 + + +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & +timeout $TIMEOUT bash -c correct_alter_thread 2> /dev/null & + +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & +timeout $TIMEOUT bash -c insert_thread 2> /dev/null & + +wait + +echo "Finishing alters" + +# This alter will finish all previous +$CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>/dev/null +while [ $? -ne 0 ]; do + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_mt_1 MODIFY COLUMN value1 String SETTINGS replication_alter_partitions_sync=2" 2>/dev/null +done + + +for i in `seq $REPLICAS`; do + $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_alter_mt_$i" + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE is_done=0" # all mutations have to be done + #$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_alter_mt_$i" +done From d2d4118730dcd846dd7889bbc0d0e056e1163e8f Mon Sep 17 00:00:00 2001 From: chertus Date: Tue, 11 Feb 2020 21:27:52 +0300 Subject: [PATCH 0244/2007] add JoinSwitcher --- dbms/src/Interpreters/AnalyzedJoin.cpp | 27 ++------ dbms/src/Interpreters/AnalyzedJoin.h | 4 +- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 4 +- dbms/src/Interpreters/JoinSwitcher.h | 72 ++++++++++++++++++++ 4 files changed, 81 insertions(+), 26 deletions(-) create mode 100644 dbms/src/Interpreters/JoinSwitcher.h diff --git a/dbms/src/Interpreters/AnalyzedJoin.cpp b/dbms/src/Interpreters/AnalyzedJoin.cpp index e7115816920..5d79ad71ae2 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.cpp +++ b/dbms/src/Interpreters/AnalyzedJoin.cpp @@ -1,6 +1,4 @@ #include -#include -#include #include @@ -229,27 +227,14 @@ bool AnalyzedJoin::sameStrictnessAndKind(ASTTableJoin::Strictness strictness_, A return false; } -JoinPtr makeJoin(std::shared_ptr table_join, const Block & right_sample_block) +bool AnalyzedJoin::allowMergeJoin() const { - auto kind = table_join->kind(); - auto strictness = table_join->strictness(); + bool is_any = (strictness() == ASTTableJoin::Strictness::Any); + bool is_all = (strictness() == ASTTableJoin::Strictness::All); + bool is_semi = (strictness() == ASTTableJoin::Strictness::Semi); - bool is_any = (strictness == ASTTableJoin::Strictness::Any); - bool is_all = (strictness == ASTTableJoin::Strictness::All); - bool is_semi = (strictness == ASTTableJoin::Strictness::Semi); - - bool allow_merge_join = (isLeft(kind) && (is_any || is_all || is_semi)) || (isInner(kind) && is_all); - - if (table_join->partial_merge_join && allow_merge_join) - return std::make_shared(table_join, right_sample_block); - return std::make_shared(table_join, right_sample_block); -} - -bool isMergeJoin(const JoinPtr & join) -{ - if (join) - return typeid_cast(join.get()); - return false; + bool allow_merge_join = (isLeft(kind()) && (is_any || is_all || is_semi)) || (isInner(kind()) && is_all); + return allow_merge_join && partial_merge_join; } } diff --git a/dbms/src/Interpreters/AnalyzedJoin.h b/dbms/src/Interpreters/AnalyzedJoin.h index fe89b6f47ef..a96ea54d5fe 100644 --- a/dbms/src/Interpreters/AnalyzedJoin.h +++ b/dbms/src/Interpreters/AnalyzedJoin.h @@ -87,6 +87,7 @@ public: bool sameStrictnessAndKind(ASTTableJoin::Strictness, ASTTableJoin::Kind) const; const SizeLimits & sizeLimits() const { return size_limits; } VolumePtr getTemporaryVolume() { return tmp_volume; } + bool allowMergeJoin() const; bool forceNullableRight() const { return join_use_nulls && isLeftOrFull(table_join.kind); } bool forceNullableLeft() const { return join_use_nulls && isRightOrFull(table_join.kind); } @@ -128,9 +129,6 @@ public: void setRightKeys(const Names & keys) { key_names_right = keys; } static bool sameJoin(const AnalyzedJoin * x, const AnalyzedJoin * y); - friend JoinPtr makeJoin(std::shared_ptr table_join, const Block & right_sample_block); }; -bool isMergeJoin(const JoinPtr &); - } diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index f131afb86c6..c430e348e13 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -29,7 +29,7 @@ #include #include #include -#include +#include #include #include @@ -564,7 +564,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeTableJoin(const ASTTablesInSelectQuer /// TODO You do not need to set this up when JOIN is only needed on remote servers. subquery_for_join.setJoinActions(joined_block_actions); /// changes subquery_for_join.sample_block inside - subquery_for_join.join = makeJoin(syntax->analyzed_join, subquery_for_join.sample_block); + subquery_for_join.join = std::make_shared(syntax->analyzed_join, subquery_for_join.sample_block); } return subquery_for_join.join; diff --git a/dbms/src/Interpreters/JoinSwitcher.h b/dbms/src/Interpreters/JoinSwitcher.h new file mode 100644 index 00000000000..4c627bd7b8e --- /dev/null +++ b/dbms/src/Interpreters/JoinSwitcher.h @@ -0,0 +1,72 @@ +#pragma once + +#include +#include +#include +#include + +namespace DB +{ + +class JoinSwitcher : public IJoin +{ +public: + JoinSwitcher(std::shared_ptr table_join, const Block & right_sample_block) + { + if (table_join->allowMergeJoin()) + join = std::make_shared(table_join, right_sample_block); + else + join = std::make_shared(table_join, right_sample_block); + } + + bool addJoinedBlock(const Block & block) override + { + /// TODO: switch Join -> MergeJoin + return join->addJoinedBlock(block); + } + + void joinBlock(Block & block, std::shared_ptr & not_processed) override + { + join->joinBlock(block, not_processed); + } + + bool hasTotals() const override + { + return join->hasTotals(); + } + + void setTotals(const Block & block) override + { + join->setTotals(block); + } + + void joinTotals(Block & block) const override + { + join->joinTotals(block); + } + + size_t getTotalRowCount() const override + { + return join->getTotalRowCount(); + } + + bool alwaysReturnsEmptySet() const override + { + return join->alwaysReturnsEmptySet(); + } + + BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & block, UInt64 max_block_size) const override + { + return join->createStreamWithNonJoinedRows(block, max_block_size); + } + + bool hasStreamWithNonJoinedRows() const override + { + return join->hasStreamWithNonJoinedRows(); + } + +private: + JoinPtr join; +}; + +} From 22789a3b55ab392cf3de32db082224dbea0a72b6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Tue, 11 Feb 2020 21:34:48 +0300 Subject: [PATCH 0245/2007] ShardPartitionPiece task added --- dbms/programs/copier/CMakeLists.txt | 2 +- dbms/programs/copier/ClusterCopier.cpp | 1879 --------------------- dbms/programs/copier/ClusterCopier.h | 1570 ++++++++++++++++- dbms/programs/copier/ClusterCopierApp.cpp | 173 ++ dbms/programs/copier/ClusterCopierApp.h | 87 + dbms/programs/copier/Internals.h | 74 +- dbms/programs/copier/ZookeeperStaff.h | 224 +++ 7 files changed, 2052 insertions(+), 1957 deletions(-) delete mode 100644 dbms/programs/copier/ClusterCopier.cpp create mode 100644 dbms/programs/copier/ClusterCopierApp.cpp create mode 100644 dbms/programs/copier/ClusterCopierApp.h create mode 100644 dbms/programs/copier/ZookeeperStaff.h diff --git a/dbms/programs/copier/CMakeLists.txt b/dbms/programs/copier/CMakeLists.txt index 8e13040b29d..9852dc92242 100644 --- a/dbms/programs/copier/CMakeLists.txt +++ b/dbms/programs/copier/CMakeLists.txt @@ -1,4 +1,4 @@ -set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopier.cpp) +set(CLICKHOUSE_COPIER_SOURCES ${CMAKE_CURRENT_SOURCE_DIR}/ClusterCopierApp.cpp) set(CLICKHOUSE_COPIER_LINK PRIVATE clickhouse_common_zookeeper clickhouse_parsers clickhouse_functions clickhouse_table_functions clickhouse_aggregate_functions clickhouse_dictionaries string_utils ${Poco_XML_LIBRARY} PUBLIC daemon) set(CLICKHOUSE_COPIER_INCLUDE SYSTEM PRIVATE ${PCG_RANDOM_INCLUDE_DIR}) diff --git a/dbms/programs/copier/ClusterCopier.cpp b/dbms/programs/copier/ClusterCopier.cpp deleted file mode 100644 index e8b3018a861..00000000000 --- a/dbms/programs/copier/ClusterCopier.cpp +++ /dev/null @@ -1,1879 +0,0 @@ -#include "ClusterCopier.h" - -#include "Internals.h" - -namespace DB -{ - - -using ConfigurationPtr = Poco::AutoPtr; - -static ConfigurationPtr getConfigurationFromXMLString(const std::string & xml_data) -{ - std::stringstream ss(xml_data); - Poco::XML::InputSource input_source{ss}; - return {new Poco::Util::XMLConfiguration{&input_source}}; -} - - -class ClusterCopier -{ -public: - - ClusterCopier(String task_path_, - String host_id_, - String proxy_database_name_, - Context & context_) - : - task_zookeeper_path(std::move(task_path_)), - host_id(std::move(host_id_)), - working_database_name(std::move(proxy_database_name_)), - context(context_), - log(&Poco::Logger::get("ClusterCopier")) - { - } - - void init() - { - auto zookeeper = context.getZooKeeper(); - - task_description_watch_callback = [this] (const Coordination::WatchResponse & response) - { - if (response.error != Coordination::ZOK) - return; - UInt64 version = ++task_descprtion_version; - LOG_DEBUG(log, "Task description should be updated, local version " << version); - }; - - task_description_path = task_zookeeper_path + "/description"; - task_cluster = std::make_unique(task_zookeeper_path, working_database_name); - - reloadTaskDescription(); - task_cluster_initial_config = task_cluster_current_config; - - task_cluster->loadTasks(*task_cluster_initial_config); - context.setClustersConfig(task_cluster_initial_config, task_cluster->clusters_prefix); - - /// Set up shards and their priority - task_cluster->random_engine.seed(task_cluster->random_device()); - for (auto & task_table : task_cluster->table_tasks) - { - task_table.cluster_pull = context.getCluster(task_table.cluster_pull_name); - task_table.cluster_push = context.getCluster(task_table.cluster_push_name); - task_table.initShards(task_cluster->random_engine); - } - - LOG_DEBUG(log, "Will process " << task_cluster->table_tasks.size() << " table tasks"); - - /// Do not initialize tables, will make deferred initialization in process() - - zookeeper->createAncestors(getWorkersPathVersion() + "/"); - zookeeper->createAncestors(getWorkersPath() + "/"); - } - - template - decltype(auto) retry(T && func, UInt64 max_tries = 100) - { - std::exception_ptr exception; - - for (UInt64 try_number = 1; try_number <= max_tries; ++try_number) - { - try - { - return func(); - } - catch (...) - { - exception = std::current_exception(); - if (try_number < max_tries) - { - tryLogCurrentException(log, "Will retry"); - std::this_thread::sleep_for(default_sleep_time); - } - } - } - - std::rethrow_exception(exception); - } - - - void discoverShardPartitions(const ConnectionTimeouts & timeouts, const TaskShardPtr & task_shard) - { - TaskTable & task_table = task_shard->task_table; - - LOG_INFO(log, "Discover partitions of shard " << task_shard->getDescription()); - - auto get_partitions = [&] () { return getShardPartitions(timeouts, *task_shard); }; - auto existing_partitions_names = retry(get_partitions, 60); - Strings filtered_partitions_names; - Strings missing_partitions; - - /// Check that user specified correct partition names - auto check_partition_format = [] (const DataTypePtr & type, const String & partition_text_quoted) - { - MutableColumnPtr column_dummy = type->createColumn(); - ReadBufferFromString rb(partition_text_quoted); - - try - { - type->deserializeAsTextQuoted(*column_dummy, rb, FormatSettings()); - } - catch (Exception & e) - { - throw Exception("Partition " + partition_text_quoted + " has incorrect format. " + e.displayText(), ErrorCodes::BAD_ARGUMENTS); - } - }; - - if (task_table.has_enabled_partitions) - { - /// Process partition in order specified by - for (const String & partition_name : task_table.enabled_partitions) - { - /// Check that user specified correct partition names - check_partition_format(task_shard->partition_key_column.type, partition_name); - - auto it = existing_partitions_names.find(partition_name); - - /// Do not process partition if it is not in enabled_partitions list - if (it == existing_partitions_names.end()) - { - missing_partitions.emplace_back(partition_name); - continue; - } - - filtered_partitions_names.emplace_back(*it); - } - - for (const String & partition_name : existing_partitions_names) - { - if (!task_table.enabled_partitions_set.count(partition_name)) - { - LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in " - << "enabled_partitions of " << task_table.table_id); - } - } - } - else - { - for (const String & partition_name : existing_partitions_names) - filtered_partitions_names.emplace_back(partition_name); - } - - for (const String & partition_name : filtered_partitions_names) - { - task_shard->partition_tasks.emplace(partition_name, ShardPartition(*task_shard, partition_name)); - task_shard->checked_partitions.emplace(partition_name, true); - } - - if (!missing_partitions.empty()) - { - std::stringstream ss; - for (const String & missing_partition : missing_partitions) - ss << " " << missing_partition; - - LOG_WARNING(log, "There are no " << missing_partitions.size() << " partitions from enabled_partitions in shard " - << task_shard->getDescription() << " :" << ss.str()); - } - - LOG_DEBUG(log, "Will copy " << task_shard->partition_tasks.size() << " partitions from shard " << task_shard->getDescription()); - } - - /// Compute set of partitions, assume set of partitions aren't changed during the processing - void discoverTablePartitions(const ConnectionTimeouts & timeouts, TaskTable & task_table, UInt64 num_threads = 0) - { - /// Fetch partitions list from a shard - { - ThreadPool thread_pool(num_threads ? num_threads : 2 * getNumberOfPhysicalCPUCores()); - - for (const TaskShardPtr & task_shard : task_table.all_shards) - thread_pool.scheduleOrThrowOnError([this, timeouts, task_shard]() { discoverShardPartitions(timeouts, task_shard); }); - - LOG_DEBUG(log, "Waiting for " << thread_pool.active() << " setup jobs"); - thread_pool.wait(); - } - } - - void uploadTaskDescription(const std::string & task_path, const std::string & task_file, const bool force) - { - auto local_task_description_path = task_path + "/description"; - - String task_config_str; - { - ReadBufferFromFile in(task_file); - readStringUntilEOF(task_config_str, in); - } - if (task_config_str.empty()) - return; - - auto zookeeper = context.getZooKeeper(); - - zookeeper->createAncestors(local_task_description_path); - auto code = zookeeper->tryCreate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); - if (code && force) - zookeeper->createOrUpdate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); - - LOG_DEBUG(log, "Task description " << ((code && !force) ? "not " : "") << "uploaded to " << local_task_description_path << " with result " << code << " ("<< zookeeper->error2string(code) << ")"); - } - - void reloadTaskDescription() - { - auto zookeeper = context.getZooKeeper(); - task_description_watch_zookeeper = zookeeper; - - String task_config_str; - Coordination::Stat stat{}; - int code; - - zookeeper->tryGetWatch(task_description_path, task_config_str, &stat, task_description_watch_callback, &code); - if (code) - throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS); - - LOG_DEBUG(log, "Loading description, zxid=" << task_description_current_stat.czxid); - auto config = getConfigurationFromXMLString(task_config_str); - - /// Setup settings - task_cluster->reloadSettings(*config); - context.getSettingsRef() = task_cluster->settings_common; - - task_cluster_current_config = config; - task_description_current_stat = stat; - } - - void updateConfigIfNeeded() - { - UInt64 version_to_update = task_descprtion_version; - bool is_outdated_version = task_descprtion_current_version != version_to_update; - bool is_expired_session = !task_description_watch_zookeeper || task_description_watch_zookeeper->expired(); - - if (!is_outdated_version && !is_expired_session) - return; - - LOG_DEBUG(log, "Updating task description"); - reloadTaskDescription(); - - task_descprtion_current_version = version_to_update; - } - - void process(const ConnectionTimeouts & timeouts) - { - for (TaskTable & task_table : task_cluster->table_tasks) - { - LOG_INFO(log, "Process table task " << task_table.table_id << " with " - << task_table.all_shards.size() << " shards, " << task_table.local_shards.size() << " of them are local ones"); - - if (task_table.all_shards.empty()) - continue; - - /// Discover partitions of each shard and total set of partitions - if (!task_table.has_enabled_partitions) - { - /// If there are no specified enabled_partitions, we must discover them manually - discoverTablePartitions(timeouts, task_table); - - /// After partitions of each shard are initialized, initialize cluster partitions - for (const TaskShardPtr & task_shard : task_table.all_shards) - { - for (const auto & partition_elem : task_shard->partition_tasks) - { - const String & partition_name = partition_elem.first; - task_table.cluster_partitions.emplace(partition_name, ClusterPartition{}); - } - } - - for (auto & partition_elem : task_table.cluster_partitions) - { - const String & partition_name = partition_elem.first; - - for (const TaskShardPtr & task_shard : task_table.all_shards) - task_shard->checked_partitions.emplace(partition_name); - - task_table.ordered_partition_names.emplace_back(partition_name); - } - } - else - { - /// If enabled_partitions are specified, assume that each shard has all partitions - /// We will refine partition set of each shard in future - - for (const String & partition_name : task_table.enabled_partitions) - { - task_table.cluster_partitions.emplace(partition_name, ClusterPartition{}); - task_table.ordered_partition_names.emplace_back(partition_name); - } - } - - task_table.watch.restart(); - - /// Retry table processing - bool table_is_done = false; - for (UInt64 num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries) - { - if (tryProcessTable(timeouts, task_table)) - { - table_is_done = true; - break; - } - } - - if (!table_is_done) - { - throw Exception("Too many tries to process table " + task_table.table_id + ". Abort remaining execution", - ErrorCodes::UNFINISHED); - } - } - } - - /// Disables DROP PARTITION commands that used to clear data after errors - void setSafeMode(bool is_safe_mode_ = true) - { - is_safe_mode = is_safe_mode_; - } - - void setCopyFaultProbability(double copy_fault_probability_) - { - copy_fault_probability = copy_fault_probability_; - } - - -protected: - - String getWorkersPath() const - { - return task_cluster->task_zookeeper_path + "/task_active_workers"; - } - - String getWorkersPathVersion() const - { - return getWorkersPath() + "_version"; - } - - String getCurrentWorkerNodePath() const - { - return getWorkersPath() + "/" + host_id; - } - - zkutil::EphemeralNodeHolder::Ptr createTaskWorkerNodeAndWaitIfNeed( - const zkutil::ZooKeeperPtr & zookeeper, - const String & description, - bool unprioritized) - { - std::chrono::milliseconds current_sleep_time = default_sleep_time; - static constexpr std::chrono::milliseconds max_sleep_time(30000); // 30 sec - - if (unprioritized) - std::this_thread::sleep_for(current_sleep_time); - - String workers_version_path = getWorkersPathVersion(); - String workers_path = getWorkersPath(); - String current_worker_path = getCurrentWorkerNodePath(); - - UInt64 num_bad_version_errors = 0; - - while (true) - { - updateConfigIfNeeded(); - - Coordination::Stat stat{}; - zookeeper->get(workers_version_path, &stat); - auto version = stat.version; - zookeeper->get(workers_path, &stat); - - if (static_cast(stat.numChildren) >= task_cluster->max_workers) - { - LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" - << ". Postpone processing " << description); - - if (unprioritized) - current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); - - std::this_thread::sleep_for(current_sleep_time); - num_bad_version_errors = 0; - } - else - { - Coordination::Requests ops; - ops.emplace_back(zkutil::makeSetRequest(workers_version_path, description, version)); - ops.emplace_back(zkutil::makeCreateRequest(current_worker_path, description, zkutil::CreateMode::Ephemeral)); - Coordination::Responses responses; - auto code = zookeeper->tryMulti(ops, responses); - - if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS) - return std::make_shared(current_worker_path, *zookeeper, false, false, description); - - if (code == Coordination::ZBADVERSION) - { - ++num_bad_version_errors; - - /// Try to make fast retries - if (num_bad_version_errors > 3) - { - LOG_DEBUG(log, "A concurrent worker has just been added, will check free worker slots again"); - std::chrono::milliseconds random_sleep_time(std::uniform_int_distribution(1, 1000)(task_cluster->random_engine)); - std::this_thread::sleep_for(random_sleep_time); - num_bad_version_errors = 0; - } - } - else - throw Coordination::Exception(code); - } - } - } - - /** Checks that the whole partition of a table was copied. We should do it carefully due to dirty lock. - * State of some task could change during the processing. - * We have to ensure that all shards have the finished state and there is no dirty flag. - * Moreover, we have to check status twice and check zxid, because state can change during the checking. - */ - bool checkPartitionIsDone(const TaskTable & task_table, const String & partition_name, const TasksShard & shards_with_partition) - { - LOG_DEBUG(log, "Check that all shards processed partition " << partition_name << " successfully"); - - auto zookeeper = context.getZooKeeper(); - - Strings status_paths; - for (auto & shard : shards_with_partition) - { - ShardPartition & task_shard_partition = shard->partition_tasks.find(partition_name)->second; - status_paths.emplace_back(task_shard_partition.getShardStatusPath()); - } - - std::vector zxid1, zxid2; - - try - { - std::vector get_futures; - for (const String & path : status_paths) - get_futures.emplace_back(zookeeper->asyncGet(path)); - - // Check that state is Finished and remember zxid - for (auto & future : get_futures) - { - auto res = future.get(); - - TaskStateWithOwner status = TaskStateWithOwner::fromString(res.data); - if (status.state != TaskState::Finished) - { - LOG_INFO(log, "The task " << res.data << " is being rewritten by " << status.owner << ". Partition will be rechecked"); - return false; - } - - zxid1.push_back(res.stat.pzxid); - } - - // Check that partition is not dirty - { - CleanStateClock clean_state_clock ( - zookeeper, - task_table.getPartitionIsDirtyPath(partition_name), - task_table.getPartitionIsCleanedPath(partition_name) - ); - Coordination::Stat stat{}; - LogicalClock task_start_clock; - if (zookeeper->exists(task_table.getPartitionTaskStatusPath(partition_name), &stat)) - task_start_clock = LogicalClock(stat.mzxid); - zookeeper->get(task_table.getPartitionTaskStatusPath(partition_name), &stat); - if (!clean_state_clock.is_clean() || task_start_clock <= clean_state_clock.discovery_zxid) - { - LOG_INFO(log, "Partition " << partition_name << " become dirty"); - return false; - } - } - - get_futures.clear(); - for (const String & path : status_paths) - get_futures.emplace_back(zookeeper->asyncGet(path)); - - // Remember zxid of states again - for (auto & future : get_futures) - { - auto res = future.get(); - zxid2.push_back(res.stat.pzxid); - } - } - catch (const Coordination::Exception & e) - { - LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name - << ". Will recheck the partition. Error: " << e.displayText()); - return false; - } - - // If all task is finished and zxid is not changed then partition could not become dirty again - for (UInt64 shard_num = 0; shard_num < status_paths.size(); ++shard_num) - { - if (zxid1[shard_num] != zxid2[shard_num]) - { - LOG_INFO(log, "The task " << status_paths[shard_num] << " is being modified now. Partition will be rechecked"); - return false; - } - } - - LOG_INFO(log, "Partition " << partition_name << " is copied successfully"); - return true; - } - - /// Removes MATERIALIZED and ALIAS columns from create table query - static ASTPtr removeAliasColumnsFromCreateQuery(const ASTPtr & query_ast) - { - const ASTs & column_asts = query_ast->as().columns_list->columns->children; - auto new_columns = std::make_shared(); - - for (const ASTPtr & column_ast : column_asts) - { - const auto & column = column_ast->as(); - - if (!column.default_specifier.empty()) - { - ColumnDefaultKind kind = columnDefaultKindFromString(column.default_specifier); - if (kind == ColumnDefaultKind::Materialized || kind == ColumnDefaultKind::Alias) - continue; - } - - new_columns->children.emplace_back(column_ast->clone()); - } - - ASTPtr new_query_ast = query_ast->clone(); - auto & new_query = new_query_ast->as(); - - auto new_columns_list = std::make_shared(); - new_columns_list->set(new_columns_list->columns, new_columns); - if (auto indices = query_ast->as()->columns_list->indices) - new_columns_list->set(new_columns_list->indices, indices->clone()); - - new_query.replace(new_query.columns_list, new_columns_list); - - return new_query_ast; - } - - /// Replaces ENGINE and table name in a create query - static std::shared_ptr rewriteCreateQueryStorage(const ASTPtr & create_query_ast, const DatabaseAndTableName & new_table, const ASTPtr & new_storage_ast) - { - const auto & create = create_query_ast->as(); - auto res = std::make_shared(create); - - if (create.storage == nullptr || new_storage_ast == nullptr) - throw Exception("Storage is not specified", ErrorCodes::LOGICAL_ERROR); - - res->database = new_table.first; - res->table = new_table.second; - - res->children.clear(); - res->set(res->columns_list, create.columns_list->clone()); - res->set(res->storage, new_storage_ast->clone()); - - return res; - } - - /** Allows to compare two incremental counters of type UInt32 in presence of possible overflow. - * We assume that we compare values that are not too far away. - * For example, when we increment 0xFFFFFFFF, we get 0. So, 0xFFFFFFFF is less than 0. - */ - class WrappingUInt32 - { - public: - UInt32 value; - - explicit WrappingUInt32(UInt32 _value) - : value(_value) - {} - - bool operator<(const WrappingUInt32 & other) const - { - return value != other.value && *this <= other; - } - - bool operator<=(const WrappingUInt32 & other) const - { - const UInt32 HALF = 1 << 31; - return (value <= other.value && other.value - value < HALF) - || (value > other.value && value - other.value > HALF); - } - - bool operator==(const WrappingUInt32 & other) const - { - return value == other.value; - } - }; - - /** Conforming Zxid definition. - * cf. https://github.com/apache/zookeeper/blob/631d1b284f0edb1c4f6b0fb221bf2428aec71aaa/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md#guarantees-properties-and-definitions - * - * But it is better to read this: https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html - * - * Actually here is the definition of Zxid. - * Every change to the ZooKeeper state receives a stamp in the form of a zxid (ZooKeeper Transaction Id). - * This exposes the total ordering of all changes to ZooKeeper. Each change will have a unique zxid - * and if zxid1 is smaller than zxid2 then zxid1 happened before zxid2. - */ - class Zxid - { - public: - WrappingUInt32 epoch; - WrappingUInt32 counter; - explicit Zxid(UInt64 _zxid) - : epoch(_zxid >> 32) - , counter(_zxid) - {} - - bool operator<=(const Zxid & other) const - { - return (epoch < other.epoch) - || (epoch == other.epoch && counter <= other.counter); - } - - bool operator==(const Zxid & other) const - { - return epoch == other.epoch && counter == other.counter; - } - }; - - /* When multiple ClusterCopiers discover that the target partition is not empty, - * they will attempt to clean up this partition before proceeding to copying. - * - * Instead of purging is_dirty, the history of cleaning work is preserved and partition hygiene is established - * based on a happens-before relation between the events. - * This relation is encoded by LogicalClock based on the mzxid of the is_dirty ZNode and is_dirty/cleaned. - * The fact of the partition hygiene is encoded by CleanStateClock. - * - * For you to know what mzxid means: - * - * ZooKeeper Stat Structure: - * The Stat structure for each znode in ZooKeeper is made up of the following fields: - * - * -- czxid - * The zxid of the change that caused this znode to be created. - * - * -- mzxid - * The zxid of the change that last modified this znode. - * - * -- ctime - * The time in milliseconds from epoch when this znode was created. - * - * -- mtime - * The time in milliseconds from epoch when this znode was last modified. - * - * -- version - * The number of changes to the data of this znode. - * - * -- cversion - * The number of changes to the children of this znode. - * - * -- aversion - * The number of changes to the ACL of this znode. - * - * -- ephemeralOwner - * The session id of the owner of this znode if the znode is an ephemeral node. - * If it is not an ephemeral node, it will be zero. - * - * -- dataLength - * The length of the data field of this znode. - * - * -- numChildren - * The number of children of this znode. - * */ - - class LogicalClock - { - public: - std::optional zxid; - - LogicalClock() = default; - - explicit LogicalClock(UInt64 _zxid) - : zxid(_zxid) - {} - - bool hasHappened() const - { - return bool(zxid); - } - - /// happens-before relation with a reasonable time bound - bool happensBefore(const LogicalClock & other) const - { - return !zxid - || (other.zxid && *zxid <= *other.zxid); - } - - bool operator<=(const LogicalClock & other) const - { - return happensBefore(other); - } - - /// strict equality check - bool operator==(const LogicalClock & other) const - { - return zxid == other.zxid; - } - }; - - - class CleanStateClock - { - public: - LogicalClock discovery_zxid; - std::optional discovery_version; - - LogicalClock clean_state_zxid; - std::optional clean_state_version; - - std::shared_ptr stale; - - bool is_clean() const - { - return - !is_stale() - && ( - !discovery_zxid.hasHappened() - || (clean_state_zxid.hasHappened() && discovery_zxid <= clean_state_zxid)); - } - - bool is_stale() const - { - return stale->load(); - } - - CleanStateClock( - const zkutil::ZooKeeperPtr & zookeeper, - const String & discovery_path, - const String & clean_state_path) - : stale(std::make_shared(false)) - { - Coordination::Stat stat{}; - String _some_data; - auto watch_callback = - [stale = stale] (const Coordination::WatchResponse & rsp) - { - auto logger = &Poco::Logger::get("ClusterCopier"); - if (rsp.error == Coordination::ZOK) - { - switch (rsp.type) - { - case Coordination::CREATED: - LOG_DEBUG(logger, "CleanStateClock change: CREATED, at " << rsp.path); - stale->store(true); - break; - case Coordination::CHANGED: - LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at" << rsp.path); - stale->store(true); - } - } - }; - if (zookeeper->tryGetWatch(discovery_path, _some_data, &stat, watch_callback)) - { - discovery_zxid = LogicalClock(stat.mzxid); - discovery_version = stat.version; - } - if (zookeeper->tryGetWatch(clean_state_path, _some_data, &stat, watch_callback)) - { - clean_state_zxid = LogicalClock(stat.mzxid); - clean_state_version = stat.version; - } - } - - bool operator==(const CleanStateClock & other) const - { - return !is_stale() - && !other.is_stale() - && discovery_zxid == other.discovery_zxid - && discovery_version == other.discovery_version - && clean_state_zxid == other.clean_state_zxid - && clean_state_version == other.clean_state_version; - } - - bool operator!=(const CleanStateClock & other) const - { - return !(*this == other); - } - }; - - bool tryDropPartition(ShardPartition & task_partition, const zkutil::ZooKeeperPtr & zookeeper, const CleanStateClock & clean_state_clock) - { - if (is_safe_mode) - throw Exception("DROP PARTITION is prohibited in safe mode", ErrorCodes::NOT_IMPLEMENTED); - - TaskTable & task_table = task_partition.task_shard.task_table; - - const String current_shards_path = task_partition.getPartitionShardsPath(); - const String current_partition_active_workers_dir = task_partition.getPartitionActiveWorkersPath(); - const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); - const String dirty_cleaner_path = is_dirty_flag_path + "/cleaner"; - const String is_dirty_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); - - zkutil::EphemeralNodeHolder::Ptr cleaner_holder; - try - { - cleaner_holder = zkutil::EphemeralNodeHolder::create(dirty_cleaner_path, *zookeeper, host_id); - } - catch (const Coordination::Exception & e) - { - if (e.code == Coordination::ZNODEEXISTS) - { - LOG_DEBUG(log, "Partition " << task_partition.name << " is cleaning now by somebody, sleep"); - std::this_thread::sleep_for(default_sleep_time); - return false; - } - - throw; - } - - Coordination::Stat stat{}; - if (zookeeper->exists(current_partition_active_workers_dir, &stat)) - { - if (stat.numChildren != 0) - { - LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers while trying to drop it. Going to sleep."); - std::this_thread::sleep_for(default_sleep_time); - return false; - } - else - { - zookeeper->remove(current_partition_active_workers_dir); - } - } - - { - zkutil::EphemeralNodeHolder::Ptr active_workers_lock; - try - { - active_workers_lock = zkutil::EphemeralNodeHolder::create(current_partition_active_workers_dir, *zookeeper, host_id); - } - catch (const Coordination::Exception & e) - { - if (e.code == Coordination::ZNODEEXISTS) - { - LOG_DEBUG(log, "Partition " << task_partition.name << " is being filled now by somebody, sleep"); - return false; - } - - throw; - } - - // Lock the dirty flag - zookeeper->set(is_dirty_flag_path, host_id, clean_state_clock.discovery_version.value()); - zookeeper->tryRemove(task_partition.getPartitionCleanStartPath()); - CleanStateClock my_clock(zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); - - /// Remove all status nodes - { - Strings children; - if (zookeeper->tryGetChildren(current_shards_path, children) == Coordination::ZOK) - for (const auto & child : children) - { - zookeeper->removeRecursive(current_shards_path + "/" + child); - } - } - - String query = "ALTER TABLE " + getQuotedTable(task_table.table_push); - query += " DROP PARTITION " + task_partition.name + ""; - - /// TODO: use this statement after servers will be updated up to 1.1.54310 - // query += " DROP PARTITION ID '" + task_partition.name + "'"; - - ClusterPtr & cluster_push = task_table.cluster_push; - Settings settings_push = task_cluster->settings_push; - - /// It is important, DROP PARTITION must be done synchronously - settings_push.replication_alter_partitions_sync = 2; - - LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query); - /// Limit number of max executing replicas to 1 - UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1); - - if (num_shards < cluster_push->getShardCount()) - { - LOG_INFO(log, "DROP PARTITION wasn't successfully executed on " << cluster_push->getShardCount() - num_shards << " shards"); - return false; - } - - /// Update the locking node - if (!my_clock.is_stale()) - { - zookeeper->set(is_dirty_flag_path, host_id, my_clock.discovery_version.value()); - if (my_clock.clean_state_version) - zookeeper->set(is_dirty_cleaned_path, host_id, my_clock.clean_state_version.value()); - else - zookeeper->create(is_dirty_cleaned_path, host_id, zkutil::CreateMode::Persistent); - } - else - { - LOG_DEBUG(log, "Clean state is altered when dropping the partition, cowardly bailing"); - /// clean state is stale - return false; - } - - LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name); - if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) - zookeeper->set(current_shards_path, host_id); - } - - LOG_INFO(log, "Partition " << task_partition.name << " is safe for work now."); - return true; - } - - - static constexpr UInt64 max_table_tries = 1000; - static constexpr UInt64 max_shard_partition_tries = 600; - - bool tryProcessTable(const ConnectionTimeouts & timeouts, TaskTable & task_table) - { - /// A heuristic: if previous shard is already done, then check next one without sleeps due to max_workers constraint - bool previous_shard_is_instantly_finished = false; - - /// Process each partition that is present in cluster - for (const String & partition_name : task_table.ordered_partition_names) - { - if (!task_table.cluster_partitions.count(partition_name)) - throw Exception("There are no expected partition " + partition_name + ". It is a bug", ErrorCodes::LOGICAL_ERROR); - - ClusterPartition & cluster_partition = task_table.cluster_partitions[partition_name]; - - Stopwatch watch; - /// We will check all the shards of the table and check if they contain current partition. - TasksShard expected_shards; - UInt64 num_failed_shards = 0; - - ++cluster_partition.total_tries; - - LOG_DEBUG(log, "Processing partition " << partition_name << " for the whole cluster"); - - /// Process each source shard having current partition and copy current partition - /// NOTE: shards are sorted by "distance" to current host - bool has_shard_to_process = false; - for (const TaskShardPtr & shard : task_table.all_shards) - { - /// Does shard have a node with current partition? - if (shard->partition_tasks.count(partition_name) == 0) - { - /// If not, did we check existence of that partition previously? - if (shard->checked_partitions.count(partition_name) == 0) - { - auto check_shard_has_partition = [&] () { return checkShardHasPartition(timeouts, *shard, partition_name); }; - bool has_partition = retry(check_shard_has_partition); - - shard->checked_partitions.emplace(partition_name); - - if (has_partition) - { - shard->partition_tasks.emplace(partition_name, ShardPartition(*shard, partition_name)); - LOG_DEBUG(log, "Discovered partition " << partition_name << " in shard " << shard->getDescription()); - } - else - { - LOG_DEBUG(log, "Found that shard " << shard->getDescription() << " does not contain current partition " << partition_name); - continue; - } - } - else - { - /// We have already checked that partition, but did not discover it - previous_shard_is_instantly_finished = true; - continue; - } - } - - auto it_shard_partition = shard->partition_tasks.find(partition_name); - /// Previously when we discovered that shard does not contain current partition, we skipped it. - /// At this moment partition have to be present. - if (it_shard_partition == shard->partition_tasks.end()) - throw Exception("There are no such partition in a shard. This is a bug.", ErrorCodes::LOGICAL_ERROR); - auto & partition = it_shard_partition->second; - - expected_shards.emplace_back(shard); - - /// Do not sleep if there is a sequence of already processed shards to increase startup - bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote; - PartitionTaskStatus task_status = PartitionTaskStatus::Error; - bool was_error = false; - has_shard_to_process = true; - for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) - { - task_status = tryProcessPartitionTask(timeouts, partition, is_unprioritized_task); - - /// Exit if success - if (task_status == PartitionTaskStatus::Finished) - break; - - was_error = true; - - /// Skip if the task is being processed by someone - if (task_status == PartitionTaskStatus::Active) - break; - - /// Repeat on errors - std::this_thread::sleep_for(default_sleep_time); - } - - if (task_status == PartitionTaskStatus::Error) - ++num_failed_shards; - - previous_shard_is_instantly_finished = !was_error; - } - - cluster_partition.elapsed_time_seconds += watch.elapsedSeconds(); - - /// Check that whole cluster partition is done - /// Firstly check the number of failed partition tasks, then look into ZooKeeper and ensure that each partition is done - bool partition_is_done = num_failed_shards == 0; - try - { - partition_is_done = - !has_shard_to_process - || (partition_is_done && checkPartitionIsDone(task_table, partition_name, expected_shards)); - } - catch (...) - { - tryLogCurrentException(log); - partition_is_done = false; - } - - if (partition_is_done) - { - task_table.finished_cluster_partitions.emplace(partition_name); - - task_table.bytes_copied += cluster_partition.bytes_copied; - task_table.rows_copied += cluster_partition.rows_copied; - double elapsed = cluster_partition.elapsed_time_seconds; - - LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name - << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes" - << ", " << formatReadableQuantity(cluster_partition.rows_copied) << " rows" - << " and " << cluster_partition.blocks_copied << " source blocks are copied"); - - if (cluster_partition.rows_copied) - { - LOG_INFO(log, "Average partition speed: " - << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed) << " per second."); - } - - if (task_table.rows_copied) - { - LOG_INFO(log, "Average table " << task_table.table_id << " speed: " - << formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed) << " per second."); - } - } - } - - UInt64 required_partitions = task_table.cluster_partitions.size(); - UInt64 finished_partitions = task_table.finished_cluster_partitions.size(); - bool table_is_done = finished_partitions >= required_partitions; - - if (!table_is_done) - { - LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet." - << "Copied " << finished_partitions << " of " << required_partitions << ", will retry"); - } - - return table_is_done; - } - - - /// Execution status of a task - enum class PartitionTaskStatus - { - Active, - Finished, - Error, - }; - - /// Job for copying partition from particular shard. - PartitionTaskStatus tryProcessPartitionTask(const ConnectionTimeouts & timeouts, ShardPartition & task_partition, bool is_unprioritized_task) - { - PartitionTaskStatus res; - - try - { - res = processPartitionTaskImpl(timeouts, task_partition, is_unprioritized_task); - } - catch (...) - { - tryLogCurrentException(log, "An error occurred while processing partition " + task_partition.name); - res = PartitionTaskStatus::Error; - } - - /// At the end of each task check if the config is updated - try - { - updateConfigIfNeeded(); - } - catch (...) - { - tryLogCurrentException(log, "An error occurred while updating the config"); - } - - return res; - } - - PartitionTaskStatus processPartitionTaskImpl(const ConnectionTimeouts & timeouts, ShardPartition & task_partition, bool is_unprioritized_task) - { - TaskShard & task_shard = task_partition.task_shard; - TaskTable & task_table = task_shard.task_table; - ClusterPartition & cluster_partition = task_table.getClusterPartition(task_partition.name); - const size_t number_of_splits = task_table.number_of_splits; - UNUSED(number_of_splits); - - /// We need to update table definitions for each partition, it could be changed after ALTER - createShardInternalTables(timeouts, task_shard); - - auto zookeeper = context.getZooKeeper(); - - const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); - const String is_dirty_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); - const String current_task_is_active_path = task_partition.getActiveWorkerPath(); - const String current_task_status_path = task_partition.getShardStatusPath(); - - /// Auxiliary functions: - - /// Creates is_dirty node to initialize DROP PARTITION - auto create_is_dirty_node = [&, this] (const CleanStateClock & clock) - { - if (clock.is_stale()) - LOG_DEBUG(log, "Clean state clock is stale while setting dirty flag, cowardly bailing"); - else if (!clock.is_clean()) - LOG_DEBUG(log, "Thank you, Captain Obvious"); - else if (clock.discovery_version) - { - LOG_DEBUG(log, "Updating clean state clock"); - zookeeper->set(is_dirty_flag_path, host_id, clock.discovery_version.value()); - } - else - { - LOG_DEBUG(log, "Creating clean state clock"); - zookeeper->create(is_dirty_flag_path, host_id, zkutil::CreateMode::Persistent); - } - }; - - /// Returns SELECT query filtering current partition and applying user filter - auto get_select_query = [&] (const DatabaseAndTableName & from_table, const String & fields, String limit = "", - bool enable_splitting = false, size_t current_piece_number = 0) - { - String query; - query += "SELECT " + fields + " FROM " + getQuotedTable(from_table); - /// TODO: Bad, it is better to rewrite with ASTLiteral(partition_key_field) - query += " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + task_partition.name + " AS partition_key))"; - if (enable_splitting) - query += " AND ( cityHash64(*) = " + std::to_string(current_piece_number) + " )"; - if (!task_table.where_condition_str.empty()) - query += " AND (" + task_table.where_condition_str + ")"; - if (!limit.empty()) - query += " LIMIT " + limit; - - ParserQuery p_query(query.data() + query.size()); - return parseQuery(p_query, query, 0); - }; - - /// Load balancing - auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, is_unprioritized_task); - - LOG_DEBUG(log, "Processing " << current_task_status_path); - - CleanStateClock clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); - - LogicalClock task_start_clock; - { - Coordination::Stat stat{}; - if (zookeeper->exists(task_partition.getPartitionShardsPath(), &stat)) - task_start_clock = LogicalClock(stat.mzxid); - } - - /// Do not start if partition is dirty, try to clean it - if (clean_state_clock.is_clean() - && (!task_start_clock.hasHappened() || clean_state_clock.discovery_zxid <= task_start_clock)) - { - LOG_DEBUG(log, "Partition " << task_partition.name << " appears to be clean"); - zookeeper->createAncestors(current_task_status_path); - } - else - { - LOG_DEBUG(log, "Partition " << task_partition.name << " is dirty, try to drop it"); - - try - { - tryDropPartition(task_partition, zookeeper, clean_state_clock); - } - catch (...) - { - tryLogCurrentException(log, "An error occurred when clean partition"); - } - - return PartitionTaskStatus::Error; - } - - /// Create ephemeral node to mark that we are active and process the partition - zookeeper->createAncestors(current_task_is_active_path); - zkutil::EphemeralNodeHolderPtr partition_task_node_holder; - try - { - partition_task_node_holder = zkutil::EphemeralNodeHolder::create(current_task_is_active_path, *zookeeper, host_id); - } - catch (const Coordination::Exception & e) - { - if (e.code == Coordination::ZNODEEXISTS) - { - LOG_DEBUG(log, "Someone is already processing " << current_task_is_active_path); - return PartitionTaskStatus::Active; - } - - throw; - } - - /// Exit if task has been already processed; - /// create blocking node to signal cleaning up if it is abandoned - { - String status_data; - if (zookeeper->tryGet(current_task_status_path, status_data)) - { - TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); - if (status.state == TaskState::Finished) - { - LOG_DEBUG(log, "Task " << current_task_status_path << " has been successfully executed by " << status.owner); - return PartitionTaskStatus::Finished; - } - - // Task is abandoned, initialize DROP PARTITION - LOG_DEBUG(log, "Task " << current_task_status_path << " has not been successfully finished by " << - status.owner << ". Partition will be dropped and refilled."); - - create_is_dirty_node(clean_state_clock); - return PartitionTaskStatus::Error; - } - } - - /// Check that destination partition is empty if we are first worker - /// NOTE: this check is incorrect if pull and push tables have different partition key! - String clean_start_status; - if (!zookeeper->tryGet(task_partition.getPartitionCleanStartPath(), clean_start_status) || clean_start_status != "ok") - { - zookeeper->createIfNotExists(task_partition.getPartitionCleanStartPath(), ""); - auto checker = zkutil::EphemeralNodeHolder::create(task_partition.getPartitionCleanStartPath() + "/checker", - *zookeeper, host_id); - // Maybe we are the first worker - ASTPtr query_select_ast = get_select_query(task_shard.table_split_shard, "count()"); - UInt64 count; - { - Context local_context = context; - // Use pull (i.e. readonly) settings, but fetch data from destination servers - local_context.getSettingsRef() = task_cluster->settings_pull; - local_context.getSettingsRef().skip_unavailable_shards = true; - - Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_select_ast, local_context)->execute().in); - count = (block) ? block.safeGetByPosition(0).column->getUInt(0) : 0; - } - - if (count != 0) - { - Coordination::Stat stat_shards{}; - zookeeper->get(task_partition.getPartitionShardsPath(), &stat_shards); - - /// NOTE: partition is still fresh if dirt discovery happens before cleaning - if (stat_shards.numChildren == 0) - { - LOG_WARNING(log, "There are no workers for partition " << task_partition.name - << ", but destination table contains " << count << " rows" - << ". Partition will be dropped and refilled."); - - create_is_dirty_node(clean_state_clock); - return PartitionTaskStatus::Error; - } - } - zookeeper->set(task_partition.getPartitionCleanStartPath(), "ok"); - } - /// At this point, we need to sync that the destination table is clean - /// before any actual work - - /// Try start processing, create node about it - { - String start_state = TaskStateWithOwner::getData(TaskState::Started, host_id); - CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); - if (clean_state_clock != new_clean_state_clock) - { - LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); - return PartitionTaskStatus::Error; - } - else if (!new_clean_state_clock.is_clean()) - { - LOG_INFO(log, "Partition " << task_partition.name << " is dirty and will be dropped and refilled"); - create_is_dirty_node(new_clean_state_clock); - return PartitionTaskStatus::Error; - } - zookeeper->create(current_task_status_path, start_state, zkutil::CreateMode::Persistent); - } - - /// Try create table (if not exists) on each shard - { - auto create_query_push_ast = rewriteCreateQueryStorage(task_shard.current_pull_table_create_query, - task_table.table_push, task_table.engine_push_ast); - create_query_push_ast->as().if_not_exists = true; - String query = queryToString(create_query_push_ast); - - LOG_DEBUG(log, "Create destination tables. Query: " << query); - UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, - create_query_push_ast, &task_cluster->settings_push, - PoolMode::GET_MANY); - LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << - " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); - } - - /// Do the copying - { - bool inject_fault = false; - if (copy_fault_probability > 0) - { - double value = std::uniform_real_distribution<>(0, 1)(task_table.task_cluster.random_engine); - inject_fault = value < copy_fault_probability; - } - - // Select all fields - ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", inject_fault ? "1" : ""); - - LOG_DEBUG(log, "Executing SELECT query and pull from " << task_shard.getDescription() - << " : " << queryToString(query_select_ast)); - - ASTPtr query_insert_ast; - { - String query; - query += "INSERT INTO " + getQuotedTable(task_shard.table_split_shard) + " VALUES "; - - ParserQuery p_query(query.data() + query.size()); - query_insert_ast = parseQuery(p_query, query, 0); - - LOG_DEBUG(log, "Executing INSERT query: " << query); - } - - try - { - /// Custom INSERT SELECT implementation - Context context_select = context; - context_select.getSettingsRef() = task_cluster->settings_pull; - - Context context_insert = context; - context_insert.getSettingsRef() = task_cluster->settings_push; - - BlockInputStreamPtr input; - BlockOutputStreamPtr output; - { - BlockIO io_select = InterpreterFactory::get(query_select_ast, context_select)->execute(); - BlockIO io_insert = InterpreterFactory::get(query_insert_ast, context_insert)->execute(); - - input = io_select.in; - output = io_insert.out; - } - - /// Fail-fast optimization to abort copying when the current clean state expires - std::future future_is_dirty_checker; - - Stopwatch watch(CLOCK_MONOTONIC_COARSE); - constexpr UInt64 check_period_milliseconds = 500; - - /// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copying data - auto cancel_check = [&] () - { - if (zookeeper->expired()) - throw Exception("ZooKeeper session is expired, cancel INSERT SELECT", ErrorCodes::UNFINISHED); - - if (!future_is_dirty_checker.valid()) - future_is_dirty_checker = zookeeper->asyncExists(is_dirty_flag_path); - - /// check_period_milliseconds should less than average insert time of single block - /// Otherwise, the insertion will slow a little bit - if (watch.elapsedMilliseconds() >= check_period_milliseconds) - { - Coordination::ExistsResponse status = future_is_dirty_checker.get(); - - if (status.error != Coordination::ZNONODE) - { - LogicalClock dirt_discovery_epoch (status.stat.mzxid); - if (dirt_discovery_epoch == clean_state_clock.discovery_zxid) - return false; - throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED); - } - } - - return false; - }; - - /// Update statistics - /// It is quite rough: bytes_copied don't take into account DROP PARTITION. - auto update_stats = [&cluster_partition] (const Block & block) - { - cluster_partition.bytes_copied += block.bytes(); - cluster_partition.rows_copied += block.rows(); - cluster_partition.blocks_copied += 1; - }; - - /// Main work is here - copyData(*input, *output, cancel_check, update_stats); - - // Just in case - if (future_is_dirty_checker.valid()) - future_is_dirty_checker.get(); - - if (inject_fault) - throw Exception("Copy fault injection is activated", ErrorCodes::UNFINISHED); - } - catch (...) - { - tryLogCurrentException(log, "An error occurred during copying, partition will be marked as dirty"); - return PartitionTaskStatus::Error; - } - } - - /// Finalize the processing, change state of current partition task (and also check is_dirty flag) - { - String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); - CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); - if (clean_state_clock != new_clean_state_clock) - { - LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); - return PartitionTaskStatus::Error; - } - else if (!new_clean_state_clock.is_clean()) - { - LOG_INFO(log, "Partition " << task_partition.name << " became dirty and will be dropped and refilled"); - create_is_dirty_node(new_clean_state_clock); - return PartitionTaskStatus::Error; - } - zookeeper->set(current_task_status_path, state_finished, 0); - } - - LOG_INFO(log, "Partition " << task_partition.name << " copied"); - return PartitionTaskStatus::Finished; - } - - void dropAndCreateLocalTable(const ASTPtr & create_ast) - { - const auto & create = create_ast->as(); - dropLocalTableIfExists({create.database, create.table}); - - InterpreterCreateQuery interpreter(create_ast, context); - interpreter.execute(); - } - - void dropLocalTableIfExists(const DatabaseAndTableName & table_name) const - { - auto drop_ast = std::make_shared(); - drop_ast->if_exists = true; - drop_ast->database = table_name.first; - drop_ast->table = table_name.second; - - InterpreterDropQuery interpreter(drop_ast, context); - interpreter.execute(); - } - - String getRemoteCreateTable(const DatabaseAndTableName & table, Connection & connection, const Settings * settings = nullptr) - { - String query = "SHOW CREATE TABLE " + getQuotedTable(table); - Block block = getBlockWithAllStreamData(std::make_shared( - connection, query, InterpreterShowCreateQuery::getSampleBlock(), context, settings)); - - return typeid_cast(*block.safeGetByPosition(0).column).getDataAt(0).toString(); - } - - ASTPtr getCreateTableForPullShard(const ConnectionTimeouts & timeouts, TaskShard & task_shard) - { - /// Fetch and parse (possibly) new definition - auto connection_entry = task_shard.info.pool->get(timeouts, &task_cluster->settings_pull); - String create_query_pull_str = getRemoteCreateTable( - task_shard.task_table.table_pull, - *connection_entry, - &task_cluster->settings_pull); - - ParserCreateQuery parser_create_query; - return parseQuery(parser_create_query, create_query_pull_str, 0); - } - - void createShardInternalTables(const ConnectionTimeouts & timeouts, TaskShard & task_shard, bool create_split = true) - { - TaskTable & task_table = task_shard.task_table; - - /// We need to update table definitions for each part, it could be changed after ALTER - task_shard.current_pull_table_create_query = getCreateTableForPullShard(timeouts, task_shard); - - /// Create local Distributed tables: - /// a table fetching data from current shard and a table inserting data to the whole destination cluster - String read_shard_prefix = ".read_shard_" + toString(task_shard.indexInCluster()) + "."; - String split_shard_prefix = ".split."; - task_shard.table_read_shard = DatabaseAndTableName(working_database_name, read_shard_prefix + task_table.table_id); - task_shard.table_split_shard = DatabaseAndTableName(working_database_name, split_shard_prefix + task_table.table_id); - - /// Create special cluster with single shard - String shard_read_cluster_name = read_shard_prefix + task_table.cluster_pull_name; - ClusterPtr cluster_pull_current_shard = task_table.cluster_pull->getClusterWithSingleShard(task_shard.indexInCluster()); - context.setCluster(shard_read_cluster_name, cluster_pull_current_shard); - - auto storage_shard_ast = createASTStorageDistributed(shard_read_cluster_name, task_table.table_pull.first, task_table.table_pull.second); - const auto & storage_split_ast = task_table.engine_split_ast; - - auto create_query_ast = removeAliasColumnsFromCreateQuery(task_shard.current_pull_table_create_query); - auto create_table_pull_ast = rewriteCreateQueryStorage(create_query_ast, task_shard.table_read_shard, storage_shard_ast); - auto create_table_split_ast = rewriteCreateQueryStorage(create_query_ast, task_shard.table_split_shard, storage_split_ast); - - dropAndCreateLocalTable(create_table_pull_ast); - - if (create_split) - dropAndCreateLocalTable(create_table_split_ast); - } - - - std::set getShardPartitions(const ConnectionTimeouts & timeouts, TaskShard & task_shard) - { - createShardInternalTables(timeouts, task_shard, false); - - TaskTable & task_table = task_shard.task_table; - - String query; - { - WriteBufferFromOwnString wb; - wb << "SELECT DISTINCT " << queryToString(task_table.engine_push_partition_key_ast) << " AS partition FROM" - << " " << getQuotedTable(task_shard.table_read_shard) << " ORDER BY partition DESC"; - query = wb.str(); - } - - ParserQuery parser_query(query.data() + query.size()); - ASTPtr query_ast = parseQuery(parser_query, query, 0); - - LOG_DEBUG(log, "Computing destination partition set, executing query: " << query); - - Context local_context = context; - local_context.setSettings(task_cluster->settings_pull); - Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_ast, local_context)->execute().in); - - std::set res; - if (block) - { - ColumnWithTypeAndName & column = block.getByPosition(0); - task_shard.partition_key_column = column; - - for (size_t i = 0; i < column.column->size(); ++i) - { - WriteBufferFromOwnString wb; - column.type->serializeAsTextQuoted(*column.column, i, wb, FormatSettings()); - res.emplace(wb.str()); - } - } - - LOG_DEBUG(log, "There are " << res.size() << " destination partitions in shard " << task_shard.getDescription()); - - return res; - } - - bool checkShardHasPartition(const ConnectionTimeouts & timeouts, TaskShard & task_shard, const String & partition_quoted_name) - { - createShardInternalTables(timeouts, task_shard, false); - - TaskTable & task_table = task_shard.task_table; - - std::string query = "SELECT 1 FROM " + getQuotedTable(task_shard.table_read_shard) - + " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + partition_quoted_name + " AS partition_key))"; - - if (!task_table.where_condition_str.empty()) - query += " AND (" + task_table.where_condition_str + ")"; - - query += " LIMIT 1"; - - LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " - << partition_quoted_name << " existence, executing query: " << query); - - ParserQuery parser_query(query.data() + query.size()); - ASTPtr query_ast = parseQuery(parser_query, query, 0); - - Context local_context = context; - local_context.setSettings(task_cluster->settings_pull); - return InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows() != 0; - } - - /** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster - * Returns number of shards for which at least one replica executed query successfully - */ - UInt64 executeQueryOnCluster( - const ClusterPtr & cluster, - const String & query, - const ASTPtr & query_ast_ = nullptr, - const Settings * settings = nullptr, - PoolMode pool_mode = PoolMode::GET_ALL, - UInt64 max_successful_executions_per_shard = 0) const - { - auto num_shards = cluster->getShardsInfo().size(); - std::vector per_shard_num_successful_replicas(num_shards, 0); - - ASTPtr query_ast; - if (query_ast_ == nullptr) - { - ParserQuery p_query(query.data() + query.size()); - query_ast = parseQuery(p_query, query, 0); - } - else - query_ast = query_ast_; - - - /// We need to execute query on one replica at least - auto do_for_shard = [&] (UInt64 shard_index) - { - const Cluster::ShardInfo & shard = cluster->getShardsInfo().at(shard_index); - UInt64 & num_successful_executions = per_shard_num_successful_replicas.at(shard_index); - num_successful_executions = 0; - - auto increment_and_check_exit = [&] () -> bool - { - ++num_successful_executions; - return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard; - }; - - UInt64 num_replicas = cluster->getShardsAddresses().at(shard_index).size(); - UInt64 num_local_replicas = shard.getLocalNodeCount(); - UInt64 num_remote_replicas = num_replicas - num_local_replicas; - - /// In that case we don't have local replicas, but do it just in case - for (UInt64 i = 0; i < num_local_replicas; ++i) - { - auto interpreter = InterpreterFactory::get(query_ast, context); - interpreter->execute(); - - if (increment_and_check_exit()) - return; - } - - /// Will try to make as many as possible queries - if (shard.hasRemoteConnections()) - { - Settings current_settings = settings ? *settings : task_cluster->settings_common; - current_settings.max_parallel_replicas = num_remote_replicas ? num_remote_replicas : 1; - - auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings).getSaturated(current_settings.max_execution_time); - auto connections = shard.pool->getMany(timeouts, ¤t_settings, pool_mode); - - for (auto & connection : connections) - { - if (connection.isNull()) - continue; - - try - { - /// CREATE TABLE and DROP PARTITION queries return empty block - RemoteBlockInputStream stream{*connection, query, Block{}, context, ¤t_settings}; - NullBlockOutputStream output{Block{}}; - copyData(stream, output); - - if (increment_and_check_exit()) - return; - } - catch (const Exception &) - { - LOG_INFO(log, getCurrentExceptionMessage(false, true)); - } - } - } - }; - - { - ThreadPool thread_pool(std::min(num_shards, getNumberOfPhysicalCPUCores())); - - for (UInt64 shard_index = 0; shard_index < num_shards; ++shard_index) - thread_pool.scheduleOrThrowOnError([=] { do_for_shard(shard_index); }); - - thread_pool.wait(); - } - - UInt64 successful_shards = 0; - for (UInt64 num_replicas : per_shard_num_successful_replicas) - successful_shards += (num_replicas > 0); - - return successful_shards; - } - -private: - String task_zookeeper_path; - String task_description_path; - String host_id; - String working_database_name; - - /// Auto update config stuff - UInt64 task_descprtion_current_version = 1; - std::atomic task_descprtion_version{1}; - Coordination::WatchCallback task_description_watch_callback; - /// ZooKeeper session used to set the callback - zkutil::ZooKeeperPtr task_description_watch_zookeeper; - - ConfigurationPtr task_cluster_initial_config; - ConfigurationPtr task_cluster_current_config; - Coordination::Stat task_description_current_stat{}; - - std::unique_ptr task_cluster; - - bool is_safe_mode = false; - double copy_fault_probability = 0.0; - - Context & context; - Poco::Logger * log; - - std::chrono::milliseconds default_sleep_time{1000}; -}; - - -/// ClusterCopierApp - - -void ClusterCopierApp::initialize(Poco::Util::Application & self) -{ - is_help = config().has("help"); - if (is_help) - return; - - config_xml_path = config().getString("config-file"); - task_path = config().getString("task-path"); - log_level = config().getString("log-level", "trace"); - is_safe_mode = config().has("safe-mode"); - if (config().has("copy-fault-probability")) - copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0); - base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current(); - // process_id is '#_' - time_t timestamp = Poco::Timestamp().epochTime(); - auto curr_pid = Poco::Process::id(); - - process_id = std::to_string(DateLUT::instance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid); - host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id; - process_path = Poco::Path(base_dir + "/clickhouse-copier_" + process_id).absolute().toString(); - Poco::File(process_path).createDirectories(); - - /// Override variables for BaseDaemon - if (config().has("log-level")) - config().setString("logger.level", config().getString("log-level")); - - if (config().has("base-dir") || !config().has("logger.log")) - config().setString("logger.log", process_path + "/log.log"); - - if (config().has("base-dir") || !config().has("logger.errorlog")) - config().setString("logger.errorlog", process_path + "/log.err.log"); - - Base::initialize(self); -} - - -void ClusterCopierApp::handleHelp(const std::string &, const std::string &) -{ - Poco::Util::HelpFormatter helpFormatter(options()); - helpFormatter.setCommand(commandName()); - helpFormatter.setHeader("Copies tables from one cluster to another"); - helpFormatter.setUsage("--config-file --task-path "); - helpFormatter.format(std::cerr); - - stopOptionsProcessing(); -} - - -void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options) -{ - Base::defineOptions(options); - - options.addOption(Poco::Util::Option("task-path", "", "path to task in ZooKeeper") - .argument("task-path").binding("task-path")); - options.addOption(Poco::Util::Option("task-file", "", "path to task file for uploading in ZooKeeper to task-path") - .argument("task-file").binding("task-file")); - options.addOption(Poco::Util::Option("task-upload-force", "", "Force upload task-file even node already exists") - .argument("task-upload-force").binding("task-upload-force")); - options.addOption(Poco::Util::Option("safe-mode", "", "disables ALTER DROP PARTITION in case of errors") - .binding("safe-mode")); - options.addOption(Poco::Util::Option("copy-fault-probability", "", "the copying fails with specified probability (used to test partition state recovering)") - .argument("copy-fault-probability").binding("copy-fault-probability")); - options.addOption(Poco::Util::Option("log-level", "", "sets log level") - .argument("log-level").binding("log-level")); - options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consecutive copier launches will populate /base-dir/launch_id/* directories") - .argument("base-dir").binding("base-dir")); - - using Me = std::decay_t; - options.addOption(Poco::Util::Option("help", "", "produce this help message").binding("help") - .callback(Poco::Util::OptionCallback(this, &Me::handleHelp))); -} - - -void ClusterCopierApp::mainImpl() -{ - StatusFile status_file(process_path + "/status"); - ThreadStatus thread_status; - - auto log = &logger(); - LOG_INFO(log, "Starting clickhouse-copier (" - << "id " << process_id << ", " - << "host_id " << host_id << ", " - << "path " << process_path << ", " - << "revision " << ClickHouseRevision::get() << ")"); - - auto context = std::make_unique(Context::createGlobal()); - context->makeGlobalContext(); - SCOPE_EXIT(context->shutdown()); - - context->setConfig(loaded_config.configuration); - context->setApplicationType(Context::ApplicationType::LOCAL); - context->setPath(process_path); - - registerFunctions(); - registerAggregateFunctions(); - registerTableFunctions(); - registerStorages(); - registerDictionaries(); - registerDisks(); - - static const std::string default_database = "_local"; - context->addDatabase(default_database, std::make_shared(default_database)); - context->setCurrentDatabase(default_database); - - /// Initialize query scope just in case. - CurrentThread::QueryScope query_scope(*context); - - auto copier = std::make_unique(task_path, host_id, default_database, *context); - copier->setSafeMode(is_safe_mode); - copier->setCopyFaultProbability(copy_fault_probability); - - auto task_file = config().getString("task-file", ""); - if (!task_file.empty()) - copier->uploadTaskDescription(task_path, task_file, config().getBool("task-upload-force", false)); - - copier->init(); - copier->process(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(context->getSettingsRef())); - - /// Reset ZooKeeper before removing ClusterCopier. - /// Otherwise zookeeper watch can call callback which use already removed ClusterCopier object. - context->resetZooKeeper(); -} - - -int ClusterCopierApp::main(const std::vector &) -{ - if (is_help) - return 0; - - try - { - mainImpl(); - } - catch (...) - { - tryLogCurrentException(&Poco::Logger::root(), __PRETTY_FUNCTION__); - auto code = getCurrentExceptionCode(); - - return (code) ? code : -1; - } - - return 0; -} - - -} - -#pragma GCC diagnostic ignored "-Wunused-function" -#pragma GCC diagnostic ignored "-Wmissing-declarations" - -int mainEntryClickHouseClusterCopier(int argc, char ** argv) -{ - try - { - DB::ClusterCopierApp app; - return app.run(argc, argv); - } - catch (...) - { - std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; - auto code = DB::getCurrentExceptionCode(); - - return (code) ? code : -1; - } -} diff --git a/dbms/programs/copier/ClusterCopier.h b/dbms/programs/copier/ClusterCopier.h index fe228fd6194..f4f37b97c26 100644 --- a/dbms/programs/copier/ClusterCopier.h +++ b/dbms/programs/copier/ClusterCopier.h @@ -1,87 +1,1525 @@ #pragma once -#include -#include -/* clickhouse cluster copier util - * Copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed fault-tolerant manner. - * - * See overview in the docs: docs/en/utils/clickhouse-copier.md - * - * Implementation details: - * - * cluster-copier workers pull each partition of each shard of the source cluster and push it to the destination cluster through - * Distributed table (to preform data resharding). So, worker job is a partition of a source shard. - * A job has three states: Active, Finished and Abandoned. Abandoned means that worker died and did not finish the job. - * - * If an error occurred during the copying (a worker failed or a worker did not finish the INSERT), then the whole partition (on - * all destination servers) should be dropped and refilled. So, copying entity is a partition of all destination shards. - * If a failure is detected a special /is_dirty node is created in ZooKeeper signalling that other workers copying the same partition - * should stop, after a refilling procedure should start. - * - * ZooKeeper task node has the following structure: - * /task/path_root - path passed in --task-path parameter - * /description - contains user-defined XML config of the task - * /task_active_workers - contains ephemeral nodes of all currently active workers, used to implement max_workers limitation - * /server_fqdn#PID_timestamp - cluster-copier worker ID - * ... - * /tables - directory with table tasks - * /cluster.db.table1 - directory of table_hits task - * /partition1 - directory for partition1 - * /shards - directory for source cluster shards - * /1 - worker job for the first shard of partition1 of table test.hits - * Contains info about current status (Active or Finished) and worker ID. - * /2 - * ... - * /partition_active_workers - * /1 - for each job in /shards a corresponding ephemeral node created in /partition_active_workers - * It is used to detect Abandoned jobs (if there is Active node in /shards and there is no node in - * /partition_active_workers). - * Also, it is used to track active workers in the partition (when we need to refill the partition we do - * not DROP PARTITION while there are active workers) - * /2 - * ... - * /is_dirty - the node is set if some worker detected that an error occurred (the INSERT is failed or an Abandoned node is - * detected). If the node appeared workers in this partition should stop and start cleaning and refilling - * partition procedure. - * During this procedure a single 'cleaner' worker is selected. The worker waits for stopping all partition - * workers, removes /shards node, executes DROP PARTITION on each destination node and removes /is_dirty node. - * /cleaner- An ephemeral node used to select 'cleaner' worker. Contains ID of the worker. - * /cluster.db.table2 - * ... - */ +#include "Internals.h" +#include "ZookeeperStaff.h" namespace DB { -class ClusterCopierApp : public BaseDaemon +using ConfigurationPtr = Poco::AutoPtr; + +static ConfigurationPtr getConfigurationFromXMLString(const std::string & xml_data) +{ + std::stringstream ss(xml_data); + Poco::XML::InputSource input_source{ss}; + return {new Poco::Util::XMLConfiguration{&input_source}}; +} + +class ClusterCopier { public: - void initialize(Poco::Util::Application & self) override; + ClusterCopier(String task_path_, + String host_id_, + String proxy_database_name_, + Context & context_) + : + task_zookeeper_path(std::move(task_path_)), + host_id(std::move(host_id_)), + working_database_name(std::move(proxy_database_name_)), + context(context_), + log(&Poco::Logger::get("ClusterCopier")) + { + } - void handleHelp(const std::string &, const std::string &); + void init() + { + auto zookeeper = context.getZooKeeper(); - void defineOptions(Poco::Util::OptionSet & options) override; + task_description_watch_callback = [this] (const Coordination::WatchResponse & response) + { + if (response.error != Coordination::ZOK) + return; + UInt64 version = ++task_descprtion_version; + LOG_DEBUG(log, "Task description should be updated, local version " << version); + }; - int main(const std::vector &) override; + task_description_path = task_zookeeper_path + "/description"; + task_cluster = std::make_unique(task_zookeeper_path, working_database_name); + + reloadTaskDescription(); + task_cluster_initial_config = task_cluster_current_config; + + task_cluster->loadTasks(*task_cluster_initial_config); + context.setClustersConfig(task_cluster_initial_config, task_cluster->clusters_prefix); + + /// Set up shards and their priority + task_cluster->random_engine.seed(task_cluster->random_device()); + for (auto & task_table : task_cluster->table_tasks) + { + task_table.cluster_pull = context.getCluster(task_table.cluster_pull_name); + task_table.cluster_push = context.getCluster(task_table.cluster_push_name); + task_table.initShards(task_cluster->random_engine); + } + + LOG_DEBUG(log, "Will process " << task_cluster->table_tasks.size() << " table tasks"); + + /// Do not initialize tables, will make deferred initialization in process() + + zookeeper->createAncestors(getWorkersPathVersion() + "/"); + zookeeper->createAncestors(getWorkersPath() + "/"); + } + + template + decltype(auto) retry(T && func, UInt64 max_tries = 100) + { + std::exception_ptr exception; + + for (UInt64 try_number = 1; try_number <= max_tries; ++try_number) + { + try + { + return func(); + } + catch (...) + { + exception = std::current_exception(); + if (try_number < max_tries) + { + tryLogCurrentException(log, "Will retry"); + std::this_thread::sleep_for(default_sleep_time); + } + } + } + + std::rethrow_exception(exception); + } + + + void discoverShardPartitions(const ConnectionTimeouts & timeouts, const TaskShardPtr & task_shard) + { + TaskTable & task_table = task_shard->task_table; + + LOG_INFO(log, "Discover partitions of shard " << task_shard->getDescription()); + + auto get_partitions = [&] () { return getShardPartitions(timeouts, *task_shard); }; + auto existing_partitions_names = retry(get_partitions, 60); + Strings filtered_partitions_names; + Strings missing_partitions; + + /// Check that user specified correct partition names + auto check_partition_format = [] (const DataTypePtr & type, const String & partition_text_quoted) + { + MutableColumnPtr column_dummy = type->createColumn(); + ReadBufferFromString rb(partition_text_quoted); + + try + { + type->deserializeAsTextQuoted(*column_dummy, rb, FormatSettings()); + } + catch (Exception & e) + { + throw Exception("Partition " + partition_text_quoted + " has incorrect format. " + e.displayText(), ErrorCodes::BAD_ARGUMENTS); + } + }; + + if (task_table.has_enabled_partitions) + { + /// Process partition in order specified by + for (const String & partition_name : task_table.enabled_partitions) + { + /// Check that user specified correct partition names + check_partition_format(task_shard->partition_key_column.type, partition_name); + + auto it = existing_partitions_names.find(partition_name); + + /// Do not process partition if it is not in enabled_partitions list + if (it == existing_partitions_names.end()) + { + missing_partitions.emplace_back(partition_name); + continue; + } + + filtered_partitions_names.emplace_back(*it); + } + + for (const String & partition_name : existing_partitions_names) + { + if (!task_table.enabled_partitions_set.count(partition_name)) + { + LOG_DEBUG(log, "Partition " << partition_name << " will not be processed, since it is not in " + << "enabled_partitions of " << task_table.table_id); + } + } + } + else + { + for (const String & partition_name : existing_partitions_names) + filtered_partitions_names.emplace_back(partition_name); + } + + for (const String & partition_name : filtered_partitions_names) + { + task_shard->partition_tasks.emplace(partition_name, ShardPartition(*task_shard, partition_name, 10)); + task_shard->checked_partitions.emplace(partition_name, true); + } + + if (!missing_partitions.empty()) + { + std::stringstream ss; + for (const String & missing_partition : missing_partitions) + ss << " " << missing_partition; + + LOG_WARNING(log, "There are no " << missing_partitions.size() << " partitions from enabled_partitions in shard " + << task_shard->getDescription() << " :" << ss.str()); + } + + LOG_DEBUG(log, "Will copy " << task_shard->partition_tasks.size() << " partitions from shard " << task_shard->getDescription()); + } + + /// Compute set of partitions, assume set of partitions aren't changed during the processing + void discoverTablePartitions(const ConnectionTimeouts & timeouts, TaskTable & task_table, UInt64 num_threads = 0) + { + /// Fetch partitions list from a shard + { + ThreadPool thread_pool(num_threads ? num_threads : 2 * getNumberOfPhysicalCPUCores()); + + for (const TaskShardPtr & task_shard : task_table.all_shards) + thread_pool.scheduleOrThrowOnError([this, timeouts, task_shard]() { discoverShardPartitions(timeouts, task_shard); }); + + LOG_DEBUG(log, "Waiting for " << thread_pool.active() << " setup jobs"); + thread_pool.wait(); + } + } + + void uploadTaskDescription(const std::string & task_path, const std::string & task_file, const bool force) + { + auto local_task_description_path = task_path + "/description"; + + String task_config_str; + { + ReadBufferFromFile in(task_file); + readStringUntilEOF(task_config_str, in); + } + if (task_config_str.empty()) + return; + + auto zookeeper = context.getZooKeeper(); + + zookeeper->createAncestors(local_task_description_path); + auto code = zookeeper->tryCreate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); + if (code && force) + zookeeper->createOrUpdate(local_task_description_path, task_config_str, zkutil::CreateMode::Persistent); + + LOG_DEBUG(log, "Task description " << ((code && !force) ? "not " : "") << "uploaded to " << local_task_description_path << " with result " << code << " ("<< zookeeper->error2string(code) << ")"); + } + + void reloadTaskDescription() + { + auto zookeeper = context.getZooKeeper(); + task_description_watch_zookeeper = zookeeper; + + String task_config_str; + Coordination::Stat stat{}; + int code; + + zookeeper->tryGetWatch(task_description_path, task_config_str, &stat, task_description_watch_callback, &code); + if (code) + throw Exception("Can't get description node " + task_description_path, ErrorCodes::BAD_ARGUMENTS); + + LOG_DEBUG(log, "Loading description, zxid=" << task_description_current_stat.czxid); + auto config = getConfigurationFromXMLString(task_config_str); + + /// Setup settings + task_cluster->reloadSettings(*config); + context.getSettingsRef() = task_cluster->settings_common; + + task_cluster_current_config = config; + task_description_current_stat = stat; + } + + void updateConfigIfNeeded() + { + UInt64 version_to_update = task_descprtion_version; + bool is_outdated_version = task_descprtion_current_version != version_to_update; + bool is_expired_session = !task_description_watch_zookeeper || task_description_watch_zookeeper->expired(); + + if (!is_outdated_version && !is_expired_session) + return; + + LOG_DEBUG(log, "Updating task description"); + reloadTaskDescription(); + + task_descprtion_current_version = version_to_update; + } + + void process(const ConnectionTimeouts & timeouts) + { + for (TaskTable & task_table : task_cluster->table_tasks) + { + LOG_INFO(log, "Process table task " << task_table.table_id << " with " + << task_table.all_shards.size() << " shards, " << task_table.local_shards.size() << " of them are local ones"); + + if (task_table.all_shards.empty()) + continue; + + /// Discover partitions of each shard and total set of partitions + if (!task_table.has_enabled_partitions) + { + /// If there are no specified enabled_partitions, we must discover them manually + discoverTablePartitions(timeouts, task_table); + + /// After partitions of each shard are initialized, initialize cluster partitions + for (const TaskShardPtr & task_shard : task_table.all_shards) + { + for (const auto & partition_elem : task_shard->partition_tasks) + { + const String & partition_name = partition_elem.first; + task_table.cluster_partitions.emplace(partition_name, ClusterPartition{}); + } + } + + for (auto & partition_elem : task_table.cluster_partitions) + { + const String & partition_name = partition_elem.first; + + for (const TaskShardPtr & task_shard : task_table.all_shards) + task_shard->checked_partitions.emplace(partition_name); + + task_table.ordered_partition_names.emplace_back(partition_name); + } + } + else + { + /// If enabled_partitions are specified, assume that each shard has all partitions + /// We will refine partition set of each shard in future + + for (const String & partition_name : task_table.enabled_partitions) + { + task_table.cluster_partitions.emplace(partition_name, ClusterPartition{}); + task_table.ordered_partition_names.emplace_back(partition_name); + } + } + + task_table.watch.restart(); + + /// Retry table processing + bool table_is_done = false; + for (UInt64 num_table_tries = 0; num_table_tries < max_table_tries; ++num_table_tries) + { + if (tryProcessTable(timeouts, task_table)) + { + table_is_done = true; + break; + } + } + + if (!table_is_done) + { + throw Exception("Too many tries to process table " + task_table.table_id + ". Abort remaining execution", + ErrorCodes::UNFINISHED); + } + } + } + + /// Disables DROP PARTITION commands that used to clear data after errors + void setSafeMode(bool is_safe_mode_ = true) + { + is_safe_mode = is_safe_mode_; + } + + void setCopyFaultProbability(double copy_fault_probability_) + { + copy_fault_probability = copy_fault_probability_; + } + + + protected: + + String getWorkersPath() const + { + return task_cluster->task_zookeeper_path + "/task_active_workers"; + } + + String getWorkersPathVersion() const + { + return getWorkersPath() + "_version"; + } + + String getCurrentWorkerNodePath() const + { + return getWorkersPath() + "/" + host_id; + } + + zkutil::EphemeralNodeHolder::Ptr createTaskWorkerNodeAndWaitIfNeed( + const zkutil::ZooKeeperPtr & zookeeper, + const String & description, + bool unprioritized) + { + std::chrono::milliseconds current_sleep_time = default_sleep_time; + static constexpr std::chrono::milliseconds max_sleep_time(30000); // 30 sec + + if (unprioritized) + std::this_thread::sleep_for(current_sleep_time); + + String workers_version_path = getWorkersPathVersion(); + String workers_path = getWorkersPath(); + String current_worker_path = getCurrentWorkerNodePath(); + + UInt64 num_bad_version_errors = 0; + + while (true) + { + updateConfigIfNeeded(); + + Coordination::Stat stat{}; + zookeeper->get(workers_version_path, &stat); + auto version = stat.version; + zookeeper->get(workers_path, &stat); + + if (static_cast(stat.numChildren) >= task_cluster->max_workers) + { + LOG_DEBUG(log, "Too many workers (" << stat.numChildren << ", maximum " << task_cluster->max_workers << ")" + << ". Postpone processing " << description); + + if (unprioritized) + current_sleep_time = std::min(max_sleep_time, current_sleep_time + default_sleep_time); + + std::this_thread::sleep_for(current_sleep_time); + num_bad_version_errors = 0; + } + else + { + Coordination::Requests ops; + ops.emplace_back(zkutil::makeSetRequest(workers_version_path, description, version)); + ops.emplace_back(zkutil::makeCreateRequest(current_worker_path, description, zkutil::CreateMode::Ephemeral)); + Coordination::Responses responses; + auto code = zookeeper->tryMulti(ops, responses); + + if (code == Coordination::ZOK || code == Coordination::ZNODEEXISTS) + return std::make_shared(current_worker_path, *zookeeper, false, false, description); + + if (code == Coordination::ZBADVERSION) + { + ++num_bad_version_errors; + + /// Try to make fast retries + if (num_bad_version_errors > 3) + { + LOG_DEBUG(log, "A concurrent worker has just been added, will check free worker slots again"); + std::chrono::milliseconds random_sleep_time(std::uniform_int_distribution(1, 1000)(task_cluster->random_engine)); + std::this_thread::sleep_for(random_sleep_time); + num_bad_version_errors = 0; + } + } + else + throw Coordination::Exception(code); + } + } + } + + /** Checks that the whole partition of a table was copied. We should do it carefully due to dirty lock. + * State of some task could change during the processing. + * We have to ensure that all shards have the finished state and there is no dirty flag. + * Moreover, we have to check status twice and check zxid, because state can change during the checking. + */ + bool checkPartitionIsDone(const TaskTable & task_table, const String & partition_name, const TasksShard & shards_with_partition) + { + LOG_DEBUG(log, "Check that all shards processed partition " << partition_name << " successfully"); + + auto zookeeper = context.getZooKeeper(); + + Strings status_paths; + for (auto & shard : shards_with_partition) + { + ShardPartition & task_shard_partition = shard->partition_tasks.find(partition_name)->second; + status_paths.emplace_back(task_shard_partition.getShardStatusPath()); + } + + std::vector zxid1, zxid2; + + try + { + std::vector get_futures; + for (const String & path : status_paths) + get_futures.emplace_back(zookeeper->asyncGet(path)); + + // Check that state is Finished and remember zxid + for (auto & future : get_futures) + { + auto res = future.get(); + + TaskStateWithOwner status = TaskStateWithOwner::fromString(res.data); + if (status.state != TaskState::Finished) + { + LOG_INFO(log, "The task " << res.data << " is being rewritten by " << status.owner << ". Partition will be rechecked"); + return false; + } + + zxid1.push_back(res.stat.pzxid); + } + + // Check that partition is not dirty + { + CleanStateClock clean_state_clock ( + zookeeper, + task_table.getPartitionIsDirtyPath(partition_name), + task_table.getPartitionIsCleanedPath(partition_name) + ); + Coordination::Stat stat{}; + LogicalClock task_start_clock; + if (zookeeper->exists(task_table.getPartitionTaskStatusPath(partition_name), &stat)) + task_start_clock = LogicalClock(stat.mzxid); + zookeeper->get(task_table.getPartitionTaskStatusPath(partition_name), &stat); + if (!clean_state_clock.is_clean() || task_start_clock <= clean_state_clock.discovery_zxid) + { + LOG_INFO(log, "Partition " << partition_name << " become dirty"); + return false; + } + } + + get_futures.clear(); + for (const String & path : status_paths) + get_futures.emplace_back(zookeeper->asyncGet(path)); + + // Remember zxid of states again + for (auto & future : get_futures) + { + auto res = future.get(); + zxid2.push_back(res.stat.pzxid); + } + } + catch (const Coordination::Exception & e) + { + LOG_INFO(log, "A ZooKeeper error occurred while checking partition " << partition_name + << ". Will recheck the partition. Error: " << e.displayText()); + return false; + } + + // If all task is finished and zxid is not changed then partition could not become dirty again + for (UInt64 shard_num = 0; shard_num < status_paths.size(); ++shard_num) + { + if (zxid1[shard_num] != zxid2[shard_num]) + { + LOG_INFO(log, "The task " << status_paths[shard_num] << " is being modified now. Partition will be rechecked"); + return false; + } + } + + LOG_INFO(log, "Partition " << partition_name << " is copied successfully"); + return true; + } + + /// Removes MATERIALIZED and ALIAS columns from create table query + static ASTPtr removeAliasColumnsFromCreateQuery(const ASTPtr & query_ast) + { + const ASTs & column_asts = query_ast->as().columns_list->columns->children; + auto new_columns = std::make_shared(); + + for (const ASTPtr & column_ast : column_asts) + { + const auto & column = column_ast->as(); + + if (!column.default_specifier.empty()) + { + ColumnDefaultKind kind = columnDefaultKindFromString(column.default_specifier); + if (kind == ColumnDefaultKind::Materialized || kind == ColumnDefaultKind::Alias) + continue; + } + + new_columns->children.emplace_back(column_ast->clone()); + } + + ASTPtr new_query_ast = query_ast->clone(); + auto & new_query = new_query_ast->as(); + + auto new_columns_list = std::make_shared(); + new_columns_list->set(new_columns_list->columns, new_columns); + if (auto indices = query_ast->as()->columns_list->indices) + new_columns_list->set(new_columns_list->indices, indices->clone()); + + new_query.replace(new_query.columns_list, new_columns_list); + + return new_query_ast; + } + + /// Replaces ENGINE and table name in a create query + static std::shared_ptr rewriteCreateQueryStorage(const ASTPtr & create_query_ast, const DatabaseAndTableName & new_table, const ASTPtr & new_storage_ast) + { + const auto & create = create_query_ast->as(); + auto res = std::make_shared(create); + + if (create.storage == nullptr || new_storage_ast == nullptr) + throw Exception("Storage is not specified", ErrorCodes::LOGICAL_ERROR); + + res->database = new_table.first; + res->table = new_table.second; + + res->children.clear(); + res->set(res->columns_list, create.columns_list->clone()); + res->set(res->storage, new_storage_ast->clone()); + + return res; + } + + bool tryDropPartition(ShardPartition & task_partition, const zkutil::ZooKeeperPtr & zookeeper, const CleanStateClock & clean_state_clock) + { + if (is_safe_mode) + throw Exception("DROP PARTITION is prohibited in safe mode", ErrorCodes::NOT_IMPLEMENTED); + + TaskTable & task_table = task_partition.task_shard.task_table; + + const String current_shards_path = task_partition.getPartitionShardsPath(); + const String current_partition_active_workers_dir = task_partition.getPartitionActiveWorkersPath(); + const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); + const String dirty_cleaner_path = is_dirty_flag_path + "/cleaner"; + const String is_dirty_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); + + zkutil::EphemeralNodeHolder::Ptr cleaner_holder; + try + { + cleaner_holder = zkutil::EphemeralNodeHolder::create(dirty_cleaner_path, *zookeeper, host_id); + } + catch (const Coordination::Exception & e) + { + if (e.code == Coordination::ZNODEEXISTS) + { + LOG_DEBUG(log, "Partition " << task_partition.name << " is cleaning now by somebody, sleep"); + std::this_thread::sleep_for(default_sleep_time); + return false; + } + + throw; + } + + Coordination::Stat stat{}; + if (zookeeper->exists(current_partition_active_workers_dir, &stat)) + { + if (stat.numChildren != 0) + { + LOG_DEBUG(log, "Partition " << task_partition.name << " contains " << stat.numChildren << " active workers while trying to drop it. Going to sleep."); + std::this_thread::sleep_for(default_sleep_time); + return false; + } + else + { + zookeeper->remove(current_partition_active_workers_dir); + } + } + + { + zkutil::EphemeralNodeHolder::Ptr active_workers_lock; + try + { + active_workers_lock = zkutil::EphemeralNodeHolder::create(current_partition_active_workers_dir, *zookeeper, host_id); + } + catch (const Coordination::Exception & e) + { + if (e.code == Coordination::ZNODEEXISTS) + { + LOG_DEBUG(log, "Partition " << task_partition.name << " is being filled now by somebody, sleep"); + return false; + } + + throw; + } + + // Lock the dirty flag + zookeeper->set(is_dirty_flag_path, host_id, clean_state_clock.discovery_version.value()); + zookeeper->tryRemove(task_partition.getPartitionCleanStartPath()); + CleanStateClock my_clock(zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); + + /// Remove all status nodes + { + Strings children; + if (zookeeper->tryGetChildren(current_shards_path, children) == Coordination::ZOK) + for (const auto & child : children) + { + zookeeper->removeRecursive(current_shards_path + "/" + child); + } + } + + String query = "ALTER TABLE " + getQuotedTable(task_table.table_push); + query += " DROP PARTITION " + task_partition.name + ""; + + /// TODO: use this statement after servers will be updated up to 1.1.54310 + // query += " DROP PARTITION ID '" + task_partition.name + "'"; + + ClusterPtr & cluster_push = task_table.cluster_push; + Settings settings_push = task_cluster->settings_push; + + /// It is important, DROP PARTITION must be done synchronously + settings_push.replication_alter_partitions_sync = 2; + + LOG_DEBUG(log, "Execute distributed DROP PARTITION: " << query); + /// Limit number of max executing replicas to 1 + UInt64 num_shards = executeQueryOnCluster(cluster_push, query, nullptr, &settings_push, PoolMode::GET_ONE, 1); + + if (num_shards < cluster_push->getShardCount()) + { + LOG_INFO(log, "DROP PARTITION wasn't successfully executed on " << cluster_push->getShardCount() - num_shards << " shards"); + return false; + } + + /// Update the locking node + if (!my_clock.is_stale()) + { + zookeeper->set(is_dirty_flag_path, host_id, my_clock.discovery_version.value()); + if (my_clock.clean_state_version) + zookeeper->set(is_dirty_cleaned_path, host_id, my_clock.clean_state_version.value()); + else + zookeeper->create(is_dirty_cleaned_path, host_id, zkutil::CreateMode::Persistent); + } + else + { + LOG_DEBUG(log, "Clean state is altered when dropping the partition, cowardly bailing"); + /// clean state is stale + return false; + } + + LOG_INFO(log, "Partition " << task_partition.name << " was dropped on cluster " << task_table.cluster_push_name); + if (zookeeper->tryCreate(current_shards_path, host_id, zkutil::CreateMode::Persistent) == Coordination::ZNODEEXISTS) + zookeeper->set(current_shards_path, host_id); + } + + LOG_INFO(log, "Partition " << task_partition.name << " is safe for work now."); + return true; + } + + + static constexpr UInt64 max_table_tries = 1000; + static constexpr UInt64 max_shard_partition_tries = 600; + + bool tryProcessTable(const ConnectionTimeouts & timeouts, TaskTable & task_table) + { + /// A heuristic: if previous shard is already done, then check next one without sleeps due to max_workers constraint + bool previous_shard_is_instantly_finished = false; + + /// Process each partition that is present in cluster + for (const String & partition_name : task_table.ordered_partition_names) + { + if (!task_table.cluster_partitions.count(partition_name)) + throw Exception("There are no expected partition " + partition_name + ". It is a bug", ErrorCodes::LOGICAL_ERROR); + + ClusterPartition & cluster_partition = task_table.cluster_partitions[partition_name]; + + Stopwatch watch; + /// We will check all the shards of the table and check if they contain current partition. + TasksShard expected_shards; + UInt64 num_failed_shards = 0; + + ++cluster_partition.total_tries; + + LOG_DEBUG(log, "Processing partition " << partition_name << " for the whole cluster"); + + /// Process each source shard having current partition and copy current partition + /// NOTE: shards are sorted by "distance" to current host + bool has_shard_to_process = false; + for (const TaskShardPtr & shard : task_table.all_shards) + { + /// Does shard have a node with current partition? + if (shard->partition_tasks.count(partition_name) == 0) + { + /// If not, did we check existence of that partition previously? + if (shard->checked_partitions.count(partition_name) == 0) + { + auto check_shard_has_partition = [&] () { return checkShardHasPartition(timeouts, *shard, partition_name); }; + bool has_partition = retry(check_shard_has_partition); + + shard->checked_partitions.emplace(partition_name); + + if (has_partition) + { + shard->partition_tasks.emplace(partition_name, ShardPartition(*shard, partition_name, 10)); + LOG_DEBUG(log, "Discovered partition " << partition_name << " in shard " << shard->getDescription()); + /// To save references in the future. + auto shard_partition_it = shard->partition_tasks.find(partition_name); + PartitionPieces & shard_partition_pieces = shard_partition_it->second.pieces; + ///FIXME: Remove 10 + for (int piece_number = 0; piece_number < 10; ++piece_number) + { + auto res = checkPresentPartitionPiecesOnCurrentShard(timeouts, *shard, partition_name, piece_number); + shard_partition_pieces.emplace_back(shard_partition_it->second, piece_number, res); + } + } + else + { + LOG_DEBUG(log, "Found that shard " << shard->getDescription() << " does not contain current partition " << partition_name); + continue; + } + } + else + { + /// We have already checked that partition, but did not discover it + previous_shard_is_instantly_finished = true; + continue; + } + } + + auto it_shard_partition = shard->partition_tasks.find(partition_name); + /// Previously when we discovered that shard does not contain current partition, we skipped it. + /// At this moment partition have to be present. + if (it_shard_partition == shard->partition_tasks.end()) + throw Exception("There are no such partition in a shard. This is a bug.", ErrorCodes::LOGICAL_ERROR); + auto & partition = it_shard_partition->second; + + expected_shards.emplace_back(shard); + + /// Do not sleep if there is a sequence of already processed shards to increase startup + bool is_unprioritized_task = !previous_shard_is_instantly_finished && shard->priority.is_remote; + PartitionTaskStatus task_status = PartitionTaskStatus::Error; + bool was_error = false; + has_shard_to_process = true; + for (UInt64 try_num = 0; try_num < max_shard_partition_tries; ++try_num) + { + task_status = tryProcessPartitionTask(timeouts, partition, is_unprioritized_task); + + /// Exit if success + if (task_status == PartitionTaskStatus::Finished) + break; + + was_error = true; + + /// Skip if the task is being processed by someone + if (task_status == PartitionTaskStatus::Active) + break; + + /// Repeat on errors + std::this_thread::sleep_for(default_sleep_time); + } + + if (task_status == PartitionTaskStatus::Error) + ++num_failed_shards; + + previous_shard_is_instantly_finished = !was_error; + } + + cluster_partition.elapsed_time_seconds += watch.elapsedSeconds(); + + /// Check that whole cluster partition is done + /// Firstly check the number of failed partition tasks, then look into ZooKeeper and ensure that each partition is done + bool partition_is_done = num_failed_shards == 0; + try + { + partition_is_done = + !has_shard_to_process + || (partition_is_done && checkPartitionIsDone(task_table, partition_name, expected_shards)); + } + catch (...) + { + tryLogCurrentException(log); + partition_is_done = false; + } + + if (partition_is_done) + { + task_table.finished_cluster_partitions.emplace(partition_name); + + task_table.bytes_copied += cluster_partition.bytes_copied; + task_table.rows_copied += cluster_partition.rows_copied; + double elapsed = cluster_partition.elapsed_time_seconds; + + LOG_INFO(log, "It took " << std::fixed << std::setprecision(2) << elapsed << " seconds to copy partition " << partition_name + << ": " << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied) << " uncompressed bytes" + << ", " << formatReadableQuantity(cluster_partition.rows_copied) << " rows" + << " and " << cluster_partition.blocks_copied << " source blocks are copied"); + + if (cluster_partition.rows_copied) + { + LOG_INFO(log, "Average partition speed: " + << formatReadableSizeWithDecimalSuffix(cluster_partition.bytes_copied / elapsed) << " per second."); + } + + if (task_table.rows_copied) + { + LOG_INFO(log, "Average table " << task_table.table_id << " speed: " + << formatReadableSizeWithDecimalSuffix(task_table.bytes_copied / elapsed) << " per second."); + } + } + } + + UInt64 required_partitions = task_table.cluster_partitions.size(); + UInt64 finished_partitions = task_table.finished_cluster_partitions.size(); + bool table_is_done = finished_partitions >= required_partitions; + + if (!table_is_done) + { + LOG_INFO(log, "Table " + task_table.table_id + " is not processed yet." + << "Copied " << finished_partitions << " of " << required_partitions << ", will retry"); + } + + return table_is_done; + } + + + /// Execution status of a task + enum class PartitionTaskStatus + { + Active, + Finished, + Error, + }; + + /// Job for copying partition from particular shard. + PartitionTaskStatus tryProcessPartitionTask(const ConnectionTimeouts & timeouts, ShardPartition & task_partition, bool is_unprioritized_task) + { + PartitionTaskStatus res; + + try + { + res = processPartitionTaskImpl(timeouts, task_partition, is_unprioritized_task); + } + catch (...) + { + tryLogCurrentException(log, "An error occurred while processing partition " + task_partition.name); + res = PartitionTaskStatus::Error; + } + + /// At the end of each task check if the config is updated + try + { + updateConfigIfNeeded(); + } + catch (...) + { + tryLogCurrentException(log, "An error occurred while updating the config"); + } + + return res; + } + + PartitionTaskStatus processPartitionTaskImpl(const ConnectionTimeouts & timeouts, ShardPartition & task_partition, bool is_unprioritized_task) + { + TaskShard & task_shard = task_partition.task_shard; + TaskTable & task_table = task_shard.task_table; + ClusterPartition & cluster_partition = task_table.getClusterPartition(task_partition.name); + const size_t number_of_splits = task_table.number_of_splits; + UNUSED(number_of_splits); + + /// We need to update table definitions for each partition, it could be changed after ALTER + createShardInternalTables(timeouts, task_shard); + + auto zookeeper = context.getZooKeeper(); + + const String is_dirty_flag_path = task_partition.getCommonPartitionIsDirtyPath(); + const String is_dirty_cleaned_path = task_partition.getCommonPartitionIsCleanedPath(); + const String current_task_is_active_path = task_partition.getActiveWorkerPath(); + const String current_task_status_path = task_partition.getShardStatusPath(); + + /// Auxiliary functions: + + /// Creates is_dirty node to initialize DROP PARTITION + auto create_is_dirty_node = [&, this] (const CleanStateClock & clock) + { + if (clock.is_stale()) + LOG_DEBUG(log, "Clean state clock is stale while setting dirty flag, cowardly bailing"); + else if (!clock.is_clean()) + LOG_DEBUG(log, "Thank you, Captain Obvious"); + else if (clock.discovery_version) + { + LOG_DEBUG(log, "Updating clean state clock"); + zookeeper->set(is_dirty_flag_path, host_id, clock.discovery_version.value()); + } + else + { + LOG_DEBUG(log, "Creating clean state clock"); + zookeeper->create(is_dirty_flag_path, host_id, zkutil::CreateMode::Persistent); + } + }; + + /// Returns SELECT query filtering current partition and applying user filter + auto get_select_query = [&] (const DatabaseAndTableName & from_table, const String & fields, String limit = "", + bool enable_splitting = false, size_t current_piece_number = 0) + { + String query; + query += "SELECT " + fields + " FROM " + getQuotedTable(from_table); + /// TODO: Bad, it is better to rewrite with ASTLiteral(partition_key_field) + query += " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + task_partition.name + " AS partition_key))"; + if (enable_splitting) + query += " AND ( cityHash64(*) = " + std::to_string(current_piece_number) + " )"; + if (!task_table.where_condition_str.empty()) + query += " AND (" + task_table.where_condition_str + ")"; + if (!limit.empty()) + query += " LIMIT " + limit; + + ParserQuery p_query(query.data() + query.size()); + return parseQuery(p_query, query, 0); + }; + + /// Load balancing + auto worker_node_holder = createTaskWorkerNodeAndWaitIfNeed(zookeeper, current_task_status_path, is_unprioritized_task); + + LOG_DEBUG(log, "Processing " << current_task_status_path); + + CleanStateClock clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); + + LogicalClock task_start_clock; + { + Coordination::Stat stat{}; + if (zookeeper->exists(task_partition.getPartitionShardsPath(), &stat)) + task_start_clock = LogicalClock(stat.mzxid); + } + + /// Do not start if partition is dirty, try to clean it + if (clean_state_clock.is_clean() + && (!task_start_clock.hasHappened() || clean_state_clock.discovery_zxid <= task_start_clock)) + { + LOG_DEBUG(log, "Partition " << task_partition.name << " appears to be clean"); + zookeeper->createAncestors(current_task_status_path); + } + else + { + LOG_DEBUG(log, "Partition " << task_partition.name << " is dirty, try to drop it"); + + try + { + tryDropPartition(task_partition, zookeeper, clean_state_clock); + } + catch (...) + { + tryLogCurrentException(log, "An error occurred when clean partition"); + } + + return PartitionTaskStatus::Error; + } + + /// Create ephemeral node to mark that we are active and process the partition + zookeeper->createAncestors(current_task_is_active_path); + zkutil::EphemeralNodeHolderPtr partition_task_node_holder; + try + { + partition_task_node_holder = zkutil::EphemeralNodeHolder::create(current_task_is_active_path, *zookeeper, host_id); + } + catch (const Coordination::Exception & e) + { + if (e.code == Coordination::ZNODEEXISTS) + { + LOG_DEBUG(log, "Someone is already processing " << current_task_is_active_path); + return PartitionTaskStatus::Active; + } + + throw; + } + + /// Exit if task has been already processed; + /// create blocking node to signal cleaning up if it is abandoned + { + String status_data; + if (zookeeper->tryGet(current_task_status_path, status_data)) + { + TaskStateWithOwner status = TaskStateWithOwner::fromString(status_data); + if (status.state == TaskState::Finished) + { + LOG_DEBUG(log, "Task " << current_task_status_path << " has been successfully executed by " << status.owner); + return PartitionTaskStatus::Finished; + } + + // Task is abandoned, initialize DROP PARTITION + LOG_DEBUG(log, "Task " << current_task_status_path << " has not been successfully finished by " << + status.owner << ". Partition will be dropped and refilled."); + + create_is_dirty_node(clean_state_clock); + return PartitionTaskStatus::Error; + } + } + + /// Check that destination partition is empty if we are first worker + /// NOTE: this check is incorrect if pull and push tables have different partition key! + String clean_start_status; + if (!zookeeper->tryGet(task_partition.getPartitionCleanStartPath(), clean_start_status) || clean_start_status != "ok") + { + zookeeper->createIfNotExists(task_partition.getPartitionCleanStartPath(), ""); + auto checker = zkutil::EphemeralNodeHolder::create(task_partition.getPartitionCleanStartPath() + "/checker", + *zookeeper, host_id); + // Maybe we are the first worker + ASTPtr query_select_ast = get_select_query(task_shard.table_split_shard, "count()"); + UInt64 count; + { + Context local_context = context; + // Use pull (i.e. readonly) settings, but fetch data from destination servers + local_context.getSettingsRef() = task_cluster->settings_pull; + local_context.getSettingsRef().skip_unavailable_shards = true; + + Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_select_ast, local_context)->execute().in); + count = (block) ? block.safeGetByPosition(0).column->getUInt(0) : 0; + } + + if (count != 0) + { + Coordination::Stat stat_shards{}; + zookeeper->get(task_partition.getPartitionShardsPath(), &stat_shards); + + /// NOTE: partition is still fresh if dirt discovery happens before cleaning + if (stat_shards.numChildren == 0) + { + LOG_WARNING(log, "There are no workers for partition " << task_partition.name + << ", but destination table contains " << count << " rows" + << ". Partition will be dropped and refilled."); + + create_is_dirty_node(clean_state_clock); + return PartitionTaskStatus::Error; + } + } + zookeeper->set(task_partition.getPartitionCleanStartPath(), "ok"); + } + /// At this point, we need to sync that the destination table is clean + /// before any actual work + + /// Try start processing, create node about it + { + String start_state = TaskStateWithOwner::getData(TaskState::Started, host_id); + CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); + if (clean_state_clock != new_clean_state_clock) + { + LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); + return PartitionTaskStatus::Error; + } + else if (!new_clean_state_clock.is_clean()) + { + LOG_INFO(log, "Partition " << task_partition.name << " is dirty and will be dropped and refilled"); + create_is_dirty_node(new_clean_state_clock); + return PartitionTaskStatus::Error; + } + zookeeper->create(current_task_status_path, start_state, zkutil::CreateMode::Persistent); + } + + /// Try create table (if not exists) on each shard + { + auto create_query_push_ast = rewriteCreateQueryStorage(task_shard.current_pull_table_create_query, + task_table.table_push, task_table.engine_push_ast); + create_query_push_ast->as().if_not_exists = true; + String query = queryToString(create_query_push_ast); + + LOG_DEBUG(log, "Create destination tables. Query: " << query); + UInt64 shards = executeQueryOnCluster(task_table.cluster_push, query, + create_query_push_ast, &task_cluster->settings_push, + PoolMode::GET_MANY); + LOG_DEBUG(log, "Destination tables " << getQuotedTable(task_table.table_push) << + " have been created on " << shards << " shards of " << task_table.cluster_push->getShardCount()); + } + + /// Do the copying + { + bool inject_fault = false; + if (copy_fault_probability > 0) + { + double value = std::uniform_real_distribution<>(0, 1)(task_table.task_cluster.random_engine); + inject_fault = value < copy_fault_probability; + } + + // Select all fields + ASTPtr query_select_ast = get_select_query(task_shard.table_read_shard, "*", inject_fault ? "1" : ""); + + LOG_DEBUG(log, "Executing SELECT query and pull from " << task_shard.getDescription() + << " : " << queryToString(query_select_ast)); + + ASTPtr query_insert_ast; + { + String query; + query += "INSERT INTO " + getQuotedTable(task_shard.table_split_shard) + " VALUES "; + + ParserQuery p_query(query.data() + query.size()); + query_insert_ast = parseQuery(p_query, query, 0); + + LOG_DEBUG(log, "Executing INSERT query: " << query); + } + + try + { + /// Custom INSERT SELECT implementation + Context context_select = context; + context_select.getSettingsRef() = task_cluster->settings_pull; + + Context context_insert = context; + context_insert.getSettingsRef() = task_cluster->settings_push; + + BlockInputStreamPtr input; + BlockOutputStreamPtr output; + { + BlockIO io_select = InterpreterFactory::get(query_select_ast, context_select)->execute(); + BlockIO io_insert = InterpreterFactory::get(query_insert_ast, context_insert)->execute(); + + input = io_select.in; + output = io_insert.out; + } + + /// Fail-fast optimization to abort copying when the current clean state expires + std::future future_is_dirty_checker; + + Stopwatch watch(CLOCK_MONOTONIC_COARSE); + constexpr UInt64 check_period_milliseconds = 500; + + /// Will asynchronously check that ZooKeeper connection and is_dirty flag appearing while copying data + auto cancel_check = [&] () + { + if (zookeeper->expired()) + throw Exception("ZooKeeper session is expired, cancel INSERT SELECT", ErrorCodes::UNFINISHED); + + if (!future_is_dirty_checker.valid()) + future_is_dirty_checker = zookeeper->asyncExists(is_dirty_flag_path); + + /// check_period_milliseconds should less than average insert time of single block + /// Otherwise, the insertion will slow a little bit + if (watch.elapsedMilliseconds() >= check_period_milliseconds) + { + Coordination::ExistsResponse status = future_is_dirty_checker.get(); + + if (status.error != Coordination::ZNONODE) + { + LogicalClock dirt_discovery_epoch (status.stat.mzxid); + if (dirt_discovery_epoch == clean_state_clock.discovery_zxid) + return false; + throw Exception("Partition is dirty, cancel INSERT SELECT", ErrorCodes::UNFINISHED); + } + } + + return false; + }; + + /// Update statistics + /// It is quite rough: bytes_copied don't take into account DROP PARTITION. + auto update_stats = [&cluster_partition] (const Block & block) + { + cluster_partition.bytes_copied += block.bytes(); + cluster_partition.rows_copied += block.rows(); + cluster_partition.blocks_copied += 1; + }; + + /// Main work is here + copyData(*input, *output, cancel_check, update_stats); + + // Just in case + if (future_is_dirty_checker.valid()) + future_is_dirty_checker.get(); + + if (inject_fault) + throw Exception("Copy fault injection is activated", ErrorCodes::UNFINISHED); + } + catch (...) + { + tryLogCurrentException(log, "An error occurred during copying, partition will be marked as dirty"); + return PartitionTaskStatus::Error; + } + } + + /// Finalize the processing, change state of current partition task (and also check is_dirty flag) + { + String state_finished = TaskStateWithOwner::getData(TaskState::Finished, host_id); + CleanStateClock new_clean_state_clock (zookeeper, is_dirty_flag_path, is_dirty_cleaned_path); + if (clean_state_clock != new_clean_state_clock) + { + LOG_INFO(log, "Partition " << task_partition.name << " clean state changed, cowardly bailing"); + return PartitionTaskStatus::Error; + } + else if (!new_clean_state_clock.is_clean()) + { + LOG_INFO(log, "Partition " << task_partition.name << " became dirty and will be dropped and refilled"); + create_is_dirty_node(new_clean_state_clock); + return PartitionTaskStatus::Error; + } + zookeeper->set(current_task_status_path, state_finished, 0); + } + + LOG_INFO(log, "Partition " << task_partition.name << " copied"); + return PartitionTaskStatus::Finished; + } + + void dropAndCreateLocalTable(const ASTPtr & create_ast) + { + const auto & create = create_ast->as(); + dropLocalTableIfExists({create.database, create.table}); + + InterpreterCreateQuery interpreter(create_ast, context); + interpreter.execute(); + } + + void dropLocalTableIfExists(const DatabaseAndTableName & table_name) const + { + auto drop_ast = std::make_shared(); + drop_ast->if_exists = true; + drop_ast->database = table_name.first; + drop_ast->table = table_name.second; + + InterpreterDropQuery interpreter(drop_ast, context); + interpreter.execute(); + } + + String getRemoteCreateTable(const DatabaseAndTableName & table, Connection & connection, const Settings * settings = nullptr) + { + String query = "SHOW CREATE TABLE " + getQuotedTable(table); + Block block = getBlockWithAllStreamData(std::make_shared( + connection, query, InterpreterShowCreateQuery::getSampleBlock(), context, settings)); + + return typeid_cast(*block.safeGetByPosition(0).column).getDataAt(0).toString(); + } + + ASTPtr getCreateTableForPullShard(const ConnectionTimeouts & timeouts, TaskShard & task_shard) + { + /// Fetch and parse (possibly) new definition + auto connection_entry = task_shard.info.pool->get(timeouts, &task_cluster->settings_pull); + String create_query_pull_str = getRemoteCreateTable( + task_shard.task_table.table_pull, + *connection_entry, + &task_cluster->settings_pull); + + ParserCreateQuery parser_create_query; + return parseQuery(parser_create_query, create_query_pull_str, 0); + } + + void createShardInternalTables(const ConnectionTimeouts & timeouts, TaskShard & task_shard, bool create_split = true) + { + TaskTable & task_table = task_shard.task_table; + + /// We need to update table definitions for each part, it could be changed after ALTER + task_shard.current_pull_table_create_query = getCreateTableForPullShard(timeouts, task_shard); + + /// Create local Distributed tables: + /// a table fetching data from current shard and a table inserting data to the whole destination cluster + String read_shard_prefix = ".read_shard_" + toString(task_shard.indexInCluster()) + "."; + String split_shard_prefix = ".split."; + task_shard.table_read_shard = DatabaseAndTableName(working_database_name, read_shard_prefix + task_table.table_id); + task_shard.table_split_shard = DatabaseAndTableName(working_database_name, split_shard_prefix + task_table.table_id); + + /// Create special cluster with single shard + String shard_read_cluster_name = read_shard_prefix + task_table.cluster_pull_name; + ClusterPtr cluster_pull_current_shard = task_table.cluster_pull->getClusterWithSingleShard(task_shard.indexInCluster()); + context.setCluster(shard_read_cluster_name, cluster_pull_current_shard); + + auto storage_shard_ast = createASTStorageDistributed(shard_read_cluster_name, task_table.table_pull.first, task_table.table_pull.second); + const auto & storage_split_ast = task_table.engine_split_ast; + + auto create_query_ast = removeAliasColumnsFromCreateQuery(task_shard.current_pull_table_create_query); + auto create_table_pull_ast = rewriteCreateQueryStorage(create_query_ast, task_shard.table_read_shard, storage_shard_ast); + auto create_table_split_ast = rewriteCreateQueryStorage(create_query_ast, task_shard.table_split_shard, storage_split_ast); + + dropAndCreateLocalTable(create_table_pull_ast); + + if (create_split) + dropAndCreateLocalTable(create_table_split_ast); + } + + + std::set getShardPartitions(const ConnectionTimeouts & timeouts, TaskShard & task_shard) + { + createShardInternalTables(timeouts, task_shard, false); + + TaskTable & task_table = task_shard.task_table; + + String query; + { + WriteBufferFromOwnString wb; + wb << "SELECT DISTINCT " << queryToString(task_table.engine_push_partition_key_ast) << " AS partition FROM" + << " " << getQuotedTable(task_shard.table_read_shard) << " ORDER BY partition DESC"; + query = wb.str(); + } + + ParserQuery parser_query(query.data() + query.size()); + ASTPtr query_ast = parseQuery(parser_query, query, 0); + + LOG_DEBUG(log, "Computing destination partition set, executing query: " << query); + + Context local_context = context; + local_context.setSettings(task_cluster->settings_pull); + Block block = getBlockWithAllStreamData(InterpreterFactory::get(query_ast, local_context)->execute().in); + + std::set res; + if (block) + { + ColumnWithTypeAndName & column = block.getByPosition(0); + task_shard.partition_key_column = column; + + for (size_t i = 0; i < column.column->size(); ++i) + { + WriteBufferFromOwnString wb; + column.type->serializeAsTextQuoted(*column.column, i, wb, FormatSettings()); + res.emplace(wb.str()); + } + } + + LOG_DEBUG(log, "There are " << res.size() << " destination partitions in shard " << task_shard.getDescription()); + + return res; + } + + bool checkShardHasPartition(const ConnectionTimeouts & timeouts, TaskShard & task_shard, const String & partition_quoted_name) + { + createShardInternalTables(timeouts, task_shard, false); + + TaskTable & task_table = task_shard.task_table; + + std::string query = "SELECT 1 FROM " + getQuotedTable(task_shard.table_read_shard) + + " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + partition_quoted_name + " AS partition_key))"; + + if (!task_table.where_condition_str.empty()) + query += " AND (" + task_table.where_condition_str + ")"; + + query += " LIMIT 1"; + + LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " + << partition_quoted_name << " existence, executing query: " << query); + + ParserQuery parser_query(query.data() + query.size()); + ASTPtr query_ast = parseQuery(parser_query, query, 0); + + Context local_context = context; + local_context.setSettings(task_cluster->settings_pull); + return InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows() != 0; + } + + /// TODO: Implement checkPresentPartitionPiecesOnCurrentShard(); + /// Just copypaste the function above + bool checkPresentPartitionPiecesOnCurrentShard(const ConnectionTimeouts & timeouts, + TaskShard & task_shard, const String & partition_quoted_name, size_t current_piece_number) + { + createShardInternalTables(timeouts, task_shard, false); + + TaskTable & task_table = task_shard.task_table; + + std::string query = "SELECT 1 FROM " + getQuotedTable(task_shard.table_read_shard) + + " WHERE (" + queryToString(task_table.engine_push_partition_key_ast) + " = (" + partition_quoted_name + " AS partition_key))"; + + query += " AND (cityHash64(*) = " + std::to_string(current_piece_number) + " )"; + + if (!task_table.where_condition_str.empty()) + query += " AND (" + task_table.where_condition_str + ")"; + + query += " LIMIT 1"; + + LOG_DEBUG(log, "Checking shard " << task_shard.getDescription() << " for partition " + << partition_quoted_name << " piece " << std::to_string(current_piece_number) << "existence, executing query: " << query); + + ParserQuery parser_query(query.data() + query.size()); + ASTPtr query_ast = parseQuery(parser_query, query, 0); + + Context local_context = context; + local_context.setSettings(task_cluster->settings_pull); + return InterpreterFactory::get(query_ast, local_context)->execute().in->read().rows() != 0; + } + + /** Executes simple query (without output streams, for example DDL queries) on each shard of the cluster + * Returns number of shards for which at least one replica executed query successfully + */ + UInt64 executeQueryOnCluster( + const ClusterPtr & cluster, + const String & query, + const ASTPtr & query_ast_ = nullptr, + const Settings * settings = nullptr, + PoolMode pool_mode = PoolMode::GET_ALL, + UInt64 max_successful_executions_per_shard = 0) const + { + auto num_shards = cluster->getShardsInfo().size(); + std::vector per_shard_num_successful_replicas(num_shards, 0); + + ASTPtr query_ast; + if (query_ast_ == nullptr) + { + ParserQuery p_query(query.data() + query.size()); + query_ast = parseQuery(p_query, query, 0); + } + else + query_ast = query_ast_; + + + /// We need to execute query on one replica at least + auto do_for_shard = [&] (UInt64 shard_index) + { + const Cluster::ShardInfo & shard = cluster->getShardsInfo().at(shard_index); + UInt64 & num_successful_executions = per_shard_num_successful_replicas.at(shard_index); + num_successful_executions = 0; + + auto increment_and_check_exit = [&] () -> bool + { + ++num_successful_executions; + return max_successful_executions_per_shard && num_successful_executions >= max_successful_executions_per_shard; + }; + + UInt64 num_replicas = cluster->getShardsAddresses().at(shard_index).size(); + UInt64 num_local_replicas = shard.getLocalNodeCount(); + UInt64 num_remote_replicas = num_replicas - num_local_replicas; + + /// In that case we don't have local replicas, but do it just in case + for (UInt64 i = 0; i < num_local_replicas; ++i) + { + auto interpreter = InterpreterFactory::get(query_ast, context); + interpreter->execute(); + + if (increment_and_check_exit()) + return; + } + + /// Will try to make as many as possible queries + if (shard.hasRemoteConnections()) + { + Settings current_settings = settings ? *settings : task_cluster->settings_common; + current_settings.max_parallel_replicas = num_remote_replicas ? num_remote_replicas : 1; + + auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings).getSaturated(current_settings.max_execution_time); + auto connections = shard.pool->getMany(timeouts, ¤t_settings, pool_mode); + + for (auto & connection : connections) + { + if (connection.isNull()) + continue; + + try + { + /// CREATE TABLE and DROP PARTITION queries return empty block + RemoteBlockInputStream stream{*connection, query, Block{}, context, ¤t_settings}; + NullBlockOutputStream output{Block{}}; + copyData(stream, output); + + if (increment_and_check_exit()) + return; + } + catch (const Exception &) + { + LOG_INFO(log, getCurrentExceptionMessage(false, true)); + } + } + } + }; + + { + ThreadPool thread_pool(std::min(num_shards, getNumberOfPhysicalCPUCores())); + + for (UInt64 shard_index = 0; shard_index < num_shards; ++shard_index) + thread_pool.scheduleOrThrowOnError([=] { do_for_shard(shard_index); }); + + thread_pool.wait(); + } + + UInt64 successful_shards = 0; + for (UInt64 num_replicas : per_shard_num_successful_replicas) + successful_shards += (num_replicas > 0); + + return successful_shards; + } private: + String task_zookeeper_path; + String task_description_path; + String host_id; + String working_database_name; - using Base = BaseDaemon; + /// Auto update config stuff + UInt64 task_descprtion_current_version = 1; + std::atomic task_descprtion_version{1}; + Coordination::WatchCallback task_description_watch_callback; + /// ZooKeeper session used to set the callback + zkutil::ZooKeeperPtr task_description_watch_zookeeper; - void mainImpl(); + ConfigurationPtr task_cluster_initial_config; + ConfigurationPtr task_cluster_current_config; + Coordination::Stat task_description_current_stat{}; + + std::unique_ptr task_cluster; - std::string config_xml_path; - std::string task_path; - std::string log_level = "trace"; bool is_safe_mode = false; - double copy_fault_probability = 0; - bool is_help = false; + double copy_fault_probability = 0.0; - std::string base_dir; - std::string process_path; - std::string process_id; - std::string host_id; + Context & context; + Poco::Logger * log; + + std::chrono::milliseconds default_sleep_time{1000}; }; - } diff --git a/dbms/programs/copier/ClusterCopierApp.cpp b/dbms/programs/copier/ClusterCopierApp.cpp new file mode 100644 index 00000000000..37b02296086 --- /dev/null +++ b/dbms/programs/copier/ClusterCopierApp.cpp @@ -0,0 +1,173 @@ +#include "ClusterCopierApp.h" +#include "ClusterCopier.h" + +namespace DB +{ + +/// ClusterCopierApp + +void ClusterCopierApp::initialize(Poco::Util::Application & self) +{ + is_help = config().has("help"); + if (is_help) + return; + + config_xml_path = config().getString("config-file"); + task_path = config().getString("task-path"); + log_level = config().getString("log-level", "trace"); + is_safe_mode = config().has("safe-mode"); + if (config().has("copy-fault-probability")) + copy_fault_probability = std::max(std::min(config().getDouble("copy-fault-probability"), 1.0), 0.0); + base_dir = (config().has("base-dir")) ? config().getString("base-dir") : Poco::Path::current(); + // process_id is '#_' + time_t timestamp = Poco::Timestamp().epochTime(); + auto curr_pid = Poco::Process::id(); + + process_id = std::to_string(DateLUT::instance().toNumYYYYMMDDhhmmss(timestamp)) + "_" + std::to_string(curr_pid); + host_id = escapeForFileName(getFQDNOrHostName()) + '#' + process_id; + process_path = Poco::Path(base_dir + "/clickhouse-copier_" + process_id).absolute().toString(); + Poco::File(process_path).createDirectories(); + + /// Override variables for BaseDaemon + if (config().has("log-level")) + config().setString("logger.level", config().getString("log-level")); + + if (config().has("base-dir") || !config().has("logger.log")) + config().setString("logger.log", process_path + "/log.log"); + + if (config().has("base-dir") || !config().has("logger.errorlog")) + config().setString("logger.errorlog", process_path + "/log.err.log"); + + Base::initialize(self); +} + + +void ClusterCopierApp::handleHelp(const std::string &, const std::string &) +{ + Poco::Util::HelpFormatter helpFormatter(options()); + helpFormatter.setCommand(commandName()); + helpFormatter.setHeader("Copies tables from one cluster to another"); + helpFormatter.setUsage("--config-file --task-path "); + helpFormatter.format(std::cerr); + + stopOptionsProcessing(); +} + + +void ClusterCopierApp::defineOptions(Poco::Util::OptionSet & options) +{ + Base::defineOptions(options); + + options.addOption(Poco::Util::Option("task-path", "", "path to task in ZooKeeper") + .argument("task-path").binding("task-path")); + options.addOption(Poco::Util::Option("task-file", "", "path to task file for uploading in ZooKeeper to task-path") + .argument("task-file").binding("task-file")); + options.addOption(Poco::Util::Option("task-upload-force", "", "Force upload task-file even node already exists") + .argument("task-upload-force").binding("task-upload-force")); + options.addOption(Poco::Util::Option("safe-mode", "", "disables ALTER DROP PARTITION in case of errors") + .binding("safe-mode")); + options.addOption(Poco::Util::Option("copy-fault-probability", "", "the copying fails with specified probability (used to test partition state recovering)") + .argument("copy-fault-probability").binding("copy-fault-probability")); + options.addOption(Poco::Util::Option("log-level", "", "sets log level") + .argument("log-level").binding("log-level")); + options.addOption(Poco::Util::Option("base-dir", "", "base directory for copiers, consecutive copier launches will populate /base-dir/launch_id/* directories") + .argument("base-dir").binding("base-dir")); + + using Me = std::decay_t; + options.addOption(Poco::Util::Option("help", "", "produce this help message").binding("help") + .callback(Poco::Util::OptionCallback(this, &Me::handleHelp))); +} + + +void ClusterCopierApp::mainImpl() +{ + StatusFile status_file(process_path + "/status"); + ThreadStatus thread_status; + + auto log = &logger(); + LOG_INFO(log, "Starting clickhouse-copier (" + << "id " << process_id << ", " + << "host_id " << host_id << ", " + << "path " << process_path << ", " + << "revision " << ClickHouseRevision::get() << ")"); + + auto context = std::make_unique(Context::createGlobal()); + context->makeGlobalContext(); + SCOPE_EXIT(context->shutdown()); + + context->setConfig(loaded_config.configuration); + context->setApplicationType(Context::ApplicationType::LOCAL); + context->setPath(process_path); + + registerFunctions(); + registerAggregateFunctions(); + registerTableFunctions(); + registerStorages(); + registerDictionaries(); + registerDisks(); + + static const std::string default_database = "_local"; + context->addDatabase(default_database, std::make_shared(default_database)); + context->setCurrentDatabase(default_database); + + /// Initialize query scope just in case. + CurrentThread::QueryScope query_scope(*context); + + auto copier = std::make_unique(task_path, host_id, default_database, *context); + copier->setSafeMode(is_safe_mode); + copier->setCopyFaultProbability(copy_fault_probability); + + auto task_file = config().getString("task-file", ""); + if (!task_file.empty()) + copier->uploadTaskDescription(task_path, task_file, config().getBool("task-upload-force", false)); + + copier->init(); + copier->process(ConnectionTimeouts::getTCPTimeoutsWithoutFailover(context->getSettingsRef())); + + /// Reset ZooKeeper before removing ClusterCopier. + /// Otherwise zookeeper watch can call callback which use already removed ClusterCopier object. + context->resetZooKeeper(); +} + + +int ClusterCopierApp::main(const std::vector &) +{ + if (is_help) + return 0; + + try + { + mainImpl(); + } + catch (...) + { + tryLogCurrentException(&Poco::Logger::root(), __PRETTY_FUNCTION__); + auto code = getCurrentExceptionCode(); + + return (code) ? code : -1; + } + + return 0; +} + + +} + +#pragma GCC diagnostic ignored "-Wunused-function" +#pragma GCC diagnostic ignored "-Wmissing-declarations" + +int mainEntryClickHouseClusterCopier(int argc, char ** argv) +{ + try + { + DB::ClusterCopierApp app; + return app.run(argc, argv); + } + catch (...) + { + std::cerr << DB::getCurrentExceptionMessage(true) << "\n"; + auto code = DB::getCurrentExceptionCode(); + + return (code) ? code : -1; + } +} diff --git a/dbms/programs/copier/ClusterCopierApp.h b/dbms/programs/copier/ClusterCopierApp.h new file mode 100644 index 00000000000..fe228fd6194 --- /dev/null +++ b/dbms/programs/copier/ClusterCopierApp.h @@ -0,0 +1,87 @@ +#pragma once +#include +#include + +/* clickhouse cluster copier util + * Copies tables data from one cluster to new tables of other (possibly the same) cluster in distributed fault-tolerant manner. + * + * See overview in the docs: docs/en/utils/clickhouse-copier.md + * + * Implementation details: + * + * cluster-copier workers pull each partition of each shard of the source cluster and push it to the destination cluster through + * Distributed table (to preform data resharding). So, worker job is a partition of a source shard. + * A job has three states: Active, Finished and Abandoned. Abandoned means that worker died and did not finish the job. + * + * If an error occurred during the copying (a worker failed or a worker did not finish the INSERT), then the whole partition (on + * all destination servers) should be dropped and refilled. So, copying entity is a partition of all destination shards. + * If a failure is detected a special /is_dirty node is created in ZooKeeper signalling that other workers copying the same partition + * should stop, after a refilling procedure should start. + * + * ZooKeeper task node has the following structure: + * /task/path_root - path passed in --task-path parameter + * /description - contains user-defined XML config of the task + * /task_active_workers - contains ephemeral nodes of all currently active workers, used to implement max_workers limitation + * /server_fqdn#PID_timestamp - cluster-copier worker ID + * ... + * /tables - directory with table tasks + * /cluster.db.table1 - directory of table_hits task + * /partition1 - directory for partition1 + * /shards - directory for source cluster shards + * /1 - worker job for the first shard of partition1 of table test.hits + * Contains info about current status (Active or Finished) and worker ID. + * /2 + * ... + * /partition_active_workers + * /1 - for each job in /shards a corresponding ephemeral node created in /partition_active_workers + * It is used to detect Abandoned jobs (if there is Active node in /shards and there is no node in + * /partition_active_workers). + * Also, it is used to track active workers in the partition (when we need to refill the partition we do + * not DROP PARTITION while there are active workers) + * /2 + * ... + * /is_dirty - the node is set if some worker detected that an error occurred (the INSERT is failed or an Abandoned node is + * detected). If the node appeared workers in this partition should stop and start cleaning and refilling + * partition procedure. + * During this procedure a single 'cleaner' worker is selected. The worker waits for stopping all partition + * workers, removes /shards node, executes DROP PARTITION on each destination node and removes /is_dirty node. + * /cleaner- An ephemeral node used to select 'cleaner' worker. Contains ID of the worker. + * /cluster.db.table2 + * ... + */ + +namespace DB +{ + +class ClusterCopierApp : public BaseDaemon +{ +public: + + void initialize(Poco::Util::Application & self) override; + + void handleHelp(const std::string &, const std::string &); + + void defineOptions(Poco::Util::OptionSet & options) override; + + int main(const std::vector &) override; + +private: + + using Base = BaseDaemon; + + void mainImpl(); + + std::string config_xml_path; + std::string task_path; + std::string log_level = "trace"; + bool is_safe_mode = false; + double copy_fault_probability = 0; + bool is_help = false; + + std::string base_dir; + std::string process_path; + std::string process_id; + std::string host_id; +}; + +} diff --git a/dbms/programs/copier/Internals.h b/dbms/programs/copier/Internals.h index 5f14604fbf9..6e81f16035c 100644 --- a/dbms/programs/copier/Internals.h +++ b/dbms/programs/copier/Internals.h @@ -148,12 +148,14 @@ struct TaskStateWithOwner /// Hierarchical description of the tasks +struct ShardPartitionPiece; struct ShardPartition; struct TaskShard; struct TaskTable; struct TaskCluster; struct ClusterPartition; +using PartitionPieces = std::vector; using TasksPartition = std::map>; using ShardInfo = Cluster::ShardInfo; using TaskShardPtr = std::shared_ptr; @@ -162,22 +164,59 @@ using TasksTable = std::list; using ClusterPartitions = std::map>; +struct ShardPartitionPiece +{ + ShardPartitionPiece(ShardPartition & parent, size_t current_piece_number_, bool is_absent_piece_) + : is_absent_piece(is_absent_piece_) + , current_piece_number(current_piece_number_) + , shard_partition(parent) {} + + [[maybe_unused]] String getPartitionPiecePath() const {return "Not implemented.";} + [[maybe_unused]] String getPartitionPieceCleanStartPath() const {return "Not implemented.";} + [[maybe_unused]] String getCommonPartitionPieceIsDirtyPath() const {return "Not implemented.";} + [[maybe_unused]] String getCommonPartitionPieceIsCleanedPath() const {return "Not implemented.";} + + [[maybe_unused]] String getPartitionPieceActiveWorkersPath() const {return "Not implemented.";} + [[maybe_unused]] String getActiveWorkerPath() const {return "Not implemented.";} + + /// On what shards do we have current partition. + [[maybe_unused]] String getPartitionPieceShardsPath() const {return "Not implemented.";} + [[maybe_unused]] String getShardStatusPath() const {return "Not implemented.";} + + bool is_absent_piece; + const size_t current_piece_number; + + ShardPartition & shard_partition; +}; + + /// Just destination partition of a shard /// I don't know what this comment means. /// In short, when we discovered what shards contain currently processing partition, /// This class describes a partition (name) that is stored on the shard (parent). struct ShardPartition { - ShardPartition(TaskShard & parent, String name_quoted_) : task_shard(parent), name(std::move(name_quoted_)) {} + ShardPartition(TaskShard & parent, String name_quoted_, size_t number_of_splits = 10) + : task_shard(parent) + , name(std::move(name_quoted_)) + { pieces.reserve(number_of_splits); } - String getPartitionPath() const; - String getPartitionCleanStartPath() const; - String getCommonPartitionIsDirtyPath() const; - String getCommonPartitionIsCleanedPath() const; - String getPartitionActiveWorkersPath() const; - String getActiveWorkerPath() const; - String getPartitionShardsPath() const; - String getShardStatusPath() const; + /*useful*/ String getPartitionPath() const; + [[maybe_unused]] String getPartitionPiecePath(size_t current_piece_number) const; + /*useful*/ String getPartitionCleanStartPath() const; + [[maybe_unused]] String getPartitionPieceCleanStartPath(size_t current_piece_number) const; + /*useful*/ String getCommonPartitionIsDirtyPath() const; + /*useful*/ String getCommonPartitionIsCleanedPath() const; + /*??????*/ String getPartitionActiveWorkersPath() const; + /*??????*/ String getActiveWorkerPath() const; + /*useful*/ String getPartitionShardsPath() const; + /*useful*/ String getShardStatusPath() const; + + /// What partition pieces are present in current shard. + /// FYI: Piece is a part of partition which has modulo equals to concrete constant (less than number_of_splits obliously) + /// For example SELECT ... from ... WHERE partition=current_partition AND cityHash64(*) == const; + /// Absent pieces have field is_absent_piece equals to true. + PartitionPieces pieces; TaskShard & task_shard; String name; @@ -255,7 +294,7 @@ struct TaskTable TaskCluster & task_cluster; String getPartitionPath(const String & partition_name) const; - [[maybe_unused]] String getPartitionPathWithPieceNumber(const String & partition_name, size_t current_piece_number) const; + [[maybe_unused]] String getPartitionPiecePath(const String & partition_name, size_t current_piece_number) const; String getPartitionIsDirtyPath(const String & partition_name) const; String getPartitionIsCleanedPath(const String & partition_name) const; String getPartitionTaskStatusPath(const String & partition_name) const; @@ -422,8 +461,9 @@ String TaskTable::getPartitionPath(const String & partition_name) const + "/" + escapeForFileName(partition_name); // 201701 } -String TaskTable::getPartitionPathWithPieceNumber(const String & partition_name, size_t current_piece_number) const +String TaskTable::getPartitionPiecePath(const String & partition_name, size_t current_piece_number) const { + assert(current_piece_number < number_of_splits); return getPartitionPath(partition_name) + "/" + std::to_string(current_piece_number); // 1...number_of_splits } @@ -432,11 +472,23 @@ String ShardPartition::getPartitionCleanStartPath() const return getPartitionPath() + "/clean_start"; } +String ShardPartition::getPartitionPieceCleanStartPath(size_t current_piece_number) const +{ + assert(current_piece_number < task_shard.task_table.number_of_splits); + return getPartitionPiecePath(current_piece_number) + "/clean_start"; +} + String ShardPartition::getPartitionPath() const { return task_shard.task_table.getPartitionPath(name); } +String ShardPartition::getPartitionPiecePath(size_t current_piece_number) const +{ + assert(current_piece_number < task_shard.task_table.number_of_splits); + return task_shard.task_table.getPartitionPiecePath(name, current_piece_number); +} + String ShardPartition::getShardStatusPath() const { // schema: //tables/
//shards/ diff --git a/dbms/programs/copier/ZookeeperStaff.h b/dbms/programs/copier/ZookeeperStaff.h new file mode 100644 index 00000000000..3133c68933d --- /dev/null +++ b/dbms/programs/copier/ZookeeperStaff.h @@ -0,0 +1,224 @@ +#pragma once + +/** Allows to compare two incremental counters of type UInt32 in presence of possible overflow. + * We assume that we compare values that are not too far away. + * For example, when we increment 0xFFFFFFFF, we get 0. So, 0xFFFFFFFF is less than 0. + */ +class WrappingUInt32 +{ +public: + UInt32 value; + + explicit WrappingUInt32(UInt32 _value) + : value(_value) + {} + + bool operator<(const WrappingUInt32 & other) const + { + return value != other.value && *this <= other; + } + + bool operator<=(const WrappingUInt32 & other) const + { + const UInt32 HALF = 1 << 31; + return (value <= other.value && other.value - value < HALF) + || (value > other.value && value - other.value > HALF); + } + + bool operator==(const WrappingUInt32 & other) const + { + return value == other.value; + } +}; + +/** Conforming Zxid definition. + * cf. https://github.com/apache/zookeeper/blob/631d1b284f0edb1c4f6b0fb221bf2428aec71aaa/zookeeper-docs/src/main/resources/markdown/zookeeperInternals.md#guarantees-properties-and-definitions + * + * But it is better to read this: https://zookeeper.apache.org/doc/r3.1.2/zookeeperProgrammers.html + * + * Actually here is the definition of Zxid. + * Every change to the ZooKeeper state receives a stamp in the form of a zxid (ZooKeeper Transaction Id). + * This exposes the total ordering of all changes to ZooKeeper. Each change will have a unique zxid + * and if zxid1 is smaller than zxid2 then zxid1 happened before zxid2. + */ +class Zxid +{ +public: + WrappingUInt32 epoch; + WrappingUInt32 counter; + explicit Zxid(UInt64 _zxid) + : epoch(_zxid >> 32) + , counter(_zxid) + {} + + bool operator<=(const Zxid & other) const + { + return (epoch < other.epoch) + || (epoch == other.epoch && counter <= other.counter); + } + + bool operator==(const Zxid & other) const + { + return epoch == other.epoch && counter == other.counter; + } +}; + +/* When multiple ClusterCopiers discover that the target partition is not empty, + * they will attempt to clean up this partition before proceeding to copying. + * + * Instead of purging is_dirty, the history of cleaning work is preserved and partition hygiene is established + * based on a happens-before relation between the events. + * This relation is encoded by LogicalClock based on the mzxid of the is_dirty ZNode and is_dirty/cleaned. + * The fact of the partition hygiene is encoded by CleanStateClock. + * + * For you to know what mzxid means: + * + * ZooKeeper Stat Structure: + * The Stat structure for each znode in ZooKeeper is made up of the following fields: + * + * -- czxid + * The zxid of the change that caused this znode to be created. + * + * -- mzxid + * The zxid of the change that last modified this znode. + * + * -- ctime + * The time in milliseconds from epoch when this znode was created. + * + * -- mtime + * The time in milliseconds from epoch when this znode was last modified. + * + * -- version + * The number of changes to the data of this znode. + * + * -- cversion + * The number of changes to the children of this znode. + * + * -- aversion + * The number of changes to the ACL of this znode. + * + * -- ephemeralOwner + * The session id of the owner of this znode if the znode is an ephemeral node. + * If it is not an ephemeral node, it will be zero. + * + * -- dataLength + * The length of the data field of this znode. + * + * -- numChildren + * The number of children of this znode. + * */ + +class LogicalClock +{ +public: + std::optional zxid; + + LogicalClock() = default; + + explicit LogicalClock(UInt64 _zxid) + : zxid(_zxid) + {} + + bool hasHappened() const + { + return bool(zxid); + } + + /// happens-before relation with a reasonable time bound + bool happensBefore(const LogicalClock & other) const + { + return !zxid + || (other.zxid && *zxid <= *other.zxid); + } + + bool operator<=(const LogicalClock & other) const + { + return happensBefore(other); + } + + /// strict equality check + bool operator==(const LogicalClock & other) const + { + return zxid == other.zxid; + } +}; + + +class CleanStateClock +{ +public: + LogicalClock discovery_zxid; + std::optional discovery_version; + + LogicalClock clean_state_zxid; + std::optional clean_state_version; + + std::shared_ptr stale; + + bool is_clean() const + { + return + !is_stale() + && ( + !discovery_zxid.hasHappened() + || (clean_state_zxid.hasHappened() && discovery_zxid <= clean_state_zxid)); + } + + bool is_stale() const + { + return stale->load(); + } + + CleanStateClock( + const zkutil::ZooKeeperPtr & zookeeper, + const String & discovery_path, + const String & clean_state_path) + : stale(std::make_shared(false)) + { + Coordination::Stat stat{}; + String _some_data; + auto watch_callback = + [stale = stale] (const Coordination::WatchResponse & rsp) + { + auto logger = &Poco::Logger::get("ClusterCopier"); + if (rsp.error == Coordination::ZOK) + { + switch (rsp.type) + { + case Coordination::CREATED: + LOG_DEBUG(logger, "CleanStateClock change: CREATED, at " << rsp.path); + stale->store(true); + break; + case Coordination::CHANGED: + LOG_DEBUG(logger, "CleanStateClock change: CHANGED, at" << rsp.path); + stale->store(true); + } + } + }; + if (zookeeper->tryGetWatch(discovery_path, _some_data, &stat, watch_callback)) + { + discovery_zxid = LogicalClock(stat.mzxid); + discovery_version = stat.version; + } + if (zookeeper->tryGetWatch(clean_state_path, _some_data, &stat, watch_callback)) + { + clean_state_zxid = LogicalClock(stat.mzxid); + clean_state_version = stat.version; + } + } + + bool operator==(const CleanStateClock & other) const + { + return !is_stale() + && !other.is_stale() + && discovery_zxid == other.discovery_zxid + && discovery_version == other.discovery_version + && clean_state_zxid == other.clean_state_zxid + && clean_state_version == other.clean_state_version; + } + + bool operator!=(const CleanStateClock & other) const + { + return !(*this == other); + } +}; From 3c5d6cf6f1e1af3c9c19ffa1b57bbad8efa9b6e1 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 11 Feb 2020 20:25:26 +0300 Subject: [PATCH 0246/2007] fix --- dbms/src/Interpreters/Context.cpp | 2 +- dbms/src/Interpreters/DatabaseCatalog.cpp | 19 +++++++++++-------- dbms/src/Interpreters/DatabaseCatalog.h | 2 +- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 80108fcbc15..7be0ecbccbc 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -770,7 +770,7 @@ bool Context::isDictionaryExists(const String & database_name, const String & di { auto lock = getLock(); String db = resolveDatabase(database_name); - auto db_ptr = DatabaseCatalog::instance().tryGetDatabase(database_name); + auto db_ptr = DatabaseCatalog::instance().tryGetDatabase(db); return db_ptr && db_ptr->isDictionaryExist(*this, dictionary_name); } diff --git a/dbms/src/Interpreters/DatabaseCatalog.cpp b/dbms/src/Interpreters/DatabaseCatalog.cpp index 98f7145c392..c83e8dfcf54 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.cpp +++ b/dbms/src/Interpreters/DatabaseCatalog.cpp @@ -83,17 +83,20 @@ StoragePtr DatabaseCatalog::getTable(const StorageID & table_id, const Context & // return db_and_table.second; //} - std::lock_guard _lock{databases_mutex}; - - auto it = databases.find(table_id.getDatabaseName()); - if (databases.end() == it) + DatabasePtr database; { - if (exception) - exception->emplace("Database " + backQuoteIfNeed(table_id.getDatabaseName()) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - return {}; + std::lock_guard _lock{databases_mutex}; + auto it = databases.find(table_id.getDatabaseName()); + if (databases.end() == it) + { + if (exception) + exception->emplace("Database " + backQuoteIfNeed(table_id.getDatabaseName()) + " doesn't exist", + ErrorCodes::UNKNOWN_DATABASE); + return {}; + } + database = it->second; } - auto database = it->second; auto table = database->tryGetTable(local_context, table_id.table_name); if (!table && exception) exception->emplace("Table " + table_id.getNameForLogs() + " doesn't exist.", ErrorCodes::UNKNOWN_TABLE); diff --git a/dbms/src/Interpreters/DatabaseCatalog.h b/dbms/src/Interpreters/DatabaseCatalog.h index 1a25800737a..871cd5a0ddd 100644 --- a/dbms/src/Interpreters/DatabaseCatalog.h +++ b/dbms/src/Interpreters/DatabaseCatalog.h @@ -113,7 +113,7 @@ private: private: //[[maybe_unused]] Context & global_context; - mutable std::mutex databases_mutex; + mutable std::recursive_mutex databases_mutex; //const String default_database; Databases databases; UUIDToStorageMap uuid_map; From 5dcca60d98bb5d3e75c6b26348e36f5a0210d37a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 11 Feb 2020 22:18:40 +0300 Subject: [PATCH 0247/2007] Update general_purpose_hashes_on_UUID.xml --- dbms/tests/performance/general_purpose_hashes_on_UUID.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/performance/general_purpose_hashes_on_UUID.xml b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml index 9dbac4e33cc..71a9002ee9c 100644 --- a/dbms/tests/performance/general_purpose_hashes_on_UUID.xml +++ b/dbms/tests/performance/general_purpose_hashes_on_UUID.xml @@ -41,8 +41,8 @@ table - numbers(10000000) - numbers_mt(100000000) + numbers(20000000) + numbers_mt(200000000) From f88bf5595c0a66a3b7b06e94213c15ff76a2c2c3 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 11 Feb 2020 22:21:08 +0300 Subject: [PATCH 0248/2007] Update cpu_synthetic.xml --- dbms/tests/performance/cpu_synthetic.xml | 47 ++++++++++++------------ 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/dbms/tests/performance/cpu_synthetic.xml b/dbms/tests/performance/cpu_synthetic.xml index 50c9d5b6e6d..465c1eb4011 100644 --- a/dbms/tests/performance/cpu_synthetic.xml +++ b/dbms/tests/performance/cpu_synthetic.xml @@ -15,6 +15,7 @@ hits_100m_single + hits_10m_single -SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(cityHash64(SearchPhrase)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(SearchPhrase)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(farmHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(farmHash64(SearchPhrase)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(farmHash64(SearchPhrase)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(metroHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(metroHash64(SearchPhrase)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(metroHash64(SearchPhrase)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(sipHash64(SearchPhrase)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(SearchPhrase)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(SearchPhrase)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(MD5(SearchPhrase)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(SearchPhrase)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(MD5(URL)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(MD5(URL)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(cityHash64(URL)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(URL)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(sipHash64(URL)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(sipHash64(URL)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(PageCharset)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(cityHash64(PageCharset)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(cityHash64(PageCharset)) -SELECT count() FROM hits_100m_single WHERE URL LIKE '%metrika%' SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE URL LIKE '%metrika%' SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE URL LIKE '%metrika%' -SELECT count() FROM hits_100m_single WHERE positionCaseInsensitiveUTF8(URL, 'новости') != 0 SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE positionCaseInsensitiveUTF8(URL, 'новости') != 0 SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE positionCaseInsensitiveUTF8(URL, 'новости') != 0 -SELECT count() FROM hits_100m_single WHERE match(URL, '^https?://(?:www\\.)?metri[kc]a\\.yandex\\.(?:ru|com|com\\.tr|ua|by|kz)/.+?2014') SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE match(URL, '^https?://(?:www\\.)?metri[kc]a\\.yandex\\.(?:ru|com|com\\.tr|ua|by|kz)/.+?2014') SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE match(URL, '^https?://(?:www\\.)?metri[kc]a\\.yandex\\.(?:ru|com|com\\.tr|ua|by|kz)/.+?2014') SELECT SearchEngineID, SearchPhrase, RegionID FROM hits_100m_single GROUP BY SearchEngineID, SearchPhrase, RegionID ORDER BY count() DESC LIMIT 10 SETTINGS max_threads = 1 SELECT SearchEngineID, SearchPhrase, RegionID FROM hits_100m_single GROUP BY SearchEngineID, SearchPhrase, RegionID ORDER BY count() DESC LIMIT 10 -SELECT count() FROM hits_100m_single WHERE NOT ignore(toMonday(EventTime)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(toMonday(EventTime)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(toMonday(EventTime)) -SELECT count() FROM hits_100m_single WHERE NOT ignore(cutQueryString(URL)) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(cutQueryString(URL)) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(cutQueryString(URL)) -SELECT quantilesIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_10m_single SETTINGS max_threads = 1 SELECT quantilesIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT quantilesTimingIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesTimingIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_10m_single SETTINGS max_threads = 1 SELECT quantilesTimingIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT quantilesExactIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesExactIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_10m_single SETTINGS max_threads = 1 SELECT quantilesExactIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT quantilesTDigestIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT quantilesTDigestIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_10m_single SETTINGS max_threads = 1 SELECT quantilesTDigestIf(0.5, 0.9)(SendTiming, SendTiming > 0) FROM hits_100m_single -SELECT uniq(UserID) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT uniq(UserID) FROM hits_10m_single SETTINGS max_threads = 1 SELECT uniq(UserID) FROM hits_100m_single -SELECT uniqCombined(17)(UserID) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT uniqCombined(17)(UserID) FROM hits_10m_single SETTINGS max_threads = 1 SELECT uniqCombined(17)(UserID) FROM hits_100m_single -SELECT uniqExact(UserID) FROM hits_100m_single SETTINGS max_threads = 1 +SELECT uniqExact(UserID) FROM hits_10m_single SETTINGS max_threads = 1 SELECT uniqExact(UserID) FROM hits_100m_single -SELECT RegionID, uniq(UserID) FROM hits_100m_single GROUP BY RegionID SETTINGS max_threads = 1 +SELECT RegionID, uniq(UserID) FROM hits_10m_single GROUP BY RegionID SETTINGS max_threads = 1 SELECT RegionID, uniq(UserID) FROM hits_100m_single GROUP BY RegionID -SELECT count() FROM hits_100m_single WHERE NOT ignore(*) SETTINGS max_threads = 1 +SELECT count() FROM hits_10m_single WHERE NOT ignore(*) SETTINGS max_threads = 1 SELECT count() FROM hits_100m_single WHERE NOT ignore(*) From 647994fea2b28249139b3ed96574026e6563f3cf Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 11 Feb 2020 23:32:45 +0300 Subject: [PATCH 0249/2007] pcg64_oneseq -> pcg32 --- .../TableFunctions/TableFunctionRandom.cpp | 31 ++-- .../01072_random_table_function.reference | 166 +++++++++--------- 2 files changed, 102 insertions(+), 95 deletions(-) diff --git a/dbms/src/TableFunctions/TableFunctionRandom.cpp b/dbms/src/TableFunctions/TableFunctionRandom.cpp index f23b98bfbaa..1a6c6bc5e67 100644 --- a/dbms/src/TableFunctions/TableFunctionRandom.cpp +++ b/dbms/src/TableFunctions/TableFunctionRandom.cpp @@ -92,10 +92,11 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64_oneseq generator(random_seed); + pcg32 generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { - data[i] = static_cast(generator()); + UInt64 a = static_cast(generator()) << 32 | static_cast(generator()); + data[i] = static_cast(a); } break; } @@ -138,10 +139,11 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64_oneseq generator(random_seed); + pcg32 generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { - data[i] = static_cast(generator()); + Int64 a = static_cast(generator()) << 32 | static_cast(generator()); + data[i] = static_cast(a); } break; } @@ -164,12 +166,12 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64_oneseq generator(random_seed); + pcg32 generator(random_seed); double d = 1.0; for (UInt64 i = 0; i < limit; ++i) { d = std::numeric_limits::max(); - data[i] = (d / pcg64::max()) * generator(); + data[i] = (d / pcg32::max()) * generator(); } break; } @@ -306,10 +308,11 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64_oneseq generator(random_seed); + pcg32 generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { - data[i] = static_cast(generator()); + UInt64 a = static_cast(generator()) << 32 | static_cast(generator()); + data[i] = a; } break; } @@ -317,10 +320,11 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64_oneseq generator(random_seed); + pcg32 generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { - Int128 x = static_cast(generator()) << 64 | static_cast(generator()); + Int128 x = static_cast(generator()) << 96 | static_cast(generator()) << 32 | + static_cast(generator()) << 64 | static_cast(generator()); data[i] = x; } } @@ -329,10 +333,12 @@ void fillColumnWithRandomData(IColumn & column, DataTypePtr type, UInt64 limit, { auto & data = typeid_cast &>(column).getData(); data.resize(limit); - pcg64_oneseq generator(random_seed); + pcg32 generator(random_seed); for (UInt64 i = 0; i < limit; ++i) { - auto x = UInt128(generator(), generator()); + UInt64 a = static_cast(generator()) << 32 | static_cast(generator()); + UInt64 b = static_cast(generator()) << 32 | static_cast(generator()); + auto x = UInt128(a, b); data[i] = x; } } @@ -468,3 +474,4 @@ void registerTableFunctionRandom(TableFunctionFactory & factory) } + diff --git a/dbms/tests/queries/0_stateless/01072_random_table_function.reference b/dbms/tests/queries/0_stateless/01072_random_table_function.reference index 2770d1fcaf3..3111e897928 100644 --- a/dbms/tests/queries/0_stateless/01072_random_table_function.reference +++ b/dbms/tests/queries/0_stateless/01072_random_table_function.reference @@ -1,14 +1,14 @@ UInt64 Int64 UInt32 Int32 UInt16 Int16 UInt8 Int8 -2254772619926532955 2254772619926532955 1234817989 1234817989 54213 -11323 197 -59 -9120028858397505560 9120028858397505560 1171957426 1171957426 42674 -22862 178 -78 -4555697903102013946 4555697903102013946 275100647 275100647 46055 -19481 231 -25 -5784362079052877875 5784362079052877875 1033685688 1033685688 51896 -13640 184 -72 -11035971995277520997 -7410772078432030619 180895192 180895192 15832 15832 216 -40 -7901646768096461004 7901646768096461004 135557292 135557292 28844 28844 172 -84 -6733841386518201279 6733841386518201279 716914271 716914271 15967 15967 95 95 -7736560050027905187 7736560050027905187 1012211222 1012211222 7702 7702 22 22 -2199287578947862030 2199287578947862030 2185722662 -2109244634 31526 31526 38 38 -3019483913099890467 3019483913099890467 2647224658 -1647742638 29010 29010 82 82 +5303502880439445170 5303502880439445170 1234817989 1234817989 54213 -11323 197 -59 +1181548283007126200 1181548283007126200 1171957426 1171957426 42674 -22862 178 -78 +776938933779198124 776938933779198124 275100647 275100647 46055 -19481 231 -25 +3079123348992892438 3079123348992892438 1033685688 1033685688 51896 -13640 184 -72 +9387607354063286610 -9059136719646265006 180895192 180895192 15832 15832 216 -40 +1372288647685551139 1372288647685551139 135557292 135557292 28844 28844 172 -84 +10000611939545164999 -8446132134164386617 716914271 716914271 15967 15967 95 95 +1156046020585944904 1156046020585944904 1012211222 1012211222 7702 7702 22 22 +8349022212802674610 8349022212802674610 2185722662 -2109244634 31526 31526 38 38 +6331568246612505101 6331568246612505101 2647224658 -1647742638 29010 29010 82 82 - Enum8(\'hello\' = 1, \'world\' = 5) world @@ -71,40 +71,40 @@ DateTime64(3) DateTime64(6) DateTime64(6, \'Europe/Moscow\') 1996-11-02 14:35:41.110 1996-11-02 14:35:41.183110 1996-11-02 14:35:41.183110 - Float32 Float64 -9.783235e37 2.1973467205491123e307 -9.285203e37 8.887754501811354e307 -2.1795718e37 4.4396706606805647e307 -8.1897013e37 5.637042481600483e307 -1.4331993e37 1.07549012514996e308 -1.0739954e37 7.700402896226395e307 -5.67998e37 6.562339881458101e307 -8.019563e37 7.539520705557441e307 -1.7317079e38 2.143274805821858e307 -2.0973474e38 2.9425818885529257e307 +9.783235e37 5.168430093085938e307 +9.285203e37 4.905322146512668e307 +2.1795718e37 1.151455903014231e307 +8.1897013e37 4.326574656543525e307 +1.4331993e37 7.571513877802428e306 +1.0739954e37 5.673859577292225e306 +5.67998e37 3.000702391289156e307 +8.019563e37 4.236691550453344e307 +1.7317079e38 9.148518147657713e307 +2.0973474e38 1.108017190180919e308 - Decimal32(4) Decimal64(8) Decimal64(8) -123481.7989 22547726199.26532955 4159321346419233104838.6879832895010840 -117195.7426 91200288583.97505560 8403779329565810688767.7049545291714611 -27510.0647 45556979031.02013946 -13670461591942827725055.0250490776469300 -103368.5688 57843620790.52877875 12421744869005473959544.2499747955622051 -18089.5192 -74107720784.32030619 4056969511333950153663.4915186231430947 -13555.7292 79016467680.96461004 -8819413736166121578589.4583420666183888 -71691.4271 67338413865.18201279 13058329479868658041313.8432372419860363 -101221.1222 77365600500.27905187 -4693380431928321782727.0243506636623202 --210924.4634 21992875789.47862030 13765369952377767241248.9441272127848016 --164774.2638 30194839130.99890467 -13890064946313418575619.0315227826809939 +123481.7989 53035028804.39445170 9783236031310378439643.1472294664915640 +117195.7426 11815482830.07126200 1433199368304978416824.6218455877230102 +27510.0647 7769389337.79198124 -16711157663899806765510.0649541344873437 +103368.5688 30791233489.92892438 -15580363795141897721982.8655941091887288 +18089.5192 -90591367196.46265006 15401227599802737909025.7109101500379661 +13555.7292 13722886476.85551139 7984757011464664209957.7603879176325739 +71691.4271 -84461321341.64386617 5816721179226388864892.0657225650146885 +101221.1222 11560460205.85944904 -5775130779420200933943.2970518536587084 +-210924.4634 83490222128.02674610 -5061941637011485215525.6417334537314088 +-164774.2638 63315682466.12505101 -7557281935671753244601.7559777699677732 - Tuple(Int32, Int64) -(1234817989,2254772619926532955) -(1171957426,9120028858397505560) -(275100647,4555697903102013946) -(1033685688,5784362079052877875) -(180895192,-7410772078432030619) -(135557292,7901646768096461004) -(716914271,6733841386518201279) -(1012211222,7736560050027905187) -(-2109244634,2199287578947862030) -(-1647742638,3019483913099890467) +(1234817989,5303502880439445170) +(1171957426,1181548283007126200) +(275100647,776938933779198124) +(1033685688,3079123348992892438) +(180895192,-9059136719646265006) +(135557292,1372288647685551139) +(716914271,-8446132134164386617) +(1012211222,1156046020585944904) +(-2109244634,8349022212802674610) +(-1647742638,6331568246612505101) - Array(Int8) [-59,-78,-25,-72,-40,-84,95,22,38] @@ -131,16 +131,16 @@ Array(Nullable(Int32)) [371735004,462118779,148602156,-1055384004,-1041274619,247762201,522289659,822210177] - Tuple(Int32, Array(Int64)) -(1234817989,[2254772619926532955,9120028858397505560,4555697903102013946,5784362079052877875,-7410772078432030619,7901646768096461004,6733841386518201279,7736560050027905187,2199287578947862030]) -(1171957426,[3019483913099890467,-4781013766399904222,-5327852745410412752,7078934595552553093,2990244123355912075,-2544286630298820818]) -(275100647,[6155991081669718686,7462222003717329977,-8255668614967296432,-7529819295378879967,-4777308097681484883,-4064480117123591373,6674750820081216293]) -(1033685688,[2050663721809231639,-6384194708780112896,-2808232718275215658,1619954721090656792,-5627002805867168609,-6128563945701772338,-7146544521171569603,6504888450989032669]) -(180895192,[1199208254069819846,-4069733657855461419]) -(135557292,[192577216783361448,-7343112807738526333]) -(716914271,[-9207713629233477390]) -(1012211222,[-562393447932771686,-6225026423445182831]) -(-2109244634,[-1388479317275096889,-1222297392734207149]) -(-1647742638,[3396028458740199176,8610993157653131131,-4072576266223306473,-6818310818869145616,-5713972449102020873,8197031236106666677,-1239306987803343619,8267468115072584172]) +(1234817989,[5303502880439445170,1181548283007126200,776938933779198124,3079123348992892438,-9059136719646265006,1372288647685551139,-8446132134164386617,1156046020585944904,8349022212802674610]) +(1171957426,[6331568246612505101,4328545451735172025,837735849651948139,3153250869148769217,-7783757525582213563,-3130704671775162746]) +(275100647,[-1986779751559399244,-2744084058313283668,1626824019940122840,-4096810749907976552,-101588646332188196,1984785042821053740,-4532839778647840507]) +(1033685688,[1064130551002268155,3531365820900360285,-6430432825062036096,673694157720662742,1098480112755237234,-5582206543758520198,1822983700066063177,-3760535601444222426]) +(180895192,[7337695457286344221,5872187140729212089]) +(135557292,[5142382871975234625,-3506363600639473930]) +(716914271,[2213685469994781810]) +(1012211222,[-5533057085551498458,1093259139288543534]) +(-2109244634,[2385587103426775499,6630437694997012859]) +(-1647742638,[-8881439430624548159,6128668282898355635,5437491183672208077,3669750475264598009,1788253815182834821,3436179822464488712,6060518060841781539,5049334787152748055]) - Nullable(String) )/VC)%f9 @@ -167,28 +167,28 @@ Array(String) ['\0n3;','\0bX(o2]uC','\0up_X\'','\0s','\05j|iS,','\0','\0(y.aRsVz','\0T:64 ]'] - UUID -1f4a8fc0-63ff-735b-7e90-d9ed3e183818 -3f39171b-1263-31fa-5046-2ea9fe2fd033 -9927a60f-01ac-f065-6da8-49def100c0cc -5d736910-493d-c3bf-6b5d-c8601d6440a3 -1e857066-961d-be0e-29e7-5c9efd534f23 -bda66d4f-737b-3622-b60f-aa27fe38ff30 -623d6d82-4422-2885-297f-7b2fec54178b -dcb0e0ca-3a43-5f2e-556e-7945df65729e -678f2360-36ac-d439-8d6d-f92295887e50 -9780b53e-dc0f-4a21-bdb3-9798af1913ad +4999d3c5-45da-a6b2-1065-b3e73d9ccab8 +0ac83dd8-0814-70ac-2abb-3e5f3c551e16 +82477b26-9dc9-7152-130b-59411e993423 +8ac94f92-d11e-acc7-100b-19bb11077748 +73ddaf77-aff2-ffb2-57de-3fc6327b320d +3c121527-9cce-c3b9-0ba0-3c51161d4a6b +2bc298f5-774d-dbc1-93fa-89c545e60e45 +d48d80b0-2e2f-7e86-e46d-8a57a55cf8b4 +d9eb0e16-766c-a7ac-1693-a3d3ac9aa0d8 +c7253454-2bbb-be98-fe97-15aa162839dc - Array(Nullable(UUID)) -['1f4a8fc0-63ff-735b-7e90-d9ed3e183818','3f39171b-1263-31fa-5046-2ea9fe2fd033','9927a60f-01ac-f065-6da8-49def100c0cc','5d736910-493d-c3bf-6b5d-c8601d6440a3','1e857066-961d-be0e-29e7-5c9efd534f23','bda66d4f-737b-3622-b60f-aa27fe38ff30','623d6d82-4422-2885-297f-7b2fec54178b','dcb0e0ca-3a43-5f2e-556e-7945df65729e','678f2360-36ac-d439-8d6d-f92295887e50'] -['9780b53e-dc0f-4a21-bdb3-9798af1913ad','c79810de-3635-d333-5ca1-7a81ab302b25','1c756bca-4438-3f17-a766-c8bcbe3ba400','d9072738-ac93-7ed6-167b-3c3c66d35a18','b1e8dec2-de29-3c9f-aaf2-f78fd92df3ce','9cd25f9f-3c0d-f43d-5a46-0194f0be04dd'] -['10a4718d-ab8c-49c6-c785-66ccf112f7d5','02ac2bf5-5634-a5a8-9a18-05ce8d1fb583','8037a13d-2004-08f2-f831-fa2387f5c29a','a99c4373-1121-2691-ecbb-216adbd748c7','ef0986ff-5031-0353-2f21-1de3ea53af08','778064a7-653b-ef7b-c77b-4d769b12b917','a1607e6f-691a-0ff0-b0b3-e454dae7bef7'] -['71c1b47a-c0eb-42b5-eecd-18dc585284fd','72bbf272-9ec5-09ec-f339-b5dac55c037b','26e5bce5-43f7-59b0-84c6-ef509f4c45eb','305fcbff-c366-2033-a8c5-d648f236e754','3a0d329f-f897-84e9-9e87-9501a713e63d','54bda20c-d5cd-a08a-c078-3c4fd81f4f55','43f549d1-3e5b-d5bf-ed32-b4850648bdc8','7eb6ac4f-06e0-ff48-6330-3c7afa5f2644'] -['17b9a4a5-fef8-a3f9-5af4-3b6e67ca62c9','3f524d8e-320d-00dc-c210-e199206550db'] -['005c592e-5081-9f3d-1fcb-5a9e82f39f97','29cf228d-b325-4a34-3eff-e80494a79260'] -['6c08b54b-8cf8-b96d-f087-8b54f5e72d0e'] -['7122e162-ab8b-a84a-6b71-c0846cf0204d','51c1de1a-24c7-18d6-39ed-e9023205610c'] -['f09d6779-1106-d667-e7c9-9a0cad544afe','62060fec-ee13-7c66-5da4-02c8f4d50dc9'] -['df1d0d54-d639-9c9b-2070-622fc9d82203','f23ef5b9-3797-9b0e-b8ac-67ea31b99c3e','e48afe73-9e22-7439-afed-d53b6ea204f4','d7f1ab47-4928-7623-283e-fb3f16aebeba','ea270407-d32f-a407-add2-3ae2d1113ccb','c43e9fff-2980-a1d1-f1bb-ff94d3cffbc2','a0cd54e6-0a2d-07ec-88ad-4f5d29c15b06','5e93413f-2eb9-5363-17ab-e2215b8b19e0'] +['4999d3c5-45da-a6b2-1065-b3e73d9ccab8','0ac83dd8-0814-70ac-2abb-3e5f3c551e16','82477b26-9dc9-7152-130b-59411e993423','8ac94f92-d11e-acc7-100b-19bb11077748','73ddaf77-aff2-ffb2-57de-3fc6327b320d','3c121527-9cce-c3b9-0ba0-3c51161d4a6b','2bc298f5-774d-dbc1-93fa-89c545e60e45','d48d80b0-2e2f-7e86-e46d-8a57a55cf8b4','d9eb0e16-766c-a7ac-1693-a3d3ac9aa0d8'] +['c7253454-2bbb-be98-fe97-15aa162839dc','1b8b5f7b-08db-7d2c-c118-1e3cc1ef6905','0ec48d19-1f21-81fb-3101-ee810eb8c05d','a6c2836a-5e2a-6d80-0959-71445e3232d6','0f3e95d7-b19b-6172-b288-04b96869907a','194c8a04-41c3-d349-cbcf-e4cb6eb50a26'] +['65d4bb24-418c-ce1d-517e-3317a1c80cb9','475d69f9-ff23-0c41-cf56-e4e1fa8646f6','1eb89734-d37b-8c72-b336-a1e565beab26','0f2c0965-14be-312e-211b-4ed4c88189cb','5c040bf4-3ba8-217b-84be-c9e4cb79eac1','550d6755-0c5c-15b3-4b75-d96c2819aacd','32ed929d-3477-9ff9-18d1-275ece35bc85'] +['2fafc357-c65c-5508-541b-4910fb504923','4612d73c-8443-1e17-1626-c13c17ccc201','2223f59a-b8f4-6b52-f13a-4c1e8c23aacc','d788aee1-23f3-570e-4f3b-130d38585201','6afb1f5e-3a1c-1610-a14c-e746b9618304','b003bd9a-5c3a-4132-9928-bc27b6b51628','c520f34b-e6e2-d2dd-e584-1a6eba57b77e','dbb4f033-dd30-090b-0b4a-a0ca4ec23e69'] +['725baf45-9f31-35bd-1019-914968c49767','8a96d309-cf7c-a95f-461a-e7764a64bccf'] +['8f12d466-baa6-d680-c17c-f399935f6b62','bb22e0c8-0ec2-15b1-3250-421c9806fd49'] +['b7013645-4614-d52e-c59c-65ed621e8414'] +['d4ed7d7f-8b1f-6991-74f7-cb187d47628b','51a3ec18-4967-32ae-d38f-656770d52224'] +['040cb3cc-a02d-58f9-7fc5-6bb0366b7366','f4bb3a59-ec6a-9c74-2f11-992c5b6d4b05'] +['8a9514e5-d533-7930-eb5a-6dd256a041c0','150b4bbf-f6a8-52d2-ec0d-a81401100801','a8ad6b08-d188-00a1-e4e0-616b324a4348','c209640a-5bc6-3e77-4556-30775457b60f','a482190e-eaae-f6d0-d1da-0915d03ccc02','16fbefc0-3eb7-7dec-6853-1557fa2fb964','67c6e040-dab2-56f9-0694-e9f57fad2000','66fcaeaf-4a5b-476a-cb8d-f9dd9ec16abe'] - FixedString(4) Ų @@ -223,18 +223,18 @@ String \0> \0XjbW:s< - -[77] -124167.6723 ('2061-04-17 21:59:44.573','3f72f405-ec3e-13c8-44ca-66ef335f7835') -[32,110] -141397.7312 ('1979-02-09 03:43:48.526','982486d1-5a5d-a308-e525-7bd8b80ffa73') -[68] -67417.0770 ('2080-03-12 14:17:31.269','110425e5-413f-10a6-05ba-fa6b3e929f15') +[77] -124167.6723 ('2061-04-17 21:59:44.573','b5fd844d-abb8-6b20-d7d0-f86e1120e744') +[32,110] -141397.7312 ('1979-02-09 03:43:48.526','bce332bd-cf45-f2cb-7da5-ddf9370fb1c7') +[68] -67417.0770 ('2080-03-12 14:17:31.269','bebf8e52-1ceb-73c7-2ead-f1eed124f71d') - -[-59,-78,-25,-72,-40,-84,95,22,38] 1234817989 )/VC)%f9 123481.7989 o 2.1973467205491123e307 ('2106-02-07','2009-02-16 23:59:49','2007-02-20 10:43:46.989','1f4a8fc0-63ff-735b-7e90-d9ed3e183818') Ų -[82,65,35,-110,-57,-69] 1171957426 \0ih|;B 117195.7426 w 8.887754501811354e307 ('2086-11-02','2007-02-20 10:43:46','2002-10-04 02:54:48.647','3f39171b-1263-31fa-5046-2ea9fe2fd033') -[72,119,-78,-58,13,39,-71] 275100647 \0J"Z,kd 27510.0647 w 4.4396706606805647e307 ('2096-02-04','1978-09-20 03:50:47','1974-04-19 01:48:12.192','9927a60f-01ac-f065-6da8-49def100c0cc') ج -[81,107,-11,-63,-59,69,-80,-122] 1033685688 \0m"m]$35 103368.5688 w 5.637042481600483e307 ('2106-02-07','2002-10-04 02:54:48','2002-01-28 12:47:02.271','5d736910-493d-c3bf-6b5d-c8601d6440a3') _ -[87,-76] 180895192 \00 18089.5192 h 1.07549012514996e308 ('2013-05-07','1975-09-25 19:39:52','2053-11-20 07:10:58.662','1e857066-961d-be0e-29e7-5c9efd534f23') &R -[22,-84] 135557292 \0( 13555.7292 w 7.700402896226395e307 ('2048-12-21','1974-04-19 01:48:12','1986-04-08 19:07:15.849','bda66d4f-737b-3622-b60f-aa27fe38ff30') A# -[-45] 716914271 \0 71691.4271 h 6.562339881458101e307 ('2013-09-19','1992-09-19 18:51:11','2081-03-06 04:00:55.914','623d6d82-4422-2885-297f-7b2fec54178b') -[-40,84] 1012211222 \0g 101221.1222 h 7.539520705557441e307 ('1991-02-02','2002-01-28 12:47:02','1979-01-20 20:39:20.939','dcb0e0ca-3a43-5f2e-556e-7945df65729e') H -[-104,-86] 2185722662 \0> -210924.4634 w 2.143274805821858e307 ('2056-04-25','2039-04-06 20:11:02','2063-07-18 01:46:10.215','678f2360-36ac-d439-8d6d-f92295887e50') w -[-36,123,44,60,5,25,-5,-127] 2647224658 \0XjbW:s< -164774.2638 o 2.9425818885529257e307 ('2049-06-05','2053-11-20 07:10:58','1996-11-02 14:35:41.110','9780b53e-dc0f-4a21-bdb3-9798af1913ad') \r +[-59,-78,-25,-72,-40,-84,95,22,38] 1234817989 )/VC)%f9 123481.7989 o 5.168430093085938e307 ('2106-02-07','2009-02-16 23:59:49','2007-02-20 10:43:46.989','4999d3c5-45da-a6b2-1065-b3e73d9ccab8') Ų +[82,65,35,-110,-57,-69] 1171957426 \0ih|;B 117195.7426 w 4.905322146512668e307 ('2086-11-02','2007-02-20 10:43:46','2002-10-04 02:54:48.647','0ac83dd8-0814-70ac-2abb-3e5f3c551e16') +[72,119,-78,-58,13,39,-71] 275100647 \0J"Z,kd 27510.0647 w 1.151455903014231e307 ('2096-02-04','1978-09-20 03:50:47','1974-04-19 01:48:12.192','82477b26-9dc9-7152-130b-59411e993423') ج +[81,107,-11,-63,-59,69,-80,-122] 1033685688 \0m"m]$35 103368.5688 w 4.326574656543525e307 ('2106-02-07','2002-10-04 02:54:48','2002-01-28 12:47:02.271','8ac94f92-d11e-acc7-100b-19bb11077748') _ +[87,-76] 180895192 \00 18089.5192 h 7.571513877802428e306 ('2013-05-07','1975-09-25 19:39:52','2053-11-20 07:10:58.662','73ddaf77-aff2-ffb2-57de-3fc6327b320d') &R +[22,-84] 135557292 \0( 13555.7292 w 5.673859577292225e306 ('2048-12-21','1974-04-19 01:48:12','1986-04-08 19:07:15.849','3c121527-9cce-c3b9-0ba0-3c51161d4a6b') A# +[-45] 716914271 \0 71691.4271 h 3.000702391289156e307 ('2013-09-19','1992-09-19 18:51:11','2081-03-06 04:00:55.914','2bc298f5-774d-dbc1-93fa-89c545e60e45') +[-40,84] 1012211222 \0g 101221.1222 h 4.236691550453344e307 ('1991-02-02','2002-01-28 12:47:02','1979-01-20 20:39:20.939','d48d80b0-2e2f-7e86-e46d-8a57a55cf8b4') H +[-104,-86] 2185722662 \0> -210924.4634 w 9.148518147657713e307 ('2056-04-25','2039-04-06 20:11:02','2063-07-18 01:46:10.215','d9eb0e16-766c-a7ac-1693-a3d3ac9aa0d8') w +[-36,123,44,60,5,25,-5,-127] 2647224658 \0XjbW:s< -164774.2638 o 1.108017190180919e308 ('2049-06-05','2053-11-20 07:10:58','1996-11-02 14:35:41.110','c7253454-2bbb-be98-fe97-15aa162839dc') \r - From 59c4f53fec9df04f1369e1d320fae9b84c203662 Mon Sep 17 00:00:00 2001 From: CurtizJ Date: Wed, 12 Feb 2020 02:29:34 +0300 Subject: [PATCH 0250/2007] fix polymorphic parts fetching --- dbms/src/Storages/MergeTree/DataPartsExchange.cpp | 5 ++--- dbms/src/Storages/MergeTree/MergeTreeDataPartType.h | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp index b3325d1f0dc..6a209223b33 100644 --- a/dbms/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/dbms/src/Storages/MergeTree/DataPartsExchange.cpp @@ -259,9 +259,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( part_file.createDirectory(); - MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, reservation->getDisk(),relative_part_path); - new_data_part->is_temp = true; - MergeTreeData::DataPart::Checksums checksums; for (size_t i = 0; i < files; ++i) { @@ -305,6 +302,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPart( assertEOF(in); + MergeTreeData::MutableDataPartPtr new_data_part = data.createPart(part_name, reservation->getDisk(), relative_part_path); + new_data_part->is_temp = true; new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); new_data_part->checksums.checkEqual(checksums, false); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h index a7d176f7baa..bb87918d3a5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h +++ b/dbms/src/Storages/MergeTree/MergeTreeDataPartType.h @@ -41,7 +41,7 @@ public: String toString() const; private: - Value value; + Value value; }; } From 526428ffe0f58782d5fdd8c4f9cf3952d0098783 Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 12 Feb 2020 13:00:24 +0300 Subject: [PATCH 0251/2007] Better --- .../MergeTree/ReplicatedMergeTreeQueue.cpp | 18 +++++++------- .../MergeTree/ReplicatedQueueAlterState.h | 24 ++++++++++++------- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 6b0322d1e7e..1a1e7fd4aee 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/dbms/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -288,7 +288,7 @@ void ReplicatedMergeTreeQueue::removePartFromMutations(const String & part_name) { MutationStatus & status = *it->second; - LOG_DEBUG(log, "Removing part name:" << part_name << " from mutation:" << status.entry->znode_name); + LOG_DEBUG(log, "Removing part name:" << part_name << " from mutation:" << status.entry->znode_name << " block number :" << status.entry->block_numbers.begin()->second); status.parts_to_do.removePartAndCoveredParts(part_name); if (status.parts_to_do.size() == 0) some_mutations_are_probably_done = true; @@ -995,19 +995,17 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( MergeTreeData & data, std::lock_guard & state_lock) const { - if (entry.type == LogEntry::GET_PART) - { - if (!entry.actual_new_part_name.empty() && !alter_sequence.canExecuteGetEntry(entry.actual_new_part_name, format_version, state_lock)) - return false; - - if (!entry.new_part_name.empty() && !alter_sequence.canExecuteGetEntry(entry.new_part_name, format_version, state_lock)) - return false; - } - if (entry.type == LogEntry::MERGE_PARTS || entry.type == LogEntry::GET_PART || entry.type == LogEntry::MUTATE_PART) { + if (!entry.actual_new_part_name.empty() + && !alter_sequence.canExecuteGetEntry(entry.actual_new_part_name, format_version, state_lock)) + return false; + + if (!entry.new_part_name.empty() && !alter_sequence.canExecuteGetEntry(entry.new_part_name, format_version, state_lock)) + return false; + for (const String & new_part_name : entry.getBlockingPartNames()) { if (!isNotCoveredByFuturePartsImpl(new_part_name, out_postpone_reason, state_lock)) diff --git a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h index 7e10a81247c..d065ee0a350 100644 --- a/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h +++ b/dbms/src/Storages/MergeTree/ReplicatedQueueAlterState.h @@ -55,7 +55,7 @@ public: void addMetadataAlter(int alter_version, std::lock_guard & /*state_lock*/) { - LOG_DEBUG(log, "Adding meta with alter version:" << alter_version); + //LOG_DEBUG(log, "Adding meta with alter version:" << alter_version); if (!queue_state.count(alter_version)) queue_state.emplace(alter_version, AlterInQueue({}, false)); else @@ -73,14 +73,17 @@ public: for (const auto & [block_number, state] : queue_state) { - LOG_DEBUG(log, "Looking at block:" << block_number << " with part name:" << part_name); - if (state.block_numbers.count(info.partition_id)) + LOG_DEBUG(log, "Looking at alter:" << block_number << " with part name:" << part_name); + if (!state.block_numbers.empty()) { - LOG_DEBUG(log, "Block number:" << block_number << " has part name " << part_name << " version " << state.block_numbers.at(info.partition_id)); - return info.getDataVersion() < state.block_numbers.at(info.partition_id); + LOG_DEBUG(log, "Block number:" << block_number << " has part name " << part_name << " version " << state.block_numbers.at(info.partition_id) << " metadata is done:" << state.metadata_finished); + if (!state.metadata_finished) + return info.getDataVersion() < state.block_numbers.at(info.partition_id); + else + return info.getDataVersion() <= state.block_numbers.at(info.partition_id); } } - LOG_DEBUG(log, "Nobody has block number for part " << part_name); + //LOG_DEBUG(log, "Nobody has block number for part " << part_name); return true; } @@ -95,10 +98,11 @@ public: if (queue_state.begin()->first != alter_version) { - LOG_DEBUG(log, "Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue_state.begin()->first)); + //LOG_DEBUG(log, "Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue_state.begin()->first)); throw Exception("Finished metadata alter with version " + std::to_string(alter_version) + " but current alter in queue is " + std::to_string(queue_state.begin()->first), ErrorCodes::LOGICAL_ERROR); } + LOG_DEBUG(log, "FINISH METADATA ALTER: " << alter_version); if (!have_data_alter) { queue_state.erase(alter_version); @@ -115,12 +119,14 @@ public: /// queue can be empty after load of finished mutation without move of mutation pointer if (queue_state.empty()) { - LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE EMPTY"); + //LOG_DEBUG(log, "FINISHING DATA ALTER WITH VERSION:" << alter_version << " BUT QUEUE EMPTY"); return; } LOG_DEBUG(log, "FINISH DATA ALTER: " << alter_version); + if (!queue_state.count(alter_version)) + std::terminate(); queue_state.erase(alter_version); } @@ -133,6 +139,8 @@ public: } if (alter_version < queue_state.begin()->first) return true; + if (!queue_state.count(alter_version)) + std::terminate(); return queue_state.at(alter_version).metadata_finished; } bool canExecuteMetaAlter(int alter_version, std::lock_guard & /*state_lock*/) const From 92fc1a9b2b7749c0b70db45793b70af1699f18eb Mon Sep 17 00:00:00 2001 From: millb Date: Wed, 12 Feb 2020 13:06:09 +0300 Subject: [PATCH 0252/2007] fixed bugs --- dbms/src/Storages/Distributed/DirectoryMonitor.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index a9ebdb02076..77467da511e 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -190,10 +190,10 @@ ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::stri const auto & shards_info = cluster->getShardsInfo(); const auto & shards_addresses = cluster->getShardsAddresses(); - /// check format shard{shard_index}_number{number_index} + /// check new format shard{shard_index}_number{number_index} if (address.shard_index != 0) { - return shards_info[address.shard_index].per_replica_pools[address.replica_index]; + return shards_info[address.shard_index - 1].per_replica_pools[address.replica_index - 1]; } /// existing connections pool have a higher priority From 6abeabc1f2503817c167c12a77ef37116584994c Mon Sep 17 00:00:00 2001 From: millb Date: Wed, 12 Feb 2020 13:13:11 +0300 Subject: [PATCH 0253/2007] fixed bugs --- dbms/src/Interpreters/Cluster.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index b57d2ab5b50..b569a5eed81 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -139,8 +139,7 @@ String Cluster::Address::toFullString() const { return ((shard_index == 0) ? "" : "shard" + std::to_string(shard_index)) + - ((replica_index == 0) ? "" : "_replica" + std::to_string(replica_index)) + - ((secure == Protocol::Secure::Enable) ? "+secure" : ""); + ((replica_index == 0) ? "" : "_replica" + std::to_string(replica_index)); } Cluster::Address Cluster::Address::fromFullString(const String & full_string) @@ -148,17 +147,9 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string) const char * address_begin = full_string.data(); const char * address_end = address_begin + full_string.size(); - Protocol::Secure secure = Protocol::Secure::Disable; - const char * secure_tag = "+secure"; - if (endsWith(full_string, secure_tag)) - { - address_end -= strlen(secure_tag); - secure = Protocol::Secure::Enable; - } - const char * user_pw_end = strchr(full_string.data(), '@'); - /// parsing with format [shard{shard_index}[_replica{replica_index}]] + /// parsing with [shard{shard_index}[_replica{replica_index}]] format if (!user_pw_end && startsWith(full_string, "shard")) { const char * underscore = strchr(full_string.data(), '_'); @@ -169,6 +160,15 @@ Cluster::Address Cluster::Address::fromFullString(const String & full_string) return address; } + /// parsing with old user[:password]@host:port#default_database format + Protocol::Secure secure = Protocol::Secure::Disable; + const char * secure_tag = "+secure"; + if (endsWith(full_string, secure_tag)) + { + address_end -= strlen(secure_tag); + secure = Protocol::Secure::Enable; + } + const char * colon = strchr(full_string.data(), ':'); if (!user_pw_end || !colon) throw Exception("Incorrect user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR); From e5b048ab51d25ac4a0e7d14200e037f51bbc2002 Mon Sep 17 00:00:00 2001 From: Guillaume Tassery Date: Wed, 12 Feb 2020 11:33:43 +0100 Subject: [PATCH 0254/2007] Can use bit(And|Or|Xor) using a bitset set as a FixedString --- dbms/src/Functions/FunctionBinaryArithmetic.h | 497 ++++++++++++------ dbms/src/Functions/bitAnd.cpp | 2 +- dbms/src/Functions/bitOr.cpp | 2 +- dbms/src/Functions/bitXor.cpp | 2 +- 4 files changed, 328 insertions(+), 175 deletions(-) diff --git a/dbms/src/Functions/FunctionBinaryArithmetic.h b/dbms/src/Functions/FunctionBinaryArithmetic.h index 4afec074f56..c18516decc6 100644 --- a/dbms/src/Functions/FunctionBinaryArithmetic.h +++ b/dbms/src/Functions/FunctionBinaryArithmetic.h @@ -12,10 +12,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include "IFunctionImpl.h" @@ -87,6 +89,33 @@ struct BinaryOperationImplBase } }; +template +struct FixedStringOperationImpl +{ + static void NO_INLINE vector_vector(const ColumnFixedString::Chars & a, const ColumnFixedString::Chars & b, ColumnFixedString::Chars & c) + { + size_t size = a.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a[i], b[i]); + } + + static void NO_INLINE vector_constant(const ColumnFixedString::Chars & a, const ColumnFixedString::Chars & b, ColumnFixedString::Chars & c) + { + size_t size = a.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a[i], b[i % b.size()]); + } + + static void NO_INLINE constant_vector(const ColumnFixedString::Chars & a, const ColumnFixedString::Chars & b, ColumnFixedString::Chars & c) + { + size_t size = b.size(); + for (size_t i = 0; i < size; ++i) + c[i] = Op::template apply(a[i % a.size()], b[i]); + } +}; + + + template struct BinaryOperationImpl : BinaryOperationImplBase { @@ -439,7 +468,7 @@ public: }; -template