From f8eb9028fd3bf7f6eea93a20b5a7564dc1fe33a0 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Tue, 6 Dec 2016 23:55:13 +0300 Subject: [PATCH 1/9] Refined query_log, processes and merges metrics. Also, dbms/CMakeLists.txt was rewritten. --- dbms/CMakeLists.txt | 113 +++++++++++++++--- dbms/include/DB/DataStreams/BlockIO.h | 8 +- .../DataStreams/CountingBlockOutputStream.h | 62 ++++++++++ .../InputStreamFromASTInsertQuery.h | 83 +++++++++++++ dbms/include/DB/Interpreters/ProcessList.h | 30 +++-- dbms/include/DB/Interpreters/QueryLog.h | 18 +-- .../include/DB/Storages/MergeTree/MergeList.h | 13 +- .../IProfilingBlockInputStream.cpp | 13 +- .../Interpreters/InterpreterInsertQuery.cpp | 5 + dbms/src/Interpreters/QueryLog.cpp | 6 + dbms/src/Interpreters/executeQuery.cpp | 74 ++++++------ .../MergeTree/MergeTreeDataMerger.cpp | 29 ++--- .../Storages/System/StorageSystemMerges.cpp | 12 +- .../System/StorageSystemProcesses.cpp | 16 ++- dbms/tests/clickhouse-test | 84 ++++++------- 15 files changed, 397 insertions(+), 169 deletions(-) create mode 100644 dbms/include/DB/DataStreams/CountingBlockOutputStream.h create mode 100644 dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index d4ca884e067..10985e8ca3b 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -17,25 +17,80 @@ if (NOT ENABLE_LIBTCMALLOC) add_definitions(-D NO_TCMALLOC) endif () -if (APPLE) - set (AIO_CPP_FILES "") - set (AIO_H_FILES "") - set (APPLE_ICONV_LIB iconv) -else() - set (AIO_H_FILES include/DB/Common/AIO.h - include/DB/IO/WriteBufferAIO.h - include/DB/IO/ReadBufferAIO.h) - set (AIO_CPP_FILES - src/IO/ReadBufferAIO.cpp - src/IO/WriteBufferAIO.cpp) - set (APPLE_ICONV_LIB "") -endif() - -add_library (string_utils +add_library(string_utils include/DB/Common/StringUtils.h src/Common/StringUtils.cpp) -add_library (dbms +set(AIO_H_FILES + include/DB/Common/AIO.h + include/DB/IO/WriteBufferAIO.h + include/DB/IO/ReadBufferAIO.h) +set (AIO_CPP_FILES + src/IO/ReadBufferAIO.cpp + src/IO/WriteBufferAIO.cpp) +set (APPLE_ICONV_LIB) + +if (APPLE) + set(APPLE_ICONV_LIB iconv) +endif() + +macro(add_glob cur_list) + file(GLOB __tmp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${ARGN}) + list(APPEND ${cur_list} ${__tmp}) +endmacro() + +macro(add_headers_and_sources prefix common_path) + add_glob(${prefix}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include/DB/${common_path}/*.h include/DB/${common_path}/*.inl) + add_glob(${prefix}_sources src/${common_path}/*.cpp src/${common_path}/*.h) +endmacro() + +macro(add_headers_only prefix common_path) + add_glob(${prefix}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include/DB/${common_path}/*.h src/${common_path}/*.h) +endmacro() + +set(dbms_headers) +set(dbms_sources) +add_headers_and_sources(dbms Functions) +add_headers_and_sources(dbms Functions/Conditional) +add_headers_and_sources(dbms TableFunctions) +add_headers_and_sources(dbms Parsers) +add_headers_and_sources(dbms AggregateFunctions) +add_headers_and_sources(dbms Core) +add_headers_and_sources(dbms DataStreams) +add_headers_and_sources(dbms DataTypes) +add_headers_and_sources(dbms Databases) +add_headers_and_sources(dbms DataBases/Distributed) +add_headers_and_sources(dbms Dictionaries) +add_headers_and_sources(dbms Dictionaries/Embedded) +add_headers_and_sources(dbms Interpreters) +add_headers_and_sources(dbms Interpreters/ClusterProxy) +add_headers_and_sources(dbms Common) +add_headers_and_sources(dbms Common/HashTable) +add_headers_and_sources(dbms IO) +add_headers_and_sources(dbms Columns) +add_headers_and_sources(dbms Storages) +add_headers_and_sources(dbms Storages/Distributed) +add_headers_and_sources(dbms Storages/MergeTree) +add_headers_and_sources(dbms Storages/System) +add_headers_and_sources(dbms Core) +add_headers_and_sources(dbms Client) +add_headers_only(dbms Server) + +list(REMOVE_ITEM dbms_sources + src/Client/Client.cpp + src/Client/Benchmark.cpp + src/Storages/StorageCloud.cpp + src/Databases/DatabaseCloud.cpp + src/Common/StringUtils.cpp) + +if (APPLE) + list(REMOVE_ITEM dbms_headers ${AIO_H_FILES}) + list(REMOVE_ITEM dbms_sources ${AIO_CPP_FILES}) +endif() + +add_library(dbms ${dbms_headers} ${dbms_sources}) + +set (dbms_all_sources_old src/Server/InterserverIOHTTPHandler.h src/Server/Server.h src/Server/TCPHandler.h @@ -221,6 +276,7 @@ add_library (dbms include/DB/DataStreams/glueBlockInputStreams.h include/DB/DataStreams/CollapsingSortedBlockInputStream.h include/DB/DataStreams/IBlockOutputStream.h + include/DB/DataStreams/InputStreamFromASTInsertQuery.h include/DB/DataStreams/AsynchronousBlockInputStream.h include/DB/DataStreams/MaterializingBlockInputStream.h include/DB/DataStreams/ParallelInputsProcessor.h @@ -977,6 +1033,31 @@ add_library (dbms ${MONGODB_FILES} ) +###### +list(REMOVE_DUPLICATES dbms_all_sources_old) +list(SORT dbms_all_sources_old) + +set(dbms_all_sources ${dbms_headers} ${dbms_sources}) +list(REMOVE_DUPLICATES dbms_all_sources) +list(SORT dbms_all_sources) + +list(LENGTH dbms_all_sources_old len_old) +list(LENGTH dbms_all_sources len_new) +set(dbms_all_sources_union ${dbms_all_sources} ${dbms_all_sources_old}) +list(LENGTH dbms_all_sources_union len_union) +list(REMOVE_DUPLICATES dbms_all_sources_union) +list(LENGTH dbms_all_sources_union len_union_uniq) +message(STATUS "len_old ${len_old}, len_new ${len_new}, len_union ${len_union}, len_union_uniq ${len_union_uniq}") + +set(dbms_all_sources_not_presented ${dbms_all_sources_old}) +list(REMOVE_ITEM dbms_all_sources_not_presented ${dbms_all_sources}) +message(STATUS "not presented: ${dbms_all_sources_not_presented}") + +set(dbms_all_sources_extra ${dbms_all_sources}) +list(REMOVE_ITEM dbms_all_sources_extra ${dbms_all_sources_old}) +message(STATUS "dbms_all_sources_extra: ${dbms_all_sources_extra}") +###### + if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. set_source_files_properties( diff --git a/dbms/include/DB/DataStreams/BlockIO.h b/dbms/include/DB/DataStreams/BlockIO.h index 8169b3614fb..7f81eba193e 100644 --- a/dbms/include/DB/DataStreams/BlockIO.h +++ b/dbms/include/DB/DataStreams/BlockIO.h @@ -24,15 +24,15 @@ struct BlockIO Block in_sample; /// Пример блока, который будет прочитан из in. Block out_sample; /// Пример блока, которого нужно писать в out. - /// Здесь могут быть установлены колбэки для логгирования запроса. - std::function finish_callback; - std::function exception_callback; + /// Callbacks for query logging could be set here. + std::function finish_callback; + std::function exception_callback; /// Вызывайте эти функции, если нужно логгировать запрос. void onFinish() { if (finish_callback) - finish_callback(in.get()); + finish_callback(in.get(), out.get()); } void onException() diff --git a/dbms/include/DB/DataStreams/CountingBlockOutputStream.h b/dbms/include/DB/DataStreams/CountingBlockOutputStream.h new file mode 100644 index 00000000000..8694e18c05d --- /dev/null +++ b/dbms/include/DB/DataStreams/CountingBlockOutputStream.h @@ -0,0 +1,62 @@ +#pragma once +#include +#include +#include + + +namespace DB +{ + + +/// Proxy class which counts number of written block, rows, bytes +class CountingBlockOutputStream : public IBlockOutputStream +{ +public: + + CountingBlockOutputStream(const BlockOutputStreamPtr & stream_) + : stream(stream_) {} + + void setProgressCallback(ProgressCallback callback) + { + progress_callback = callback; + } + + void setProcessListElement(ProcessListElement * elem) + { + process_elem = elem; + } + + const Progress & getProgress() const + { + return progress; + } + + void write(const Block & block) override + { + stream->write(block); + + Progress local_progress(block.rowsInFirstColumn(), block.bytes(), 0); + progress.incrementPiecewiseAtomically(local_progress); + + if (process_elem) + process_elem->updateProgressOut(local_progress); + + if (progress_callback) + progress_callback(local_progress); + } + + void writePrefix() override { stream->writePrefix(); } + void writeSuffix() override { stream->writeSuffix(); } + void flush() override { stream->flush(); } + void onProgress(const Progress & progress) override { stream->onProgress(progress); } + String getContentType() const override { return stream->getContentType(); } + +protected: + + BlockOutputStreamPtr stream; + Progress progress; + ProgressCallback progress_callback; + ProcessListElement * process_elem = nullptr; +}; + +} diff --git a/dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h b/dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h new file mode 100644 index 00000000000..28ccf8db1c8 --- /dev/null +++ b/dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h @@ -0,0 +1,83 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + +class InputStreamFromASTInsertQuery : public IProfilingBlockInputStream +{ +public: + + InputStreamFromASTInsertQuery(const ASTPtr & ast, ReadBuffer & istr, const BlockIO & streams, Context & context) + { + const ASTInsertQuery * ast_insert_query = dynamic_cast(ast.get()); + + if (!ast_insert_query) + throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); + + String format = ast_insert_query->format; + if (format.empty()) + format = "Values"; + + /// Data could be in parsed (ast_insert_query.data) and in not parsed yet (istr) part of query. + + buf1 = std::make_unique( + const_cast(ast_insert_query->data), ast_insert_query->data ? ast_insert_query->end - ast_insert_query->data : 0, 0); + + if (ast_insert_query->data) + buffers.push_back(buf1.get()); + buffers.push_back(&istr); + + /** NOTE Must not read from 'istr' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. + * - because 'query.data' could refer to memory piece, used as buffer for 'istr'. + */ + + data_istr = std::make_unique(buffers); + + res_stream = context.getInputFormat(format, *data_istr, streams.out_sample, context.getSettings().max_insert_block_size); + } + + Block readImpl() override + { + return res_stream->read(); + } + + void readPrefixImpl() override + { + return res_stream->readPrefix(); + } + + void readSuffixImpl() override + { + return res_stream->readSuffix(); + } + + String getName() const override + { + return "InputStreamFromASTInsertQuery"; + } + + String getID() const override + { + return "InputStreamFromASTInsertQuery(" + toString(this) + ")"; + } + +private: + ConcatReadBuffer::ReadBuffers buffers; + std::unique_ptr buf1; + std::unique_ptr data_istr; + + BlockInputStreamPtr res_stream; +}; + +} diff --git a/dbms/include/DB/Interpreters/ProcessList.h b/dbms/include/DB/Interpreters/ProcessList.h index 7bd010cb2f0..02e84c8f4b5 100644 --- a/dbms/include/DB/Interpreters/ProcessList.h +++ b/dbms/include/DB/Interpreters/ProcessList.h @@ -37,9 +37,11 @@ struct ProcessInfo { String query; double elapsed_seconds; - size_t rows; - size_t bytes; + size_t read_rows; + size_t read_bytes; size_t total_rows; + size_t written_rows; + size_t written_bytes; Int64 memory_usage; ClientInfo client_info; }; @@ -53,7 +55,10 @@ struct ProcessListElement Stopwatch watch; - Progress progress; + /// Progress of input stream + Progress progress_in; + /// Progress of output stream + Progress progress_out; MemoryTracker memory_tracker; @@ -88,9 +93,9 @@ struct ProcessListElement current_memory_tracker = nullptr; } - bool update(const Progress & value) + bool updateProgressIn(const Progress & value) { - progress.incrementPiecewiseAtomically(value); + progress_in.incrementPiecewiseAtomically(value); if (priority_handle) priority_handle->waitIfNeed(std::chrono::seconds(1)); /// NOTE Could make timeout customizable. @@ -98,6 +103,13 @@ struct ProcessListElement return !is_cancelled; } + bool updateProgressOut(const Progress & value) + { + progress_out.incrementPiecewiseAtomically(value); + return !is_cancelled; + } + + ProcessInfo getInfo() const { ProcessInfo res; @@ -105,9 +117,11 @@ struct ProcessListElement res.query = query; res.client_info = client_info; res.elapsed_seconds = watch.elapsedSeconds(); - res.rows = progress.rows; - res.bytes = progress.bytes; - res.total_rows = progress.total_rows; + res.read_rows = progress_in.rows; + res.read_bytes = progress_in.bytes; + res.total_rows = progress_in.total_rows; + res.written_rows = progress_out.rows; + res.written_bytes = progress_out.bytes; res.memory_usage = memory_tracker.get(); return res; diff --git a/dbms/include/DB/Interpreters/QueryLog.h b/dbms/include/DB/Interpreters/QueryLog.h index 3e37e199ad9..f04cfb0f614 100644 --- a/dbms/include/DB/Interpreters/QueryLog.h +++ b/dbms/include/DB/Interpreters/QueryLog.h @@ -7,14 +7,13 @@ namespace DB { -/** Позволяет логгировать информацию о выполнении запросов: - * - о начале выполнения запроса; - * - метрики производительности, после выполнения запроса; - * - об ошибках при выполнении запроса. +/** Allows to log information about queries execution: + * - info about start of query execution; + * - performance metrics (are set at the end of query execution); + * - info about errors of query execution. */ -/** Что логгировать. - */ +/// A struct which will be inserted as row into query_log table struct QueryLogElement { enum Type @@ -27,7 +26,7 @@ struct QueryLogElement Type type = QUERY_START; - /// В зависимости от типа, не все поля могут быть заполнены. + /// Depending on the type of query and type of stage, not all the fields may be filled. time_t event_time{}; time_t query_start_time{}; @@ -36,6 +35,11 @@ struct QueryLogElement UInt64 read_rows{}; UInt64 read_bytes{}; + UInt64 written_rows{}; + UInt64 written_bytes{}; + + /// NOTE: Not obvious metric. + /// It always approximately equal to read_rows or written_rows at the end of query execution. UInt64 result_rows{}; UInt64 result_bytes{}; diff --git a/dbms/include/DB/Storages/MergeTree/MergeList.h b/dbms/include/DB/Storages/MergeTree/MergeList.h index 28b09100da5..5a7882aeee3 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeList.h +++ b/dbms/include/DB/Storages/MergeTree/MergeList.h @@ -34,20 +34,13 @@ struct MergeInfo std::atomic bytes_read_uncompressed{}; std::atomic bytes_written_uncompressed{}; - /// Updated only for Horizontal algorithm + /// In case of Vertical algorithm they are actual only for primary key columns std::atomic rows_read{}; std::atomic rows_written{}; /// Updated only for Vertical algorithm - /// mutually exclusive with rows_read and rows_written, updated either rows_written either columns_written std::atomic columns_written{}; - /// Updated in both cases - /// Number of rows for which primary key columns have been written - std::atomic rows_with_key_columns_read{}; - std::atomic rows_with_key_columns_written{}; - - MergeInfo(const std::string & database, const std::string & table, const std::string & result_part_name) : database{database}, table{table}, result_part_name{result_part_name} { @@ -66,9 +59,7 @@ struct MergeInfo bytes_written_uncompressed(other.bytes_written_uncompressed.load(std::memory_order_relaxed)), rows_read(other.rows_read.load(std::memory_order_relaxed)), rows_written(other.rows_written.load(std::memory_order_relaxed)), - columns_written(other.columns_written.load(std::memory_order_relaxed)), - rows_with_key_columns_read(other.rows_with_key_columns_read.load(std::memory_order_relaxed)), - rows_with_key_columns_written(other.rows_with_key_columns_written.load(std::memory_order_relaxed)) + columns_written(other.columns_written.load(std::memory_order_relaxed)) { } }; diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp index d3055c53ac9..51adad29139 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp @@ -6,7 +6,6 @@ #include #include - namespace DB { @@ -228,15 +227,15 @@ void IProfilingBlockInputStream::progressImpl(const Progress & value) if (process_list_elem) { - if (!process_list_elem->update(value)) + if (!process_list_elem->updateProgressIn(value)) cancel(); /// Общее количество данных, обработанных или предполагаемых к обработке во всех листовых источниках, возможно, на удалённых серверах. - size_t rows_processed = process_list_elem->progress.rows; - size_t bytes_processed = process_list_elem->progress.bytes; + size_t rows_processed = process_list_elem->progress_in.rows; + size_t bytes_processed = process_list_elem->progress_in.bytes; - size_t total_rows_estimate = std::max(rows_processed, process_list_elem->progress.total_rows.load(std::memory_order_relaxed)); + size_t total_rows_estimate = std::max(rows_processed, process_list_elem->progress_in.total_rows.load(std::memory_order_relaxed)); /** Проверяем ограничения на объём данных для чтения, скорость выполнения запроса, квоту на объём данных для чтения. * NOTE: Может быть, имеет смысл сделать, чтобы они проверялись прямо в ProcessList? @@ -270,7 +269,7 @@ void IProfilingBlockInputStream::progressImpl(const Progress & value) throw Exception("Logical error: unknown overflow mode", ErrorCodes::LOGICAL_ERROR); } - size_t total_rows = process_list_elem->progress.total_rows; + size_t total_rows = process_list_elem->progress_in.total_rows; if (limits.min_execution_speed || (total_rows && limits.timeout_before_checking_execution_speed != 0)) { @@ -283,7 +282,7 @@ void IProfilingBlockInputStream::progressImpl(const Progress & value) + " rows/sec., minimum: " + toString(limits.min_execution_speed), ErrorCodes::TOO_SLOW); - size_t total_rows = process_list_elem->progress.total_rows; + size_t total_rows = process_list_elem->progress_in.total_rows; /// Если предсказанное время выполнения больше, чем max_execution_time. if (limits.max_execution_time != 0 && total_rows) diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 89b951d7b01..02e0064fafc 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -101,6 +102,10 @@ BlockIO InterpreterInsertQuery::execute() context.getSettingsRef().min_insert_block_size_rows, context.getSettingsRef().min_insert_block_size_bytes); + auto out_wrapper = std::make_shared(out); + out_wrapper->setProcessListElement(context.getProcessListElement()); + out = std::move(out_wrapper); + BlockIO res; res.out_sample = getSampleBlock(); diff --git a/dbms/src/Interpreters/QueryLog.cpp b/dbms/src/Interpreters/QueryLog.cpp index a39b695ef5b..5a047036159 100644 --- a/dbms/src/Interpreters/QueryLog.cpp +++ b/dbms/src/Interpreters/QueryLog.cpp @@ -28,6 +28,9 @@ Block QueryLogElement::createBlock() {std::make_shared(), std::make_shared(), "read_rows"}, {std::make_shared(), std::make_shared(), "read_bytes"}, + {std::make_shared(), std::make_shared(), "written_rows"}, + {std::make_shared(), std::make_shared(), "written_bytes"}, + {std::make_shared(), std::make_shared(), "result_rows"}, {std::make_shared(), std::make_shared(), "result_bytes"}, @@ -102,6 +105,9 @@ void QueryLogElement::appendToBlock(Block & block) const block.unsafeGetByPosition(i++).column->insert(UInt64(read_rows)); block.unsafeGetByPosition(i++).column->insert(UInt64(read_bytes)); + block.unsafeGetByPosition(i++).column->insert(UInt64(written_rows)); + block.unsafeGetByPosition(i++).column->insert(UInt64(written_bytes)); + block.unsafeGetByPosition(i++).column->insert(UInt64(result_rows)); block.unsafeGetByPosition(i++).column->insert(UInt64(result_bytes)); diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index 7f8f8b488d0..d29e04d6b44 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include @@ -191,13 +193,23 @@ static std::tuple executeQueryImpl( if (res.in) { - if (IProfilingBlockInputStream * stream = dynamic_cast(res.in.get())) + if (auto stream = dynamic_cast(res.in.get())) { stream->setProgressCallback(context.getProgressCallback()); stream->setProcessListElement(context.getProcessListElement()); } } + if (res.out) + { + if (auto stream = dynamic_cast(res.out.get())) + { + stream->setProcessListElement(context.getProcessListElement()); + } + } + if (!res.out || !dynamic_cast(res.out.get())) + LOG_DEBUG(&Logger::get("executeQuery"), "res.out " << res.out.get() << " is empty"); + /// Everything related to query log. { QueryLogElement elem; @@ -218,7 +230,7 @@ static std::tuple executeQueryImpl( context.getQueryLog().add(elem); /// Also make possible for caller to log successful query finish and exception during execution. - res.finish_callback = [elem, &context, log_queries] (IBlockInputStream * stream) mutable + res.finish_callback = [elem, &context, log_queries] (IBlockInputStream * stream_in, IBlockOutputStream * stream_out) mutable { ProcessListElement * process_list_elem = context.getProcessListElement(); @@ -232,22 +244,35 @@ static std::tuple executeQueryImpl( elem.event_time = time(0); elem.query_duration_ms = elapsed_seconds * 1000; - elem.read_rows = process_list_elem->progress.rows; - elem.read_bytes = process_list_elem->progress.bytes; + elem.read_rows = process_list_elem->progress_in.rows; + elem.read_bytes = process_list_elem->progress_in.bytes; + + elem.written_rows = process_list_elem->progress_out.rows; + elem.written_bytes = process_list_elem->progress_out.bytes; auto memory_usage = process_list_elem->memory_tracker.getPeak(); elem.memory_usage = memory_usage > 0 ? memory_usage : 0; - if (stream) + if (stream_in) { - if (IProfilingBlockInputStream * profiling_stream = dynamic_cast(stream)) + if (auto profiling_stream = dynamic_cast(stream_in)) { const BlockStreamProfileInfo & info = profiling_stream->getProfileInfo(); + /// NOTE: INSERT SELECT query contains zero metrics elem.result_rows = info.rows; elem.result_bytes = info.bytes; } } + else if (stream_out) /// will be used only for ordinary INSERT queries + { + if (auto counting_stream = dynamic_cast(stream_out)) + { + /// NOTE: Redundancy. The same values could be extracted from process_list_elem->progress_out. + elem.result_rows = counting_stream->getProgress().rows; + elem.result_bytes = counting_stream->getProgress().bytes; + } + } if (elem.read_rows != 0) { @@ -280,8 +305,8 @@ static std::tuple executeQueryImpl( elem.query_duration_ms = elapsed_seconds * 1000; - elem.read_rows = process_list_elem->progress.rows; - elem.read_bytes = process_list_elem->progress.bytes; + elem.read_rows = process_list_elem->progress_in.rows; + elem.read_bytes = process_list_elem->progress_in.bytes; auto memory_usage = process_list_elem->memory_tracker.getPeak(); elem.memory_usage = memory_usage > 0 ? memory_usage : 0; @@ -369,35 +394,8 @@ void executeQuery( { if (streams.out) { - const ASTInsertQuery * ast_insert_query = dynamic_cast(ast.get()); - - if (!ast_insert_query) - throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); - - String format = ast_insert_query->format; - if (format.empty()) - format = "Values"; - - /// Data could be in parsed (ast_insert_query.data) and in not parsed yet (istr) part of query. - - ConcatReadBuffer::ReadBuffers buffers; - ReadBuffer buf1(const_cast(ast_insert_query->data), ast_insert_query->data ? ast_insert_query->end - ast_insert_query->data : 0, 0); - - if (ast_insert_query->data) - buffers.push_back(&buf1); - buffers.push_back(&istr); - - /** NOTE Must not read from 'istr' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. - * - because 'query.data' could refer to memory piece, used as buffer for 'istr'. - */ - - ConcatReadBuffer data_istr(buffers); - - BlockInputStreamPtr in{ - context.getInputFormat( - format, data_istr, streams.out_sample, context.getSettings().max_insert_block_size)}; - - copyData(*in, *streams.out); + InputStreamFromASTInsertQuery in(ast, istr, streams, context); + copyData(in, *streams.out); } if (streams.in) @@ -410,7 +408,7 @@ void executeQuery( BlockOutputStreamPtr out = context.getOutputFormat(format_name, ostr, streams.in_sample); - if (IProfilingBlockInputStream * stream = dynamic_cast(streams.in.get())) + if (auto stream = dynamic_cast(streams.in.get())) { /// NOTE Progress callback takes shared ownership of 'out'. stream->setProgressCallback([out] (const Progress & progress) { out->onProgress(progress); }); diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index 7b0b5710e1e..b79c693f04e 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -392,10 +392,9 @@ public: const ColumnSizeEstimator & column_sizes) : merge_entry(merge_entry_), merge_alg(merge_alg_) { - if (merge_alg == MergeAlgorithm::Horizontal) - average_elem_progress = 1.0 / num_total_rows; - else - average_elem_progress = column_sizes.keyColumnsProgress(1, num_total_rows); + average_elem_progress = (merge_alg == MergeAlgorithm::Horizontal) + ? 1.0 / num_total_rows + : column_sizes.keyColumnsProgress(1, num_total_rows); } MergeList::Entry & merge_entry; @@ -405,19 +404,11 @@ public: void operator() (const Progress & value) { ProfileEvents::increment(ProfileEvents::MergedUncompressedBytes, value.bytes); - merge_entry->bytes_read_uncompressed += value.bytes; - merge_entry->rows_with_key_columns_read += value.rows; + ProfileEvents::increment(ProfileEvents::MergedRows, value.rows); - if (merge_alg == MergeAlgorithm::Horizontal) - { - ProfileEvents::increment(ProfileEvents::MergedRows, value.rows); - merge_entry->rows_read += value.rows; - merge_entry->progress = average_elem_progress * merge_entry->rows_read; - } - else - { - merge_entry->progress = average_elem_progress * merge_entry->rows_with_key_columns_read; - } + merge_entry->bytes_read_uncompressed += value.bytes; + merge_entry->rows_read += value.rows; + merge_entry->progress = average_elem_progress * merge_entry->rows_read; }; }; @@ -597,9 +588,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart rows_written += block.rows(); to.write(block); - if (merge_alg == MergeAlgorithm::Horizontal) - merge_entry->rows_written = merged_stream->getProfileInfo().rows; - merge_entry->rows_with_key_columns_written = merged_stream->getProfileInfo().rows; + merge_entry->rows_written = merged_stream->getProfileInfo().rows; merge_entry->bytes_written_uncompressed = merged_stream->getProfileInfo().bytes; /// This update is unactual for VERTICAL algorithm sicne it requires more accurate per-column updates @@ -619,7 +608,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart /// Gather ordinary columns if (merge_alg == MergeAlgorithm::Vertical) { - size_t sum_input_rows_exact = merge_entry->rows_with_key_columns_read; + size_t sum_input_rows_exact = merge_entry->rows_read; merge_entry->columns_written = merging_column_names.size(); merge_entry->progress = column_sizes.keyColumnsProgress(sum_input_rows_exact, sum_input_rows_exact); diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp index 4bdce11acc0..0982c6f4ca6 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.cpp +++ b/dbms/src/Storages/System/StorageSystemMerges.cpp @@ -25,9 +25,7 @@ StorageSystemMerges::StorageSystemMerges(const std::string & name) { "rows_read", std::make_shared() }, { "bytes_written_uncompressed", std::make_shared() }, { "rows_written", std::make_shared() }, - { "columns_written", std::make_shared() }, - { "rows_with_key_columns_read", std::make_shared() }, - { "rows_with_key_columns_written", std::make_shared() } + { "columns_written", std::make_shared() } } { } @@ -62,8 +60,6 @@ BlockInputStreams StorageSystemMerges::read( ColumnWithTypeAndName col_bytes_written_uncompressed{std::make_shared(), std::make_shared(), "bytes_written_uncompressed"}; ColumnWithTypeAndName col_rows_written{std::make_shared(), std::make_shared(), "rows_written"}; ColumnWithTypeAndName col_columns_written{std::make_shared(), std::make_shared(), "columns_written"}; - ColumnWithTypeAndName col_rows_with_key_columns_read{std::make_shared(), std::make_shared(), "rows_with_key_columns_read"}; - ColumnWithTypeAndName col_rows_with_key_columns_written{std::make_shared(), std::make_shared(), "rows_with_key_columns_written"}; for (const auto & merge : context.getMergeList().get()) { @@ -80,8 +76,6 @@ BlockInputStreams StorageSystemMerges::read( col_bytes_written_uncompressed.column->insert(merge.bytes_written_uncompressed.load(std::memory_order_relaxed)); col_rows_written.column->insert(merge.rows_written.load(std::memory_order_relaxed)); col_columns_written.column->insert(merge.columns_written.load(std::memory_order_relaxed)); - col_rows_with_key_columns_read.column->insert(merge.rows_with_key_columns_read.load(std::memory_order_relaxed)); - col_rows_with_key_columns_written.column->insert(merge.rows_with_key_columns_written.load(std::memory_order_relaxed)); } Block block{ @@ -97,9 +91,7 @@ BlockInputStreams StorageSystemMerges::read( col_rows_read, col_bytes_written_uncompressed, col_rows_written, - col_columns_written, - col_rows_with_key_columns_read, - col_rows_with_key_columns_written + col_columns_written }; return BlockInputStreams{1, std::make_shared(block)}; diff --git a/dbms/src/Storages/System/StorageSystemProcesses.cpp b/dbms/src/Storages/System/StorageSystemProcesses.cpp index 6fc9f4fd8af..d397e8bfd7d 100644 --- a/dbms/src/Storages/System/StorageSystemProcesses.cpp +++ b/dbms/src/Storages/System/StorageSystemProcesses.cpp @@ -41,11 +41,13 @@ StorageSystemProcesses::StorageSystemProcesses(const std::string & name_) { "quota_key", std::make_shared() }, { "elapsed", std::make_shared() }, - { "rows_read", std::make_shared() }, - { "bytes_read", std::make_shared() }, - { "total_rows_approx", std::make_shared() }, + { "read_rows", std::make_shared() }, + { "read_bytes", std::make_shared() }, + { "total_rows_approx", std::make_shared() }, + { "written_rows", std::make_shared() }, + { "written_bytes", std::make_shared() }, { "memory_usage", std::make_shared() }, - { "query", std::make_shared() }, + { "query", std::make_shared() } } { } @@ -95,9 +97,11 @@ BlockInputStreams StorageSystemProcesses::read( block.unsafeGetByPosition(i++).column->insert(process.client_info.http_user_agent); block.unsafeGetByPosition(i++).column->insert(process.client_info.quota_key); block.unsafeGetByPosition(i++).column->insert(process.elapsed_seconds); - block.unsafeGetByPosition(i++).column->insert(process.rows); - block.unsafeGetByPosition(i++).column->insert(process.bytes); + block.unsafeGetByPosition(i++).column->insert(process.read_rows); + block.unsafeGetByPosition(i++).column->insert(process.read_bytes); block.unsafeGetByPosition(i++).column->insert(process.total_rows); + block.unsafeGetByPosition(i++).column->insert(process.written_rows); + block.unsafeGetByPosition(i++).column->insert(process.written_bytes); block.unsafeGetByPosition(i++).column->insert(process.memory_usage); block.unsafeGetByPosition(i++).column->insert(process.query); } diff --git a/dbms/tests/clickhouse-test b/dbms/tests/clickhouse-test index 661ebaca64e..024961d2e59 100755 --- a/dbms/tests/clickhouse-test +++ b/dbms/tests/clickhouse-test @@ -30,17 +30,17 @@ MSG_SKIPPED = OP_SQUARE_BRACKET + colored(" SKIPPED ", "cyan", attrs=['bold']) + def main(args): SERVER_DIED = False - - + + def is_data_present(): proc = Popen(args.client, stdin=PIPE, stdout=PIPE, stderr=PIPE) (stdout, stderr) = proc.communicate("EXISTS TABLE test.hits") if proc.returncode != 0: raise CalledProcessError(proc.returncode, args.client, stderr) - + return stdout.startswith('1') - - + + def dump_report(destination, suite, test_case, report): if destination is not None: destination_file = os.path.join(destination, suite, test_case + ".xml") @@ -53,23 +53,23 @@ def main(args): report_suite.append(report) report_root.append(report_suite) report_file.write(et.tostring(report_root, encoding = "UTF-8", xml_declaration=True, pretty_print=True)) - - + + if args.zookeeper is None: try: check_call(['grep', '-q', ' 0: print(colored("\nHaving {0} errors!".format(failures_total), "red", attrs=["bold"])) sys.exit(1) else: print(colored("\nAll tests passed.", "green", attrs=["bold"])) sys.exit(0) - + if __name__ == '__main__': parser = ArgumentParser(description = 'ClickHouse functional tests') @@ -214,11 +214,11 @@ if __name__ == '__main__': parser.add_argument('-o', '--output', help = 'Output xUnit compliant test report directory') parser.add_argument('-t', '--timeout', type = int, default = 600, help = 'Timeout for each test case in seconds') parser.add_argument('test', nargs = '?', help = 'Optional test case name regex') - + group = parser.add_mutually_exclusive_group(required = False) group.add_argument('--zookeeper', action = 'store_true', default = None, dest = 'zookeeper', help = 'Run zookeeper related tests') group.add_argument('--no-zookeeper', action = 'store_false', default = None, dest = 'zookeeper', help = 'Do not run zookeeper related tests') - + args = parser.parse_args() - + main(args) From 65401561000d7ce5113836b554ec5baaac6d3537 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 7 Dec 2016 17:58:31 +0300 Subject: [PATCH 2/9] Removed redundant code from CMakeLists. --- cmake/dbms_glob_sources.cmake | 13 + contrib/libre2/CMakeLists.txt | 2 +- contrib/libzstd/CMakeLists.txt | 2 +- dbms/CMakeLists.txt | 1015 +------------------------------- 4 files changed, 34 insertions(+), 998 deletions(-) create mode 100644 cmake/dbms_glob_sources.cmake diff --git a/cmake/dbms_glob_sources.cmake b/cmake/dbms_glob_sources.cmake new file mode 100644 index 00000000000..491c7a604d1 --- /dev/null +++ b/cmake/dbms_glob_sources.cmake @@ -0,0 +1,13 @@ +macro(add_glob cur_list) + file(GLOB __tmp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${ARGN}) + list(APPEND ${cur_list} ${__tmp}) +endmacro() + +macro(add_headers_and_sources prefix common_path) + add_glob(${prefix}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include/DB/${common_path}/*.h include/DB/${common_path}/*.inl) + add_glob(${prefix}_sources src/${common_path}/*.cpp src/${common_path}/*.h) +endmacro() + +macro(add_headers_only prefix common_path) + add_glob(${prefix}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include/DB/${common_path}/*.h src/${common_path}/*.h) +endmacro() diff --git a/contrib/libre2/CMakeLists.txt b/contrib/libre2/CMakeLists.txt index 93bbfee81b4..ecf6855cac6 100644 --- a/contrib/libre2/CMakeLists.txt +++ b/contrib/libre2/CMakeLists.txt @@ -43,7 +43,7 @@ add_library (re2_st ${re2_headers}) set_target_properties (re2_st PROPERTIES COMPILE_DEFINITIONS "NO_THREADS;re2=re2_st") -message ("Creating headers for re2_st library.") +message (STATUS "Creating headers for re2_st library.") file (MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/re2_st) foreach (FILENAME filtered_re2.h re2.h set.h stringpiece.h variadic_function.h) file (READ ${CMAKE_CURRENT_SOURCE_DIR}/re2/${FILENAME} CONTENT) diff --git a/contrib/libzstd/CMakeLists.txt b/contrib/libzstd/CMakeLists.txt index 136bcff3f19..117bbfa809b 100644 --- a/contrib/libzstd/CMakeLists.txt +++ b/contrib/libzstd/CMakeLists.txt @@ -47,7 +47,7 @@ FILE(READ ${LIBRARY_DIR}/zstd.h HEADER_CONTENT) # Parse version GetLibraryVersion("${HEADER_CONTENT}" LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE) -MESSAGE("ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") +MESSAGE(STATUS "ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}") SET(Sources ${LIBRARY_DIR}/common/entropy_common.c diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 10985e8ca3b..282fb889bcd 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -3,6 +3,7 @@ include_directories (/usr/include/mysql) add_subdirectory (src) + if (ENABLE_MONGODB) set (LINK_MONGOCLIENT libmongoclient.a ${OPENSSL_LIBRARIES} ${Boost_THREAD_LIBRARY}) set (MONGODB_FILES @@ -13,43 +14,21 @@ if (ENABLE_MONGODB) add_definitions(-D ENABLE_MONGODB) endif () + if (NOT ENABLE_LIBTCMALLOC) add_definitions(-D NO_TCMALLOC) endif () + add_library(string_utils include/DB/Common/StringUtils.h src/Common/StringUtils.cpp) -set(AIO_H_FILES - include/DB/Common/AIO.h - include/DB/IO/WriteBufferAIO.h - include/DB/IO/ReadBufferAIO.h) -set (AIO_CPP_FILES - src/IO/ReadBufferAIO.cpp - src/IO/WriteBufferAIO.cpp) -set (APPLE_ICONV_LIB) - -if (APPLE) - set(APPLE_ICONV_LIB iconv) -endif() - -macro(add_glob cur_list) - file(GLOB __tmp RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} ${ARGN}) - list(APPEND ${cur_list} ${__tmp}) -endmacro() - -macro(add_headers_and_sources prefix common_path) - add_glob(${prefix}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include/DB/${common_path}/*.h include/DB/${common_path}/*.inl) - add_glob(${prefix}_sources src/${common_path}/*.cpp src/${common_path}/*.h) -endmacro() - -macro(add_headers_only prefix common_path) - add_glob(${prefix}_headers RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} include/DB/${common_path}/*.h src/${common_path}/*.h) -endmacro() set(dbms_headers) set(dbms_sources) + +include(../cmake/dbms_glob_sources.cmake) add_headers_and_sources(dbms Functions) add_headers_and_sources(dbms Functions/Conditional) add_headers_and_sources(dbms TableFunctions) @@ -84,979 +63,17 @@ list(REMOVE_ITEM dbms_sources src/Common/StringUtils.cpp) if (APPLE) - list(REMOVE_ITEM dbms_headers ${AIO_H_FILES}) - list(REMOVE_ITEM dbms_sources ${AIO_CPP_FILES}) + list(REMOVE_ITEM dbms_headers + include/DB/Common/AIO.h + include/DB/IO/WriteBufferAIO.h + include/DB/IO/ReadBufferAIO.h) + list(REMOVE_ITEM dbms_sources + src/IO/ReadBufferAIO.cpp + src/IO/WriteBufferAIO.cpp) endif() add_library(dbms ${dbms_headers} ${dbms_sources}) -set (dbms_all_sources_old - src/Server/InterserverIOHTTPHandler.h - src/Server/Server.h - src/Server/TCPHandler.h - src/Server/HTTPHandler.h - src/Server/MetricsTransmitter.h - src/Server/ConfigReloader.h - src/Server/StatusFile.h - src/Server/ReplicasStatusHandler.h - src/Client/InterruptListener.h - include/DB/Functions/FunctionsLogical.h - include/DB/Functions/FunctionsReinterpret.h - include/DB/Functions/FunctionFactory.h - include/DB/Functions/FunctionsArray.h - include/DB/Functions/FunctionsDictionaries.h - include/DB/Functions/FunctionsStringArray.h - include/DB/Functions/FunctionsURL.h - include/DB/Functions/FunctionsRandom.h - include/DB/Functions/FunctionsString.h - include/DB/Functions/FunctionsRound.h - include/DB/Functions/FunctionsTransform.h - include/DB/Functions/Conditional/CondException.h - include/DB/Functions/Conditional/common.h - include/DB/Functions/Conditional/getArrayType.h - include/DB/Functions/Conditional/ArgsInfo.h - include/DB/Functions/Conditional/CondSource.h - include/DB/Functions/Conditional/NumericPerformer.h - include/DB/Functions/Conditional/NumericEvaluator.h - include/DB/Functions/Conditional/StringEvaluator.h - include/DB/Functions/Conditional/ArrayEvaluator.h - include/DB/Functions/Conditional/StringArrayEvaluator.h - include/DB/Functions/FunctionsConditional.h - include/DB/Functions/FunctionsArithmetic.h - include/DB/Functions/FunctionsConversion.h - include/DB/Functions/FunctionsCoding.h - include/DB/Functions/FunctionsStringSearch.h - include/DB/Functions/FunctionsVisitParam.h - include/DB/Functions/FunctionsHigherOrder.h - include/DB/Functions/FunctionsComparison.h - include/DB/Functions/FunctionsHashing.h - include/DB/Functions/FunctionsMath.h - include/DB/Functions/FunctionsGeo.h - include/DB/Functions/FunctionsMiscellaneous.h - include/DB/Functions/FunctionsDateTime.h - include/DB/Functions/IFunction.h - include/DB/Functions/FunctionsFormatting.h - include/DB/Functions/NumberTraits.h - include/DB/Functions/DataTypeTraits.h - include/DB/Functions/EnrichedDataTypePtr.h - include/DB/Functions/ObjectPool.h - include/DB/TableFunctions/TableFunctionRemote.h - include/DB/TableFunctions/TableFunctionFactory.h - include/DB/TableFunctions/TableFunctionMerge.h - include/DB/TableFunctions/TableFunctionShardByHash.h - include/DB/TableFunctions/ITableFunction.h - include/DB/TableFunctions/getStructureOfRemoteTable.h - include/DB/Parsers/ASTSetQuery.h - include/DB/Parsers/ASTRenameQuery.h - include/DB/Parsers/ParserQueryWithOutput.h - include/DB/Parsers/ParserOptimizeQuery.h - include/DB/Parsers/ParserSetQuery.h - include/DB/Parsers/ASTCreateQuery.h - include/DB/Parsers/ASTExpressionList.h - include/DB/Parsers/ASTQueryWithOutput.h - include/DB/Parsers/ParserSelectQuery.h - include/DB/Parsers/ParserUseQuery.h - include/DB/Parsers/ASTShowTablesQuery.h - include/DB/Parsers/ASTFunction.h - include/DB/Parsers/ASTSelectQuery.h - include/DB/Parsers/ParserCreateQuery.h - include/DB/Parsers/ASTLiteral.h - include/DB/Parsers/ASTInsertQuery.h - include/DB/Parsers/IParserBase.h - include/DB/Parsers/ASTOptimizeQuery.h - include/DB/Parsers/ASTSubquery.h - include/DB/Parsers/ASTUseQuery.h - include/DB/Parsers/ASTIdentifier.h - include/DB/Parsers/ParserTablePropertiesQuery.h - include/DB/Parsers/ParserCheckQuery.h - include/DB/Parsers/ParserRenameQuery.h - include/DB/Parsers/ParserInsertQuery.h - include/DB/Parsers/ParserAlterQuery.h - include/DB/Parsers/ASTWithAlias.h - include/DB/Parsers/ASTOrderByElement.h - include/DB/Parsers/queryToString.h - include/DB/Parsers/ASTShowProcesslistQuery.h - include/DB/Parsers/IAST.h - include/DB/Parsers/ASTNameTypePair.h - include/DB/Parsers/ASTSet.h - include/DB/Parsers/ExpressionListParsers.h - include/DB/Parsers/ParserQuery.h - include/DB/Parsers/TablePropertiesQueriesASTs.h - include/DB/Parsers/ParserDropQuery.h - include/DB/Parsers/StringRange.h - include/DB/Parsers/ASTAlterQuery.h - include/DB/Parsers/CommonParsers.h - include/DB/Parsers/formatAST.h - include/DB/Parsers/ASTDropQuery.h - include/DB/Parsers/ASTAsterisk.h - include/DB/Parsers/ASTCheckQuery.h - include/DB/Parsers/ASTColumnDeclaration.h - include/DB/Parsers/ASTEnumElement.h - include/DB/Parsers/parseQuery.h - include/DB/Parsers/ParserShowProcesslistQuery.h - include/DB/Parsers/ParserShowTablesQuery.h - include/DB/Parsers/ParserEnumElement.h - include/DB/Parsers/ExpressionElementParsers.h - include/DB/Parsers/ASTQueryWithTableAndOutput.h - include/DB/Parsers/IParser.h - include/DB/Parsers/ASTSampleRatio.h - include/DB/Parsers/ParserSampleRatio.h - include/DB/Parsers/ParserCase.h - include/DB/Parsers/ASTTablesInSelectQuery.h - include/DB/Parsers/ParserTablesInSelectQuery.h - include/DB/AggregateFunctions/AggregateFunctionMerge.h - include/DB/AggregateFunctions/AggregateFunctionUniqUpTo.h - include/DB/AggregateFunctions/AggregateFunctionIf.h - include/DB/AggregateFunctions/AggregateFunctionsArgMinMax.h - include/DB/AggregateFunctions/AggregateFunctionGroupArray.h - include/DB/AggregateFunctions/AggregateFunctionQuantileDeterministic.h - include/DB/AggregateFunctions/AggregateFunctionUniq.h - include/DB/AggregateFunctions/UniqVariadicHash.h - include/DB/AggregateFunctions/AggregateFunctionsStatistics.h - include/DB/AggregateFunctions/AggregateFunctionAvg.h - include/DB/AggregateFunctions/AggregateFunctionQuantile.h - include/DB/AggregateFunctions/AggregateFunctionQuantileExact.h - include/DB/AggregateFunctions/AggregateFunctionQuantileExactWeighted.h - include/DB/AggregateFunctions/AggregateFunctionQuantileTiming.h - include/DB/AggregateFunctions/AggregateFunctionQuantileTDigest.h - include/DB/AggregateFunctions/IBinaryAggregateFunction.h - include/DB/AggregateFunctions/AggregateFunctionState.h - include/DB/AggregateFunctions/AggregateFunctionSequenceMatch.h - include/DB/AggregateFunctions/IUnaryAggregateFunction.h - include/DB/AggregateFunctions/IAggregateFunction.h - include/DB/AggregateFunctions/AggregateFunctionsMinMaxAny.h - include/DB/AggregateFunctions/INullaryAggregateFunction.h - include/DB/AggregateFunctions/AggregateFunctionGroupUniqArray.h - include/DB/AggregateFunctions/AggregateFunctionArray.h - include/DB/AggregateFunctions/AggregateFunctionCount.h - include/DB/AggregateFunctions/AggregateFunctionSum.h - include/DB/AggregateFunctions/AggregateFunctionFactory.h - include/DB/AggregateFunctions/Helpers.h - include/DB/AggregateFunctions/HelpersMinMaxAny.h - include/DB/AggregateFunctions/UniqCombinedBiasData.h - include/DB/AggregateFunctions/ReservoirSampler.h - include/DB/AggregateFunctions/ReservoirSamplerDeterministic.h - include/DB/AggregateFunctions/UniquesHashSet.h - include/DB/AggregateFunctions/QuantilesCommon.h - include/DB/Core/BlockInfo.h - include/DB/Core/SortDescription.h - include/DB/Core/Protocol.h - include/DB/Core/Row.h - include/DB/Core/ColumnNumbers.h - include/DB/Core/ColumnsWithTypeAndName.h - include/DB/Core/QueryProcessingStage.h - include/DB/Core/Field.h - include/DB/Core/FieldVisitors.h - include/DB/Core/ColumnWithTypeAndName.h - include/DB/Core/NamesAndTypes.h - include/DB/Core/Names.h - include/DB/Core/Block.h - include/DB/Core/Defines.h - include/DB/Core/Progress.h - include/DB/Core/toField.h - include/DB/Core/Types.h - include/DB/Core/StringRef.h - include/DB/DataStreams/NullAndDoCopyBlockInputStream.h - include/DB/DataStreams/TabSeparatedRowInputStream.h - include/DB/DataStreams/OneBlockInputStream.h - include/DB/DataStreams/BlocksListBlockInputStream.h - include/DB/DataStreams/IRowInputStream.h - include/DB/DataStreams/TotalsHavingBlockInputStream.h - include/DB/DataStreams/BinaryRowOutputStream.h - include/DB/DataStreams/IRowOutputStream.h - include/DB/DataStreams/ParallelAggregatingBlockInputStream.h - include/DB/DataStreams/NullBlockInputStream.h - include/DB/DataStreams/PrettyBlockOutputStream.h - include/DB/DataStreams/BlockStreamProfileInfo.h - include/DB/DataStreams/ConcatBlockInputStream.h - include/DB/DataStreams/ProhibitColumnsBlockOutputStream.h - include/DB/DataStreams/UnionBlockInputStream.h - include/DB/DataStreams/EmptyBlockOutputStream.h - include/DB/DataStreams/PushingToViewsBlockOutputStream.h - include/DB/DataStreams/glueBlockInputStreams.h - include/DB/DataStreams/CollapsingSortedBlockInputStream.h - include/DB/DataStreams/IBlockOutputStream.h - include/DB/DataStreams/InputStreamFromASTInsertQuery.h - include/DB/DataStreams/AsynchronousBlockInputStream.h - include/DB/DataStreams/MaterializingBlockInputStream.h - include/DB/DataStreams/ParallelInputsProcessor.h - include/DB/DataStreams/RemoveColumnsBlockInputStream.h - include/DB/DataStreams/RemoteBlockInputStream.h - include/DB/DataStreams/MaterializingBlockOutputStream.h - include/DB/DataStreams/ForkBlockInputStreams.h - include/DB/DataStreams/CollapsingFinalBlockInputStream.h - include/DB/DataStreams/TabSeparatedRawRowOutputStream.h - include/DB/DataStreams/FilterBlockInputStream.h - include/DB/DataStreams/ValuesRowOutputStream.h - include/DB/DataStreams/NativeBlockInputStream.h - include/DB/DataStreams/FormatFactory.h - include/DB/DataStreams/MergingAggregatedBlockInputStream.h - include/DB/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.h - include/DB/DataStreams/SummingSortedBlockInputStream.h - include/DB/DataStreams/ReplacingSortedBlockInputStream.h - include/DB/DataStreams/GraphiteRollupSortedBlockInputStream.h - include/DB/DataStreams/AddingConstColumnBlockInputStream.h - include/DB/DataStreams/DistinctBlockInputStream.h - include/DB/DataStreams/BlockOutputStreamFromRowOutputStream.h - include/DB/DataStreams/copyData.h - include/DB/DataStreams/IBlockInputStream.h - include/DB/DataStreams/TabSeparatedRowOutputStream.h - include/DB/DataStreams/ValuesRowInputStream.h - include/DB/DataStreams/BlockIO.h - include/DB/DataStreams/QueueBlockIOStream.h - include/DB/DataStreams/VerticalRowOutputStream.h - include/DB/DataStreams/AggregatingSortedBlockInputStream.h - include/DB/DataStreams/TabSeparatedBlockOutputStream.h - include/DB/DataStreams/narrowBlockInputStreams.h - include/DB/DataStreams/IProfilingBlockInputStream.h - include/DB/DataStreams/NativeBlockOutputStream.h - include/DB/DataStreams/RemoteBlockOutputStream.h - include/DB/DataStreams/MergingSortedBlockInputStream.h - include/DB/DataStreams/JSONRowOutputStream.h - include/DB/DataStreams/JSONEachRowRowOutputStream.h - include/DB/DataStreams/JSONEachRowRowInputStream.h - include/DB/DataStreams/XMLRowOutputStream.h - include/DB/DataStreams/TSKVRowOutputStream.h - include/DB/DataStreams/TSKVRowInputStream.h - include/DB/DataStreams/ODBCDriverBlockOutputStream.h - include/DB/DataStreams/MergeSortingBlockInputStream.h - include/DB/DataStreams/ExpressionBlockInputStream.h - include/DB/DataStreams/BinaryRowInputStream.h - include/DB/DataStreams/PrettyCompactMonoBlockOutputStream.h - include/DB/DataStreams/AggregatingBlockInputStream.h - include/DB/DataStreams/AddingDefaultBlockInputStream.h - include/DB/DataStreams/PrettySpaceBlockOutputStream.h - include/DB/DataStreams/PrettyCompactBlockOutputStream.h - include/DB/DataStreams/CreatingSetsBlockInputStream.h - include/DB/DataStreams/LimitBlockInputStream.h - include/DB/DataStreams/LazyBlockInputStream.h - include/DB/DataStreams/BlockInputStreamFromRowInputStream.h - include/DB/DataStreams/NullBlockOutputStream.h - include/DB/DataStreams/AddingDefaultBlockOutputStream.h - include/DB/DataStreams/JSONCompactRowOutputStream.h - include/DB/DataStreams/PartialSortingBlockInputStream.h - include/DB/DataStreams/MarkInCompressedFile.h - include/DB/DataStreams/CSVRowOutputStream.h - include/DB/DataStreams/CSVRowInputStream.h - include/DB/DataStreams/verbosePrintString.h - include/DB/DataStreams/SquashingTransform.h - include/DB/DataStreams/SquashingBlockInputStream.h - include/DB/DataStreams/SquashingBlockOutputStream.h - include/DB/DataStreams/ColumnGathererStream.h - include/DB/DataTypes/IDataType.h - include/DB/DataTypes/IDataTypeDummy.h - include/DB/DataTypes/DataTypeSet.h - include/DB/DataTypes/DataTypeFixedString.h - include/DB/DataTypes/DataTypeEnum.h - include/DB/DataTypes/DataTypeArray.h - include/DB/DataTypes/DataTypeString.h - include/DB/DataTypes/IDataTypeNumber.h - include/DB/DataTypes/DataTypeFactory.h - include/DB/DataTypes/DataTypeDate.h - include/DB/DataTypes/FieldToDataType.h - include/DB/DataTypes/DataTypeAggregateFunction.h - include/DB/DataTypes/IDataTypeNumberFixed.h - include/DB/DataTypes/DataTypeNested.h - include/DB/DataTypes/DataTypeTuple.h - include/DB/DataTypes/DataTypeExpression.h - include/DB/DataTypes/DataTypeDateTime.h - include/DB/DataTypes/DataTypesNumberFixed.h - include/DB/Dictionaries/FileDictionarySource.h - include/DB/Dictionaries/DictionarySourceFactory.h - include/DB/Dictionaries/DictionaryStructure.h - include/DB/Dictionaries/ClickHouseDictionarySource.h - include/DB/Dictionaries/DictionaryFactory.h - include/DB/Dictionaries/IDictionary.h - include/DB/Dictionaries/MySQLBlockInputStream.h - include/DB/Dictionaries/MySQLDictionarySource.h - include/DB/Dictionaries/ODBCBlockInputStream.h - include/DB/Dictionaries/ODBCDictionarySource.h - include/DB/Dictionaries/HashedDictionary.h - include/DB/Dictionaries/FlatDictionary.h - include/DB/Dictionaries/ComplexKeyHashedDictionary.h - include/DB/Dictionaries/ComplexKeyCacheDictionary.h - include/DB/Dictionaries/IDictionarySource.h - include/DB/Dictionaries/OwningBlockInputStream.h - include/DB/Dictionaries/CacheDictionary.h - include/DB/Dictionaries/MySQLDictionarySource.h - include/DB/Dictionaries/ExecutableDictionarySource.h - include/DB/Dictionaries/HTTPDictionarySource.h - include/DB/Dictionaries/RangeHashedDictionary.h - include/DB/Dictionaries/writeParenthesisedString.h - include/DB/Dictionaries/Embedded/RegionsHierarchy.h - include/DB/Dictionaries/Embedded/RegionsHierarchies.h - include/DB/Dictionaries/Embedded/RegionsNames.h - include/DB/Dictionaries/Embedded/TechDataHierarchy.h - include/DB/Dictionaries/ExternalResultDescription.h - include/DB/Dictionaries/ExternalQueryBuilder.h - include/DB/Interpreters/InterpreterAlterQuery.h - include/DB/Interpreters/AggregationCommon.h - include/DB/Interpreters/ProcessList.h - include/DB/Interpreters/AggregateDescription.h - include/DB/Interpreters/Cluster.h - include/DB/Interpreters/loadMetadata.h - include/DB/Interpreters/ExternalDictionaries.h - include/DB/Interpreters/sortBlock.h - include/DB/Interpreters/SettingsCommon.h - include/DB/Interpreters/SpecializedAggregator.h - include/DB/Interpreters/InterpreterShowTablesQuery.h - include/DB/Interpreters/InterpreterUseQuery.h - include/DB/Interpreters/Context.h - include/DB/Interpreters/IInterpreter.h - include/DB/Interpreters/InterpreterFactory.h - include/DB/Interpreters/InterpreterOptimizeQuery.h - include/DB/Interpreters/InterpreterSetQuery.h - include/DB/Interpreters/Dictionaries.h - include/DB/Interpreters/Settings.h - include/DB/Interpreters/Set.h - include/DB/Interpreters/InterpreterDescribeQuery.h - include/DB/Interpreters/InterpreterCreateQuery.h - include/DB/Interpreters/InterpreterShowCreateQuery.h - include/DB/Interpreters/Users.h - include/DB/Interpreters/Compiler.h - include/DB/Interpreters/InJoinSubqueriesPreprocessor.h - include/DB/Interpreters/LogicalExpressionsOptimizer.h - include/DB/Interpreters/InterserverIOHandler.h - include/DB/Interpreters/evaluateMissingDefaults.h - include/DB/Interpreters/evaluateConstantExpression.h - include/DB/Interpreters/convertFieldToType.h - include/DB/Interpreters/ExpressionActions.h - include/DB/Interpreters/InterpreterRenameQuery.h - include/DB/Interpreters/Join.h - include/DB/Interpreters/Quota.h - include/DB/Interpreters/InterpreterExistsQuery.h - include/DB/Interpreters/InterpreterCheckQuery.h - include/DB/Interpreters/InterpreterInsertQuery.h - include/DB/Interpreters/Limits.h - include/DB/Interpreters/InterpreterDropQuery.h - include/DB/Interpreters/InterpreterShowProcesslistQuery.h - include/DB/Interpreters/executeQuery.h - include/DB/Interpreters/Aggregator.h - include/DB/Interpreters/ExpressionAnalyzer.h - include/DB/Interpreters/InterpreterSelectQuery.h - include/DB/Interpreters/QueryPriorities.h - include/DB/Interpreters/SystemLog.h - include/DB/Interpreters/QueryLog.h - include/DB/Interpreters/ClientInfo.h - include/DB/Interpreters/getClusterName.h - include/DB/Interpreters/ClusterProxy/IQueryConstructor.h - include/DB/Interpreters/ClusterProxy/SelectQueryConstructor.h - include/DB/Interpreters/ClusterProxy/DescribeQueryConstructor.h - include/DB/Interpreters/ClusterProxy/AlterQueryConstructor.h - include/DB/Interpreters/ClusterProxy/Query.h - include/DB/Interpreters/AsynchronousMetrics.h - include/DB/Common/Allocator.h - include/DB/Common/CombinedCardinalityEstimator.h - include/DB/Common/ExternalTable.h - include/DB/Common/PODArray.h - include/DB/Common/Arena.h - include/DB/Common/ArenaWithFreeLists.h - include/DB/Common/SmallObjectPool.h - include/DB/Common/HashTable/Hash.h - include/DB/Common/HashTable/TwoLevelHashTable.h - include/DB/Common/HashTable/ClearableHashSet.h - include/DB/Common/HashTable/ClearableHashMap.h - include/DB/Common/HashTable/TwoLevelHashMap.h - include/DB/Common/HashTable/HashTable.h - include/DB/Common/HashTable/HashTableAllocator.h - include/DB/Common/HashTable/HashSet.h - include/DB/Common/HashTable/SmallTable.h - include/DB/Common/HashTable/HashMap.h - include/DB/Common/SipHash.h - include/DB/Common/getFQDNOrHostName.h - include/DB/Common/LRUCache.h - include/DB/Common/Throttler.h - include/DB/Common/FileChecker.h - include/DB/Common/ConcurrentBoundedQueue.h - include/DB/Common/localBackup.h - include/DB/Common/UnicodeBar.h - include/DB/Common/Collator.h - include/DB/Common/isLocalAddress.h - include/DB/Common/AutoArray.h - include/DB/Common/UInt128.h - include/DB/Common/UTF8Helpers.h - include/DB/Common/Macros.h - include/DB/Common/typeid_cast.h - include/DB/Common/ProfileEvents.h - include/DB/Common/CurrentMetrics.h - include/DB/Common/formatReadable.h - include/DB/Common/VirtualColumnUtils.h - include/DB/Common/HyperLogLogWithSmallSetOptimization.h - include/DB/Common/Volnitsky.h - include/DB/Common/MemoryTracker.h - include/DB/Common/escapeForFileName.h - include/DB/Common/setThreadName.h - include/DB/Common/StringSearcher.h - include/DB/Common/SimpleCache.h - include/DB/Common/Stopwatch.h - include/DB/Common/CounterInFile.h - include/DB/Common/HyperLogLogCounter.h - include/DB/Common/Increment.h - include/DB/Common/OptimizedRegularExpression.h - include/DB/Common/OptimizedRegularExpression.inl - include/DB/Common/StackTrace.h - include/DB/Common/ConfigProcessor.h - include/DB/Common/Exception.h - include/DB/Common/HTMLForm.h - include/DB/Common/NetException.h - include/DB/Common/PoolBase.h - include/DB/Common/PoolWithFailoverBase.h - include/DB/Common/RadixSort.h - include/DB/Common/ShellCommand.h - include/DB/Common/getNumberOfPhysicalCPUCores.h - include/DB/Common/BitHelpers.h - include/DB/Common/BlockFilterCreator.h - include/DB/Common/randomSeed.h - include/DB/Common/unaligned.h - include/DB/Common/ThreadPool.h - include/DB/IO/CompressedStream.h - include/DB/IO/ReadBufferFromFileDescriptor.h - include/DB/IO/CompressedWriteBuffer.h - include/DB/IO/CompressedReadBuffer.h - include/DB/IO/ReadBufferFromFile.h - include/DB/IO/CompressedReadBufferFromFile.h - include/DB/IO/WriteBufferFromString.h - include/DB/IO/DoubleConverter.h - include/DB/IO/BufferWithOwnMemory.h - include/DB/IO/WriteHelpers.h - include/DB/IO/ConcatReadBuffer.h - include/DB/IO/WriteBufferFromOStream.h - include/DB/IO/ReadBuffer.h - include/DB/IO/WriteBufferFromVector.h - include/DB/IO/ReadBufferFromPocoSocket.h - include/DB/IO/RemoteReadBuffer.h - include/DB/IO/HashingWriteBuffer.h - include/DB/IO/ReadBufferFromString.h - include/DB/IO/CompressedReadBufferBase.h - include/DB/IO/HashingReadBuffer.h - include/DB/IO/Operators.h - include/DB/IO/WriteBufferFromFile.h - include/DB/IO/RemoteWriteBuffer.h - include/DB/IO/VarInt.h - include/DB/IO/copyData.h - include/DB/IO/LimitReadBuffer.h - include/DB/IO/HexWriteBuffer.h - include/DB/IO/CachedCompressedReadBuffer.h - include/DB/IO/BufferBase.h - include/DB/IO/AsynchronousWriteBuffer.h - ${AIO_H_FILES} - include/DB/IO/ReadHelpers.h - include/DB/IO/WriteBufferFromPocoSocket.h - include/DB/IO/ReadBufferFromHTTP.h - include/DB/IO/ReadWriteBufferFromHTTP.h - include/DB/IO/WriteBufferFromHTTPServerResponse.h - include/DB/IO/createReadBufferFromFileBase.h - include/DB/IO/WriteIntText.h - include/DB/IO/WriteBufferFromFileDescriptor.h - include/DB/IO/WriteBufferValidUTF8.h - include/DB/IO/WriteBuffer.h - include/DB/IO/WriteBufferFromFileBase.h - include/DB/IO/ReadBufferFromIStream.h - include/DB/IO/UncompressedCache.h - include/DB/IO/InterserverWriteBuffer.h - include/DB/Columns/ColumnTuple.h - include/DB/Columns/ColumnVector.h - include/DB/Columns/IColumnDummy.h - include/DB/Columns/ColumnAggregateFunction.h - include/DB/Columns/ColumnString.h - include/DB/Columns/ColumnExpression.h - include/DB/Columns/ColumnFixedString.h - include/DB/Columns/ColumnConst.h - include/DB/Columns/IColumn.h - include/DB/Columns/ColumnsCommon.h - include/DB/Columns/ColumnSet.h - include/DB/Columns/ColumnArray.h - include/DB/Columns/ColumnsNumber.h - include/DB/Client/MultiplexedConnections.h - include/DB/Client/ConnectionPoolWithFailover.h - include/DB/Client/Connection.h - include/DB/Client/ConnectionPool.h - include/DB/Storages/StorageBuffer.h - include/DB/Storages/StorageMemory.h - include/DB/Storages/System/StorageSystemProcesses.h - include/DB/Storages/System/StorageSystemDictionaries.h - include/DB/Storages/StorageDistributed.h - include/DB/Storages/StorageJoin.h - include/DB/Storages/System/StorageSystemDatabases.h - include/DB/Storages/Distributed/DirectoryMonitor.h - include/DB/Storages/Distributed/DistributedBlockOutputStream.h - include/DB/Storages/System/StorageSystemClusters.h - include/DB/Storages/System/StorageSystemZooKeeper.h - include/DB/Storages/StorageMerge.h - include/DB/Storages/System/StorageSystemOne.h - include/DB/Storages/AlterCommands.h - include/DB/Storages/System/StorageSystemFunctions.h - include/DB/Storages/StorageLog.h - include/DB/Storages/StorageFile.h - include/DB/Storages/IStorage.h - include/DB/Storages/ColumnsDescription.h - include/DB/Storages/System/StorageSystemMerges.h - include/DB/Storages/VirtualColumnFactory.h - include/DB/Storages/StorageReplicatedMergeTree.h - include/DB/Storages/StorageFactory.h - include/DB/Storages/MarkCache.h - include/DB/Storages/System/StorageSystemReplicas.h - include/DB/Storages/System/StorageSystemReplicationQueue.h - include/DB/Storages/System/StorageSystemNumbers.h - include/DB/Storages/System/StorageSystemParts.h - include/DB/Storages/StorageSet.h - include/DB/Storages/CompressionMethodSelector.h - include/DB/Storages/ColumnDefault.h - include/DB/Storages/ITableDeclaration.h - include/DB/Storages/StorageView.h - include/DB/Storages/System/StorageSystemColumns.h - include/DB/Storages/StorageMergeTree.h - include/DB/Storages/StorageMaterializedView.h - include/DB/Storages/StorageTinyLog.h - include/DB/Storages/StorageStripeLog.h - include/DB/Storages/System/StorageSystemEvents.h - include/DB/Storages/System/StorageSystemMetrics.h - include/DB/Storages/System/StorageSystemAsynchronousMetrics.h - include/DB/Storages/System/StorageSystemTables.h - include/DB/Storages/MergeTree/MarkRange.h - include/DB/Storages/MergeTree/MergeTreeDataMerger.h - include/DB/Storages/MergeTree/MergedBlockOutputStream.h - include/DB/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h - include/DB/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.h - include/DB/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h - include/DB/Storages/MergeTree/BoolMask.h - include/DB/Storages/MergeTree/BackgroundProcessingPool.h - include/DB/Storages/MergeTree/MergeTreeDataSelectExecutor.h - include/DB/Storages/MergeTree/ActiveDataPartSet.h - include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h - include/DB/Storages/MergeTree/RangesInDataPart.h - include/DB/Storages/MergeTree/MergeTreeReadPool.h - include/DB/Storages/MergeTree/ReplicatedMergeTreeLogEntry.h - include/DB/Storages/MergeTree/ReplicatedMergeTreeQueue.h - include/DB/Storages/MergeTree/MergeList.h - include/DB/Storages/MergeTree/PKCondition.h - include/DB/Storages/MergeTree/MergeTreeDataWriter.h - include/DB/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.h - include/DB/Storages/MergeTree/AbandonableLockInZooKeeper.h - include/DB/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.h - include/DB/Storages/MergeTree/MergeTreePartChecker.h - include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h - include/DB/Storages/MergeTree/MergeTreeBlockOutputStream.h - include/DB/Storages/MergeTree/MergeTreeThreadBlockInputStream.h - include/DB/Storages/MergeTree/DiskSpaceMonitor.h - include/DB/Storages/MergeTree/RemoteDiskSpaceMonitor.h - include/DB/Storages/MergeTree/MergeTreeReader.h - include/DB/Storages/MergeTree/DataPartsExchange.h - include/DB/Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h - include/DB/Storages/MergeTree/MergeTreeData.h - include/DB/Storages/MergeTree/MergeTreeDataPart.h - include/DB/Storages/MergeTree/MergeTreeSettings.h - include/DB/Storages/MergeTree/ReshardingJob.h - include/DB/Storages/MergeTree/ReshardingWorker.h - include/DB/Storages/MergeTree/MergeTreeSharder.h - include/DB/Storages/MergeTree/ShardedPartitionUploader.h - include/DB/Storages/MergeTree/RemoteQueryExecutor.h - include/DB/Storages/MergeTree/RemotePartChecker.h - include/DB/Storages/MergeTree/MergeSelector.h - include/DB/Storages/MergeTree/SimpleMergeSelector.h - include/DB/Storages/MergeTree/LevelMergeSelector.h - include/DB/Storages/MergeTree/AllMergeSelector.h - include/DB/Storages/StorageNull.h - include/DB/Storages/System/StorageSystemSettings.h - include/DB/Databases/IDatabase.h - include/DB/Databases/DatabaseMemory.h - include/DB/Databases/DatabaseOrdinary.h - include/DB/Databases/DatabaseCloud.h - include/DB/Databases/DatabaseFactory.h - include/DB/Databases/DatabasesCommon.h - - src/Common/ProfileEvents.cpp - src/Common/CurrentMetrics.cpp - src/Common/VirtualColumnUtils.cpp - src/Common/MemoryTracker.cpp - src/Common/Macros.cpp - src/Common/formatReadable.cpp - src/Common/localBackup.cpp - src/Common/getFQDNOrHostName.cpp - src/Common/setThreadName.cpp - src/Common/FileChecker.cpp - src/Common/StackTrace.cpp - src/Common/ConfigProcessor.cpp - src/Common/Exception.cpp - src/Common/ShellCommand.cpp - src/Common/isLocalAddress.cpp - src/Common/getNumberOfPhysicalCPUCores.cpp - src/Common/randomSeed.cpp - src/Common/ThreadPool.cpp - - src/Core/Field.cpp - src/Core/FieldVisitors.cpp - src/Core/Block.cpp - src/Core/BlockInfo.cpp - src/Core/NamesAndTypes.cpp - src/Core/ErrorCodes.cpp - src/Core/SortDescription.cpp - - src/IO/CachedCompressedReadBuffer.cpp - src/IO/CompressedReadBuffer.cpp - src/IO/CompressedReadBufferBase.cpp - src/IO/CompressedReadBufferFromFile.cpp - src/IO/CompressedWriteBuffer.cpp - src/IO/ReadBufferFromFile.cpp - src/IO/ReadBufferFromFileDescriptor.cpp - src/IO/WriteBufferFromFile.cpp - src/IO/WriteBufferFromFileDescriptor.cpp - src/IO/ReadHelpers.cpp - src/IO/WriteHelpers.cpp - src/IO/WriteBufferValidUTF8.cpp - src/IO/copyData.cpp - src/IO/ReadBufferFromPocoSocket.cpp - src/IO/WriteBufferFromPocoSocket.cpp - src/IO/HashingWriteBuffer.cpp - src/IO/ReadBufferFromFileBase.cpp - src/IO/WriteBufferFromFileBase.cpp - ${AIO_CPP_FILES} - src/IO/createReadBufferFromFileBase.cpp - src/IO/createWriteBufferFromFileBase.cpp - - src/IO/InterserverWriteBuffer.cpp - src/IO/ReadBufferFromHTTP.cpp - src/IO/ReadWriteBufferFromHTTP.cpp - - src/Columns/ColumnConst.cpp - src/Columns/ColumnArray.cpp - src/Columns/ColumnAggregateFunction.cpp - src/Columns/ColumnExpression.cpp - src/Columns/ColumnsCommon.cpp - src/Columns/ColumnString.cpp - - src/Storages/StorageMaterializedView.cpp - src/Storages/StorageView.cpp - src/Storages/System/StorageSystemNumbers.cpp - src/Storages/System/StorageSystemOne.cpp - src/Storages/System/StorageSystemTables.cpp - src/Storages/System/StorageSystemParts.cpp - src/Storages/System/StorageSystemDatabases.cpp - src/Storages/System/StorageSystemProcesses.cpp - src/Storages/System/StorageSystemEvents.cpp - src/Storages/System/StorageSystemMetrics.cpp - src/Storages/System/StorageSystemAsynchronousMetrics.cpp - src/Storages/System/StorageSystemMerges.cpp - src/Storages/System/StorageSystemSettings.cpp - src/Storages/System/StorageSystemZooKeeper.cpp - src/Storages/System/StorageSystemReplicas.cpp - src/Storages/System/StorageSystemReplicationQueue.cpp - src/Storages/System/StorageSystemDictionaries.cpp - src/Storages/System/StorageSystemColumns.cpp - src/Storages/System/StorageSystemFunctions.cpp - src/Storages/System/StorageSystemClusters.cpp - src/Storages/StorageLog.cpp - src/Storages/StorageTinyLog.cpp - src/Storages/StorageStripeLog.cpp - src/Storages/StorageFile.cpp - src/Storages/StorageMemory.cpp - src/Storages/StorageBuffer.cpp - src/Storages/StorageFactory.cpp - src/Storages/StorageDistributed.cpp - src/Storages/StorageMerge.cpp - src/Storages/StorageMergeTree.cpp - src/Storages/StorageReplicatedMergeTree.cpp - src/Storages/StorageSet.cpp - src/Storages/StorageJoin.cpp - src/Storages/VirtualColumnFactory.cpp - src/Storages/Distributed/DistributedBlockOutputStream.cpp - src/Storages/MergeTree/PKCondition.cpp - src/Storages/MergeTree/DiskSpaceMonitor.cpp - src/Storages/MergeTree/RemoteDiskSpaceMonitor.cpp - src/Storages/MergeTree/ActiveDataPartSet.cpp - src/Storages/MergeTree/BackgroundProcessingPool.cpp - src/Storages/MergeTree/MergeTreeData.cpp - src/Storages/MergeTree/MergeTreeDataPart.cpp - src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp - src/Storages/MergeTree/MergeTreeDataWriter.cpp - src/Storages/MergeTree/MergeTreeDataMerger.cpp - src/Storages/MergeTree/MergeTreePartChecker.cpp - src/Storages/MergeTree/ReplicatedMergeTreeLogEntry.cpp - src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp - src/Storages/MergeTree/DataPartsExchange.cpp - src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp - src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp - src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp - src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp - src/Storages/MergeTree/ReshardingJob.cpp - src/Storages/MergeTree/ReshardingWorker.cpp - src/Storages/MergeTree/MergeTreeSharder.cpp - src/Storages/MergeTree/ShardedPartitionUploader.cpp - src/Storages/MergeTree/RemoteQueryExecutor.cpp - src/Storages/MergeTree/RemotePartChecker.cpp - src/Storages/MergeTree/ReplicatedMergeTreeBlockOutputStream.cpp - src/Storages/MergeTree/SimpleMergeSelector.cpp - src/Storages/MergeTree/LevelMergeSelector.cpp - src/Storages/MergeTree/AllMergeSelector.cpp - src/Storages/MergeTree/MergeTreeReader.cpp - src/Storages/MergeTree/MergeTreeReadPool.cpp - src/Storages/MergeTree/MergeTreeBlockInputStream.cpp - src/Storages/MergeTree/MergeTreeThreadBlockInputStream.cpp - src/Storages/ITableDeclaration.cpp - src/Storages/AlterCommands.cpp - src/Storages/ColumnsDescription.cpp - - src/DataStreams/AggregatingSortedBlockInputStream.cpp - src/DataStreams/IRowOutputStream.cpp - src/DataStreams/IBlockInputStream.cpp - src/DataStreams/IProfilingBlockInputStream.cpp - src/DataStreams/BlockStreamProfileInfo.cpp - src/DataStreams/BlockInputStreamFromRowInputStream.cpp - src/DataStreams/BlockOutputStreamFromRowOutputStream.cpp - src/DataStreams/TabSeparatedRowInputStream.cpp - src/DataStreams/TabSeparatedRowOutputStream.cpp - src/DataStreams/BinaryRowInputStream.cpp - src/DataStreams/BinaryRowOutputStream.cpp - src/DataStreams/ValuesRowInputStream.cpp - src/DataStreams/ValuesRowOutputStream.cpp - src/DataStreams/TabSeparatedBlockOutputStream.cpp - src/DataStreams/NativeBlockInputStream.cpp - src/DataStreams/NativeBlockOutputStream.cpp - src/DataStreams/LimitBlockInputStream.cpp - src/DataStreams/FilterBlockInputStream.cpp - src/DataStreams/PartialSortingBlockInputStream.cpp - src/DataStreams/MergeSortingBlockInputStream.cpp - src/DataStreams/AggregatingBlockInputStream.cpp - src/DataStreams/PrettyBlockOutputStream.cpp - src/DataStreams/PrettyCompactBlockOutputStream.cpp - src/DataStreams/PrettySpaceBlockOutputStream.cpp - src/DataStreams/VerticalRowOutputStream.cpp - src/DataStreams/MergingAggregatedBlockInputStream.cpp - src/DataStreams/MergingAggregatedMemoryEfficientBlockInputStream.cpp - src/DataStreams/MergingSortedBlockInputStream.cpp - src/DataStreams/CollapsingSortedBlockInputStream.cpp - src/DataStreams/CollapsingFinalBlockInputStream.cpp - src/DataStreams/ExpressionBlockInputStream.cpp - src/DataStreams/narrowBlockInputStreams.cpp - src/DataStreams/glueBlockInputStreams.cpp - src/DataStreams/FormatFactory.cpp - src/DataStreams/copyData.cpp - src/DataStreams/JSONRowOutputStream.cpp - src/DataStreams/JSONEachRowRowOutputStream.cpp - src/DataStreams/JSONEachRowRowInputStream.cpp - src/DataStreams/XMLRowOutputStream.cpp - src/DataStreams/TSKVRowOutputStream.cpp - src/DataStreams/TSKVRowInputStream.cpp - src/DataStreams/ODBCDriverBlockOutputStream.cpp - src/DataStreams/JSONCompactRowOutputStream.cpp - src/DataStreams/PrettyCompactMonoBlockOutputStream.cpp - src/DataStreams/CSVRowOutputStream.cpp - src/DataStreams/CSVRowInputStream.cpp - src/DataStreams/SummingSortedBlockInputStream.cpp - src/DataStreams/ReplacingSortedBlockInputStream.cpp - src/DataStreams/GraphiteRollupSortedBlockInputStream.cpp - src/DataStreams/TotalsHavingBlockInputStream.cpp - src/DataStreams/CreatingSetsBlockInputStream.cpp - src/DataStreams/DistinctBlockInputStream.cpp - src/DataStreams/RemoteBlockInputStream.cpp - src/DataStreams/BlockIO.cpp - src/DataStreams/verbosePrintString.cpp - src/DataStreams/SquashingTransform.cpp - src/DataStreams/SquashingBlockInputStream.cpp - src/DataStreams/SquashingBlockOutputStream.cpp - src/DataStreams/ColumnGathererStream.cpp - - src/DataTypes/DataTypeString.cpp - src/DataTypes/DataTypeFixedString.cpp - src/DataTypes/DataTypeArray.cpp - src/DataTypes/DataTypeNested.cpp - src/DataTypes/DataTypeFactory.cpp - src/DataTypes/DataTypeAggregateFunction.cpp - src/DataTypes/DataTypeEnum.cpp - src/DataTypes/DataTypeTuple.cpp - src/DataTypes/FieldToDataType.cpp - - src/Dictionaries/DictionarySourceFactory.cpp - src/Dictionaries/FileDictionarySource.cpp - src/Dictionaries/MySQLDictionarySource.cpp - src/Dictionaries/ODBCDictionarySource.cpp - src/Dictionaries/ClickHouseDictionarySource.cpp - src/Dictionaries/ExecutableDictionarySource.cpp - src/Dictionaries/HTTPDictionarySource.cpp - src/Dictionaries/MySQLBlockInputStream.cpp - src/Dictionaries/ODBCBlockInputStream.cpp - src/Dictionaries/ExternalQueryBuilder.cpp - src/Dictionaries/ExternalResultDescription.cpp - src/Dictionaries/writeParenthesisedString.cpp - src/Dictionaries/DictionaryStructure.cpp - src/Dictionaries/FlatDictionary.cpp - src/Dictionaries/HashedDictionary.cpp - src/Dictionaries/CacheDictionary.cpp - src/Dictionaries/RangeHashedDictionary.cpp - src/Dictionaries/ComplexKeyHashedDictionary.cpp - src/Dictionaries/ComplexKeyCacheDictionary.cpp - - src/Parsers/ASTSelectQuery.cpp - src/Parsers/ASTInsertQuery.cpp - src/Parsers/ASTAlterQuery.cpp - src/Parsers/ASTOrderByElement.cpp - src/Parsers/ASTLiteral.cpp - src/Parsers/ASTFunction.cpp - src/Parsers/ASTWithAlias.cpp - src/Parsers/ASTIdentifier.cpp - src/Parsers/ASTSampleRatio.cpp - src/Parsers/ASTTablesInSelectQuery.cpp - src/Parsers/CommonParsers.cpp - src/Parsers/IAST.cpp - src/Parsers/IParserBase.cpp - src/Parsers/ExpressionElementParsers.cpp - src/Parsers/ExpressionListParsers.cpp - src/Parsers/ParserQueryWithOutput.cpp - src/Parsers/ParserCreateQuery.cpp - src/Parsers/ParserSelectQuery.cpp - src/Parsers/ParserInsertQuery.cpp - src/Parsers/ParserDropQuery.cpp - src/Parsers/ParserRenameQuery.cpp - src/Parsers/ParserSetQuery.cpp - src/Parsers/ParserShowTablesQuery.cpp - src/Parsers/ParserOptimizeQuery.cpp - src/Parsers/ParserTablePropertiesQuery.cpp - src/Parsers/ParserQuery.cpp - src/Parsers/ParserAlterQuery.cpp - src/Parsers/ParserCheckQuery.cpp - src/Parsers/ParserSampleRatio.cpp - src/Parsers/ParserCase.cpp - src/Parsers/ParserTablesInSelectQuery.cpp - src/Parsers/formatAST.cpp - src/Parsers/parseQuery.cpp - src/Parsers/queryToString.cpp - - src/TableFunctions/TableFunctionFactory.cpp - src/TableFunctions/TableFunctionMerge.cpp - src/TableFunctions/TableFunctionRemote.cpp - src/TableFunctions/TableFunctionShardByHash.cpp - src/TableFunctions/getStructureOfRemoteTable.cpp - - src/Interpreters/ExpressionActions.cpp - src/Interpreters/ExpressionAnalyzer.cpp - src/Interpreters/LogicalExpressionsOptimizer.cpp - src/Interpreters/Context.cpp - src/Interpreters/InterpreterAlterQuery.cpp - src/Interpreters/InterpreterCheckQuery.cpp - src/Interpreters/InterpreterCreateQuery.cpp - src/Interpreters/InterpreterSelectQuery.cpp - src/Interpreters/InterpreterInsertQuery.cpp - src/Interpreters/InterpreterDropQuery.cpp - src/Interpreters/InterpreterRenameQuery.cpp - src/Interpreters/InterpreterShowTablesQuery.cpp - src/Interpreters/InterpreterFactory.cpp - src/Interpreters/Aggregator.cpp - src/Interpreters/executeQuery.cpp - src/Interpreters/loadMetadata.cpp - src/Interpreters/sortBlock.cpp - src/Interpreters/evaluateMissingDefaults.cpp - src/Interpreters/evaluateConstantExpression.cpp - src/Interpreters/convertFieldToType.cpp - src/Interpreters/Set.cpp - src/Interpreters/Join.cpp - src/Interpreters/Quota.cpp - src/Interpreters/Cluster.cpp - src/Interpreters/Settings.cpp - src/Interpreters/Compiler.cpp - src/Interpreters/ExternalDictionaries.cpp - src/Interpreters/DictionaryFactory.cpp - src/Interpreters/ProcessList.cpp - src/Interpreters/QueryLog.cpp - src/Interpreters/ClientInfo.cpp - src/Interpreters/getClusterName.cpp - src/Interpreters/ClusterProxy/SelectQueryConstructor.cpp - src/Interpreters/ClusterProxy/DescribeQueryConstructor.cpp - src/Interpreters/ClusterProxy/AlterQueryConstructor.cpp - src/Interpreters/ClusterProxy/Query.cpp - src/Interpreters/AsynchronousMetrics.cpp - - src/Functions/FunctionFactory.cpp - src/Functions/FunctionsArithmetic.cpp - src/Functions/FunctionsArray.cpp - src/Functions/FunctionsCoding.cpp - src/Functions/FunctionsComparison.cpp - src/Functions/FunctionsConditional.cpp - src/Functions/FunctionsConversion.cpp - src/Functions/FunctionsDateTime.cpp - src/Functions/FunctionsDictionaries.cpp - src/Functions/FunctionsFormatting.cpp - src/Functions/FunctionsHashing.cpp - src/Functions/FunctionsHigherOrder.cpp - src/Functions/FunctionsLogical.cpp - src/Functions/FunctionsRandom.cpp - src/Functions/FunctionsReinterpret.cpp - src/Functions/FunctionsRound.cpp - src/Functions/FunctionsString.cpp - src/Functions/FunctionsStringArray.cpp - src/Functions/FunctionsStringSearch.cpp - src/Functions/FunctionsURL.cpp - src/Functions/FunctionsVisitParam.cpp - src/Functions/FunctionsMath.cpp - src/Functions/FunctionsGeo.cpp - src/Functions/FunctionsMiscellaneous.cpp - src/Functions/FunctionsTransform.cpp - src/Functions/FunctionsCharset.cpp - src/Functions/Conditional/getArrayType.cpp - src/Functions/Conditional/ArgsInfo.cpp - src/Functions/Conditional/CondSource.cpp - src/Functions/Conditional/NumericPerformer.cpp - src/Functions/Conditional/StringEvaluator.cpp - src/Functions/Conditional/StringArrayEvaluator.cpp - - src/AggregateFunctions/UniqCombinedBiasData.cpp - src/AggregateFunctions/AggregateFunctionFactory.cpp - src/AggregateFunctions/AggregateFunctionArray.cpp - src/AggregateFunctions/AggregateFunctionAvg.cpp - src/AggregateFunctions/AggregateFunctionCount.cpp - src/AggregateFunctions/AggregateFunctionGroupArray.cpp - src/AggregateFunctions/AggregateFunctionGroupUniqArray.cpp - src/AggregateFunctions/AggregateFunctionIf.cpp - src/AggregateFunctions/AggregateFunctionMerge.cpp - src/AggregateFunctions/AggregateFunctionsQuantile.cpp - src/AggregateFunctions/AggregateFunctionQuantileExact.cpp - src/AggregateFunctions/AggregateFunctionQuantileExactWeighted.cpp - src/AggregateFunctions/AggregateFunctionsQuantileDeterministic.cpp - src/AggregateFunctions/AggregateFunctionsQuantileTiming.cpp - src/AggregateFunctions/AggregateFunctionsQuantileTDigest.cpp - src/AggregateFunctions/AggregateFunctionsSequenceMatch.cpp - src/AggregateFunctions/AggregateFunctionsMinMaxAny.cpp - src/AggregateFunctions/AggregateFunctionsStatistics.cpp - src/AggregateFunctions/AggregateFunctionState.cpp - src/AggregateFunctions/AggregateFunctionSum.cpp - src/AggregateFunctions/AggregateFunctionsUniq.cpp - src/AggregateFunctions/AggregateFunctionUniqUpTo.cpp - src/AggregateFunctions/AggregateFunctionDebug.cpp - - src/Databases/DatabaseMemory.cpp - src/Databases/DatabaseOrdinary.cpp - src/Databases/DatabaseFactory.cpp - src/Databases/DatabasesCommon.cpp - - src/Client/Connection.cpp - src/Client/MultiplexedConnections.cpp - - ${MONGODB_FILES} -) - -###### -list(REMOVE_DUPLICATES dbms_all_sources_old) -list(SORT dbms_all_sources_old) - -set(dbms_all_sources ${dbms_headers} ${dbms_sources}) -list(REMOVE_DUPLICATES dbms_all_sources) -list(SORT dbms_all_sources) - -list(LENGTH dbms_all_sources_old len_old) -list(LENGTH dbms_all_sources len_new) -set(dbms_all_sources_union ${dbms_all_sources} ${dbms_all_sources_old}) -list(LENGTH dbms_all_sources_union len_union) -list(REMOVE_DUPLICATES dbms_all_sources_union) -list(LENGTH dbms_all_sources_union len_union_uniq) -message(STATUS "len_old ${len_old}, len_new ${len_new}, len_union ${len_union}, len_union_uniq ${len_union_uniq}") - -set(dbms_all_sources_not_presented ${dbms_all_sources_old}) -list(REMOVE_ITEM dbms_all_sources_not_presented ${dbms_all_sources}) -message(STATUS "not presented: ${dbms_all_sources_not_presented}") - -set(dbms_all_sources_extra ${dbms_all_sources}) -list(REMOVE_ITEM dbms_all_sources_extra ${dbms_all_sources_old}) -message(STATUS "dbms_all_sources_extra: ${dbms_all_sources_extra}") -###### if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") # Won't generate debug info for files with heavy template instantiation to achieve faster linking and lower size. @@ -1094,9 +111,15 @@ if (NOT CMAKE_BUILD_TYPE STREQUAL "Debug") PROPERTIES COMPILE_FLAGS -g0) endif () + if (NOT AARCH64) set (LINK_LIBRARIES_ONLY_ON_X86_64 cpuid) -endif () +endif() + +set (APPLE_ICONV_LIB "") +if (APPLE) + set(APPLE_ICONV_LIB iconv) +endif() target_link_libraries(dbms common From fc4c02c510a35aebc73e84afcfffaca4f04c89d5 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 7 Dec 2016 18:06:54 +0300 Subject: [PATCH 3/9] StorageSystemMerges refactoring. --- .../Storages/System/StorageSystemMerges.cpp | 83 +++++++------------ 1 file changed, 28 insertions(+), 55 deletions(-) diff --git a/dbms/src/Storages/System/StorageSystemMerges.cpp b/dbms/src/Storages/System/StorageSystemMerges.cpp index 0982c6f4ca6..69157ab9969 100644 --- a/dbms/src/Storages/System/StorageSystemMerges.cpp +++ b/dbms/src/Storages/System/StorageSystemMerges.cpp @@ -13,19 +13,19 @@ namespace DB StorageSystemMerges::StorageSystemMerges(const std::string & name) : name{name} , columns{ - { "database", std::make_shared() }, - { "table", std::make_shared() }, - { "elapsed", std::make_shared() }, - { "progress", std::make_shared() }, - { "num_parts", std::make_shared() }, - { "result_part_name", std::make_shared() }, - { "total_size_bytes_compressed", std::make_shared() }, - { "total_size_marks", std::make_shared() }, - { "bytes_read_uncompressed", std::make_shared() }, - { "rows_read", std::make_shared() }, - { "bytes_written_uncompressed", std::make_shared() }, - { "rows_written", std::make_shared() }, - { "columns_written", std::make_shared() } + { "database", std::make_shared() }, + { "table", std::make_shared() }, + { "elapsed", std::make_shared() }, + { "progress", std::make_shared() }, + { "num_parts", std::make_shared() }, + { "result_part_name", std::make_shared() }, + { "total_size_bytes_compressed", std::make_shared() }, + { "total_size_marks", std::make_shared() }, + { "bytes_read_uncompressed", std::make_shared() }, + { "rows_read", std::make_shared() }, + { "bytes_written_uncompressed", std::make_shared() }, + { "rows_written", std::make_shared() }, + { "columns_written", std::make_shared() } } { } @@ -47,53 +47,26 @@ BlockInputStreams StorageSystemMerges::read( check(column_names); processed_stage = QueryProcessingStage::FetchColumns; - ColumnWithTypeAndName col_database{std::make_shared(), std::make_shared(), "database"}; - ColumnWithTypeAndName col_table{std::make_shared(), std::make_shared(), "table"}; - ColumnWithTypeAndName col_elapsed{std::make_shared(), std::make_shared(), "elapsed"}; - ColumnWithTypeAndName col_progress{std::make_shared(), std::make_shared(), "progress"}; - ColumnWithTypeAndName col_num_parts{std::make_shared(), std::make_shared(), "num_parts"}; - ColumnWithTypeAndName col_result_part_name{std::make_shared(), std::make_shared(), "result_part_name"}; - ColumnWithTypeAndName col_total_size_bytes_compressed{std::make_shared(), std::make_shared(), "total_size_bytes_compressed"}; - ColumnWithTypeAndName col_total_size_marks{std::make_shared(), std::make_shared(), "total_size_marks"}; - ColumnWithTypeAndName col_bytes_read_uncompressed{std::make_shared(), std::make_shared(), "bytes_read_uncompressed"}; - ColumnWithTypeAndName col_rows_read{std::make_shared(), std::make_shared(), "rows_read"}; - ColumnWithTypeAndName col_bytes_written_uncompressed{std::make_shared(), std::make_shared(), "bytes_written_uncompressed"}; - ColumnWithTypeAndName col_rows_written{std::make_shared(), std::make_shared(), "rows_written"}; - ColumnWithTypeAndName col_columns_written{std::make_shared(), std::make_shared(), "columns_written"}; + Block block = getSampleBlock(); for (const auto & merge : context.getMergeList().get()) { - col_database.column->insert(merge.database); - col_table.column->insert(merge.table); - col_elapsed.column->insert(merge.watch.elapsedSeconds()); - col_progress.column->insert(std::min(1., merge.progress)); /// little cheat - col_num_parts.column->insert(merge.num_parts); - col_result_part_name.column->insert(merge.result_part_name); - col_total_size_bytes_compressed.column->insert(merge.total_size_bytes_compressed); - col_total_size_marks.column->insert(merge.total_size_marks); - col_bytes_read_uncompressed.column->insert(merge.bytes_read_uncompressed.load(std::memory_order_relaxed)); - col_rows_read.column->insert(merge.rows_read.load(std::memory_order_relaxed)); - col_bytes_written_uncompressed.column->insert(merge.bytes_written_uncompressed.load(std::memory_order_relaxed)); - col_rows_written.column->insert(merge.rows_written.load(std::memory_order_relaxed)); - col_columns_written.column->insert(merge.columns_written.load(std::memory_order_relaxed)); + size_t i = 0; + block.unsafeGetByPosition(i++).column->insert(merge.database); + block.unsafeGetByPosition(i++).column->insert(merge.table); + block.unsafeGetByPosition(i++).column->insert(merge.watch.elapsedSeconds()); + block.unsafeGetByPosition(i++).column->insert(std::min(1., merge.progress)); /// little cheat + block.unsafeGetByPosition(i++).column->insert(merge.num_parts); + block.unsafeGetByPosition(i++).column->insert(merge.result_part_name); + block.unsafeGetByPosition(i++).column->insert(merge.total_size_bytes_compressed); + block.unsafeGetByPosition(i++).column->insert(merge.total_size_marks); + block.unsafeGetByPosition(i++).column->insert(merge.bytes_read_uncompressed.load(std::memory_order_relaxed)); + block.unsafeGetByPosition(i++).column->insert(merge.rows_read.load(std::memory_order_relaxed)); + block.unsafeGetByPosition(i++).column->insert(merge.bytes_written_uncompressed.load(std::memory_order_relaxed)); + block.unsafeGetByPosition(i++).column->insert(merge.rows_written.load(std::memory_order_relaxed)); + block.unsafeGetByPosition(i++).column->insert(merge.columns_written.load(std::memory_order_relaxed)); } - Block block{ - col_database, - col_table, - col_elapsed, - col_progress, - col_num_parts, - col_result_part_name, - col_total_size_bytes_compressed, - col_total_size_marks, - col_bytes_read_uncompressed, - col_rows_read, - col_bytes_written_uncompressed, - col_rows_written, - col_columns_written - }; - return BlockInputStreams{1, std::make_shared(block)}; } From 774eb01fd3a4a7227bd3cb993d242e7c96d283ef Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 7 Dec 2016 18:59:22 +0300 Subject: [PATCH 4/9] Added MergesTime metric. [#METR-23305] --- dbms/src/Common/ProfileEvents.cpp | 1 + .../MergeTree/MergeTreeDataMerger.cpp | 35 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/dbms/src/Common/ProfileEvents.cpp b/dbms/src/Common/ProfileEvents.cpp index 0853083aee9..4a41ee0087b 100644 --- a/dbms/src/Common/ProfileEvents.cpp +++ b/dbms/src/Common/ProfileEvents.cpp @@ -86,6 +86,7 @@ \ M(MergedRows) \ M(MergedUncompressedBytes) \ + M(MergesTime)\ \ M(MergeTreeDataWriterRows) \ M(MergeTreeDataWriterUncompressedBytes) \ diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index b79c693f04e..e0317a2f1b9 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -29,6 +29,7 @@ namespace ProfileEvents { extern const Event MergedRows; extern const Event MergedUncompressedBytes; + extern const Event MergesTime; } namespace CurrentMetrics @@ -386,25 +387,37 @@ public: class MergeProgressCallback : public ProgressCallback { public: - MergeProgressCallback(MergeList::Entry & merge_entry_) : merge_entry(merge_entry_) {} + MergeProgressCallback(MergeList::Entry & merge_entry_, UInt64 & watch_prev_elapsed_) + : merge_entry(merge_entry_), watch_prev_elapsed(watch_prev_elapsed_) {} - MergeProgressCallback(MergeList::Entry & merge_entry_, MergeTreeDataMerger::MergeAlgorithm merge_alg_, size_t num_total_rows, - const ColumnSizeEstimator & column_sizes) - : merge_entry(merge_entry_), merge_alg(merge_alg_) + MergeProgressCallback(MergeList::Entry & merge_entry_, size_t num_total_rows, const ColumnSizeEstimator & column_sizes, + UInt64 & watch_prev_elapsed_, MergeTreeDataMerger::MergeAlgorithm merge_alg_ = MergeAlgorithm::Vertical) + : merge_entry(merge_entry_), watch_prev_elapsed(watch_prev_elapsed_), merge_alg(merge_alg_) { average_elem_progress = (merge_alg == MergeAlgorithm::Horizontal) ? 1.0 / num_total_rows : column_sizes.keyColumnsProgress(1, num_total_rows); + + updateWatch(); } MergeList::Entry & merge_entry; - const MergeAlgorithm merge_alg{MergeAlgorithm::Vertical}; + UInt64 & watch_prev_elapsed; Float64 average_elem_progress; + const MergeAlgorithm merge_alg{MergeAlgorithm::Vertical}; + + void updateWatch() + { + UInt64 watch_curr_elapsed = merge_entry->watch.elapsed(); + ProfileEvents::increment(ProfileEvents::MergesTime, watch_curr_elapsed - watch_prev_elapsed); + watch_prev_elapsed = watch_curr_elapsed; + } void operator() (const Progress & value) { ProfileEvents::increment(ProfileEvents::MergedUncompressedBytes, value.bytes); ProfileEvents::increment(ProfileEvents::MergedRows, value.rows); + updateWatch(); merge_entry->bytes_read_uncompressed += value.bytes; merge_entry->rows_read += value.rows; @@ -417,10 +430,11 @@ class MergeProgressCallbackVerticalStep : public MergeProgressCallback public: MergeProgressCallbackVerticalStep(MergeList::Entry & merge_entry_, size_t num_total_rows_exact, - const ColumnSizeEstimator & column_sizes, const String & column_name) - : MergeProgressCallback(merge_entry_), initial_progress(merge_entry->progress) + const ColumnSizeEstimator & column_sizes, const String & column_name, UInt64 & watch_prev_elapsed_) + : MergeProgressCallback(merge_entry_, watch_prev_elapsed_), initial_progress(merge_entry->progress) { average_elem_progress = column_sizes.columnProgress(column_name, 1, num_total_rows_exact); + updateWatch(); } Float64 initial_progress; @@ -431,6 +445,7 @@ public: { merge_entry->bytes_read_uncompressed += value.bytes; ProfileEvents::increment(ProfileEvents::MergedUncompressedBytes, value.bytes); + updateWatch(); rows_read_internal += value.rows; Float64 local_progress = average_elem_progress * rows_read_internal; @@ -503,6 +518,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart * Попутно вычисляем выражение для сортировки. */ BlockInputStreams src_streams; + UInt64 watch_prev_elapsed = 0; for (size_t i = 0; i < parts.size(); ++i) { @@ -512,7 +528,8 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart part_path, DEFAULT_MERGE_BLOCK_SIZE, merging_column_names, data, parts[i], MarkRanges(1, MarkRange(0, parts[i]->size)), false, nullptr, "", true, aio_threshold, DBMS_DEFAULT_BUFFER_SIZE, false); - input->setProgressCallback(MergeProgressCallback{merge_entry, merge_alg, sum_input_rows_upper_bound, column_sizes}); + input->setProgressCallback( + MergeProgressCallback{merge_entry, sum_input_rows_upper_bound, column_sizes, watch_prev_elapsed, merge_alg}); if (data.merging_params.mode != MergeTreeData::MergingParams::Unsorted) src_streams.emplace_back(std::make_shared( @@ -640,7 +657,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart false, true); column_part_stream->setProgressCallback( - MergeProgressCallbackVerticalStep{merge_entry, sum_input_rows_exact, column_sizes, column_name}); + MergeProgressCallbackVerticalStep{merge_entry, sum_input_rows_exact, column_sizes, column_name, watch_prev_elapsed}); column_part_streams[part_num] = std::move(column_part_stream); } From 855dc565986dc99cd26eaad807e22fa26234d04d Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Wed, 7 Dec 2016 00:02:22 +0300 Subject: [PATCH 5/9] Added treshold setting for Vertical merge algorithm. [#METR-23305] --- dbms/include/DB/Storages/MergeTree/MergeTreeSettings.h | 4 ++++ dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeSettings.h b/dbms/include/DB/Storages/MergeTree/MergeTreeSettings.h index 52256d40869..87b3168a716 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeSettings.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeSettings.h @@ -98,6 +98,9 @@ struct MergeTreeSettings /// Enable usage of Vertical merge algorithm. size_t enable_vertical_merge_algorithm = 0; + /// Minimal (approximate) sum of rows in merging parts to activate Vertical merge algorithm + size_t vertical_merge_algorithm_min_rows_to_activate = 16 * DEFAULT_MERGE_BLOCK_SIZE; + void loadFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config) { @@ -133,6 +136,7 @@ struct MergeTreeSettings SET_SIZE_T(min_relative_delay_to_close); SET_SIZE_T(min_absolute_delay_to_close); SET_SIZE_T(enable_vertical_merge_algorithm); + SET_SIZE_T(vertical_merge_algorithm_min_rows_to_activate); #undef SET_SIZE_T #undef SET_DOUBLE diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index e0317a2f1b9..4cead453a74 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -719,7 +719,7 @@ MergeTreeDataMerger::MergeAlgorithm MergeTreeDataMerger::chooseMergeAlgorithm( bool enough_ordinary_cols = data.getColumnNamesList().size() > data.getSortDescription().size(); - bool enough_total_rows = sum_rows_upper_bound >= DEFAULT_MERGE_BLOCK_SIZE; + bool enough_total_rows = sum_rows_upper_bound >= data.context.getMergeTreeSettings().vertical_merge_algorithm_min_rows_to_activate; bool no_parts_overflow = parts.size() <= RowSourcePart::MAX_PARTS; From ea11f61433d001c8782d5658f811da0e197fc956 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Fri, 9 Dec 2016 13:10:12 +0300 Subject: [PATCH 6/9] Added requested changes. [#METR-19372] --- .../InputStreamFromASTInsertQuery.h | 70 ++++--------------- dbms/include/DB/Interpreters/QueryLog.h | 5 +- .../InputStreamFromASTInsertQuery.cpp | 37 ++++++++++ dbms/src/Interpreters/executeQuery.cpp | 2 - .../MergeTree/MergeTreeDataMerger.cpp | 14 +++- 5 files changed, 65 insertions(+), 63 deletions(-) create mode 100644 dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp diff --git a/dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h b/dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h index 28ccf8db1c8..baea9331ed2 100644 --- a/dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h +++ b/dbms/include/DB/DataStreams/InputStreamFromASTInsertQuery.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace DB { @@ -13,69 +13,27 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } - +/** Prepares an input stream which produce data containing in INSERT query + * Head of inserting data could be stored in INSERT ast directly + * Remaining (tail) data could be stored in input_buffer_tail_part + */ class InputStreamFromASTInsertQuery : public IProfilingBlockInputStream { public: - InputStreamFromASTInsertQuery(const ASTPtr & ast, ReadBuffer & istr, const BlockIO & streams, Context & context) - { - const ASTInsertQuery * ast_insert_query = dynamic_cast(ast.get()); + InputStreamFromASTInsertQuery(const ASTPtr & ast, ReadBuffer & input_buffer_tail_part, const BlockIO & streams, Context & context); - if (!ast_insert_query) - throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); + Block readImpl() override { return res_stream->read(); } + void readPrefixImpl() override { return res_stream->readPrefix(); } + void readSuffixImpl() override { return res_stream->readSuffix(); } - String format = ast_insert_query->format; - if (format.empty()) - format = "Values"; - - /// Data could be in parsed (ast_insert_query.data) and in not parsed yet (istr) part of query. - - buf1 = std::make_unique( - const_cast(ast_insert_query->data), ast_insert_query->data ? ast_insert_query->end - ast_insert_query->data : 0, 0); - - if (ast_insert_query->data) - buffers.push_back(buf1.get()); - buffers.push_back(&istr); - - /** NOTE Must not read from 'istr' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. - * - because 'query.data' could refer to memory piece, used as buffer for 'istr'. - */ - - data_istr = std::make_unique(buffers); - - res_stream = context.getInputFormat(format, *data_istr, streams.out_sample, context.getSettings().max_insert_block_size); - } - - Block readImpl() override - { - return res_stream->read(); - } - - void readPrefixImpl() override - { - return res_stream->readPrefix(); - } - - void readSuffixImpl() override - { - return res_stream->readSuffix(); - } - - String getName() const override - { - return "InputStreamFromASTInsertQuery"; - } - - String getID() const override - { - return "InputStreamFromASTInsertQuery(" + toString(this) + ")"; - } + String getName() const override { return "InputStreamFromASTInsertQuery"; } + String getID() const override { return "InputStreamFromASTInsertQuery(" + toString(this) + ")"; } private: - ConcatReadBuffer::ReadBuffers buffers; - std::unique_ptr buf1; - std::unique_ptr data_istr; + + std::unique_ptr input_buffer_ast_part; + std::unique_ptr input_buffer_contacenated; BlockInputStreamPtr res_stream; }; diff --git a/dbms/include/DB/Interpreters/QueryLog.h b/dbms/include/DB/Interpreters/QueryLog.h index f04cfb0f614..210d4357b6c 100644 --- a/dbms/include/DB/Interpreters/QueryLog.h +++ b/dbms/include/DB/Interpreters/QueryLog.h @@ -32,14 +32,15 @@ struct QueryLogElement time_t query_start_time{}; UInt64 query_duration_ms{}; + /// The data fetched from DB to execute the query UInt64 read_rows{}; UInt64 read_bytes{}; + /// The data written to DB UInt64 written_rows{}; UInt64 written_bytes{}; - /// NOTE: Not obvious metric. - /// It always approximately equal to read_rows or written_rows at the end of query execution. + /// The data sent to the client UInt64 result_rows{}; UInt64 result_bytes{}; diff --git a/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp new file mode 100644 index 00000000000..42e3a86b57e --- /dev/null +++ b/dbms/src/DataStreams/InputStreamFromASTInsertQuery.cpp @@ -0,0 +1,37 @@ +#include + +namespace DB +{ + +InputStreamFromASTInsertQuery::InputStreamFromASTInsertQuery( + const ASTPtr & ast, ReadBuffer & input_buffer_tail_part, const BlockIO & streams, Context & context) +{ + const ASTInsertQuery * ast_insert_query = dynamic_cast(ast.get()); + + if (!ast_insert_query) + throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); + + String format = ast_insert_query->format; + if (format.empty()) + format = "Values"; + + /// Data could be in parsed (ast_insert_query.data) and in not parsed yet (input_buffer_tail_part) part of query. + + input_buffer_ast_part = std::make_unique( + const_cast(ast_insert_query->data), ast_insert_query->data ? ast_insert_query->end - ast_insert_query->data : 0, 0); + + ConcatReadBuffer::ReadBuffers buffers; + if (ast_insert_query->data) + buffers.push_back(input_buffer_ast_part.get()); + buffers.push_back(&input_buffer_tail_part); + + /** NOTE Must not read from 'input_buffer_tail_part' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. + * - because 'query.data' could refer to memory piece, used as buffer for 'input_buffer_tail_part'. + */ + + input_buffer_contacenated = std::make_unique(buffers); + + res_stream = context.getInputFormat(format, *input_buffer_contacenated, streams.out_sample, context.getSettings().max_insert_block_size); +} + +} diff --git a/dbms/src/Interpreters/executeQuery.cpp b/dbms/src/Interpreters/executeQuery.cpp index d29e04d6b44..a41bd62d798 100644 --- a/dbms/src/Interpreters/executeQuery.cpp +++ b/dbms/src/Interpreters/executeQuery.cpp @@ -207,8 +207,6 @@ static std::tuple executeQueryImpl( stream->setProcessListElement(context.getProcessListElement()); } } - if (!res.out || !dynamic_cast(res.out.get())) - LOG_DEBUG(&Logger::get("executeQuery"), "res.out " << res.out.get() << " is empty"); /// Everything related to query log. { diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index 4cead453a74..af96015bd54 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -383,7 +383,12 @@ public: } }; - +/** Progress callback. Is used by Horizontal merger and first step of Vertical merger. + * What it should update: + * - approximate progress + * - amount of merged rows and their size (PK columns subset is used in case of Vertical merge) + * - time elapsed for current merge. + */ class MergeProgressCallback : public ProgressCallback { public: @@ -425,6 +430,9 @@ public: }; }; +/** Progress callback for gathering step of Vertical merge. + * Updates: approximate progress, amount of merged bytes (TODO: two column case should be fixed), elapsed time. + */ class MergeProgressCallbackVerticalStep : public MergeProgressCallback { public: @@ -438,8 +446,7 @@ public: } Float64 initial_progress; - /// NOTE: not thread safe (to be copyable). It is OK in current single thread use case - size_t rows_read_internal{0}; + size_t rows_read_internal{0}; // NOTE: not thread safe (to be copyable). It is OK in current single thread use case void operator() (const Progress & value) { @@ -453,6 +460,7 @@ public: }; }; + /// parts should be sorted. MergeTreeData::MutableDataPartPtr MergeTreeDataMerger::mergePartsToTemporaryPart( MergeTreeData::DataPartsVector & parts, const String & merged_name, MergeList::Entry & merge_entry, From 096295608e0690339e990b0ba840dc1bfeb9fd86 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Fri, 9 Dec 2016 17:42:10 +0300 Subject: [PATCH 7/9] Fixed benchmark.html. --- doc/benchmark.html | 78 ++++++++++++++++++++++++---------------------- 1 file changed, 40 insertions(+), 38 deletions(-) diff --git a/doc/benchmark.html b/doc/benchmark.html index 78b3a33d9f9..4333a80a7c2 100644 --- a/doc/benchmark.html +++ b/doc/benchmark.html @@ -1434,7 +1434,7 @@ function generate_selectors(elem) { filter(function(run) { return run.data_size == current_data_size; }). map(function(run) { return run.system; }); - for (var i in systems) { + for (var i = 0; i < systems.length; i++) { var selected = current_systems.indexOf(systems[i]) != -1; var available = available_systems_for_current_data_size.indexOf(systems[i]) != -1; @@ -1449,7 +1449,7 @@ function generate_selectors(elem) { html += "Dataset size: "; - for (var i in data_sizes) { + for (var i = 0; i < data_sizes.length; i++) { html += "" + data_sizes[i].name + " "; } @@ -1458,8 +1458,8 @@ function generate_selectors(elem) { html += "Run number: "; - for (var i in runs) { - html += "" + runs[i] + " "; + for (var i = 0; i < runs.length; i++) { + html += "" + runs[i] + " "; } html += "

"; @@ -1555,13 +1555,13 @@ function generate_comparison_table() { html += ""; html += ""; html += "Query"; - for (j in filtered_results) { + for (j = 0; j < filtered_results.length; j++) { html += "" + filtered_results[j].system + (filtered_results[j].version ? " (" + filtered_results[j].version + ")" : "") + ""; } html += ""; - for (i in queries) { + for (i = 0; i < queries.length; i++) { html += ""; html += ""; @@ -1571,8 +1571,8 @@ function generate_comparison_table() { // Вычислим максимальное и минимальное время выполнения по системам, для каждого из трёх прогонов. var minimums = [0, 0, 0], maximums = [0, 0, 0]; - for (j in filtered_results) { - for (current_run_idx in current_runs) { + for (j = 0; j < filtered_results.length; j++) { + for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var value = filtered_results[j].result[i][k]; @@ -1591,12 +1591,12 @@ function generate_comparison_table() { } } - for (j in filtered_results) { + for (j = 0; j < filtered_results.length; j++) { if (!ratios[j]) { ratios[j] = []; } - for (current_run_idx in current_runs) { + for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var value = filtered_results[j].result[i][k]; @@ -1616,21 +1616,21 @@ function generate_comparison_table() { html += ""; } - if (current_systems.length > 1) { + if (current_systems.length) { html += ""; html += ""; html += "
Geometric mean of ratios
"; - for (j in filtered_results) { - for (k in current_runs) { - html += ""; + for (j = 0; j < filtered_results.length; j++) { + for (k = 0; k < current_runs.length; k++) { + html += ""; } } html += ""; html += ""; - for (j in filtered_results) { + for (j = 0; j < filtered_results.length; j++) { html += ""; } @@ -1641,7 +1641,7 @@ function generate_comparison_table() { $('#comparison_table').html(html); - for (i in queries) { + for (i = 0; i < queries.length; i++) { $('#query_checkbox' + i).click(function() { calculate_totals(); generate_diagram(); } ); } @@ -1650,15 +1650,15 @@ function generate_comparison_table() { function calculate_totals() { - if (current_systems.length <= 1) return; + if (!current_systems.length) return; var filtered_results = results.filter(function(x) { return x.data_size == current_data_size && current_systems.indexOf(x.system) != -1; }); var total_ratios = []; - for (j in filtered_results) { - for (current_run_idx in current_runs) { + for (j = 0; j < filtered_results.length; j++) { + for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var current_ratios = ratios[j][k].filter( @@ -1677,7 +1677,7 @@ function calculate_totals() { } } - for (j in filtered_results) { + for (j = 0; j < filtered_results.length; j++) { var total_ratio = Math.pow(total_ratios[j], 1 / current_runs.length); $("#absolute_totals" + j).attr("data-ratio", total_ratio).html("x" + total_ratio.toFixed(2)); } @@ -1696,8 +1696,8 @@ function generate_diagram() { var max_total_ratio = 1; var min_total_ratio = 0; - for (j in filtered_results) { - for (current_run_idx in current_runs) { + for (j = 0; j < filtered_results.length; j++) { + for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var ratio = +$("#totals" + j + "_" + k).attr("data-ratio"); @@ -1723,7 +1723,7 @@ function generate_diagram() { html += ""; - for (j in filtered_results) { + for (j = 0; j < filtered_results.length; j++) { var total_ratio = +$("#absolute_totals" + j).attr("data-ratio"); html += ""; @@ -1732,7 +1732,7 @@ function generate_diagram() { html += ""; + //min_total_ratio = 1; + //total_ratio = 1; html += ""; html += ""; } @@ -1788,20 +1790,20 @@ try { var yaCounter18343495 = new Ya.Metrika({id:18343495,

Performance comparison of analytical DBMS

From decf424dfca4dc6dc5b9742b8232b0ec69809ae4 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Fri, 9 Dec 2016 18:39:50 +0300 Subject: [PATCH 8/9] Prefer CMake option() instead of set(...CAHE BOOL...). --- CMakeLists.txt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c1986438527..9f86fc03ba9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,30 +46,30 @@ set (COMMON_WARNING_FLAGS "-Wall -Werror") set (CXX_WARNING_FLAGS "-Wnon-virtual-dtor -Wold-style-cast") set (CXX11_ABI "DEFAULT" CACHE STRING "Use C++11 ABI: DEFAULT, ENABLE, DISABLE") -set (TEST_COVERAGE FALSE CACHE BOOL "Enables flags for test coverage") -set (ENABLE_TESTS TRUE CACHE BOOL "Enables tests") +option (TEST_COVERAGE "Enables flags for test coverage" OFF) +option (ENABLE_TESTS "Enables tests" ON) -set (USE_STATIC_LIBRARIES TRUE CACHE BOOL "Set to FALSE to use shared libraries") +option (USE_STATIC_LIBRARIES "Set to FALSE to use shared libraries" ON) if (NOT $ENV{USE_STATIC_LIBRARIES}) set (USE_STATIC_LIBRARIES FALSE) endif () -set (GLIBC_COMPATIBILITY FALSE CACHE BOOL "Set to TRUE to enable compatibility with older glibc libraries. Note that it is not compatible with ASan.") +option (GLIBC_COMPATIBILITY "Set to TRUE to enable compatibility with older glibc libraries. Note that it is not compatible with ASan." OFF) if ($ENV{GLIBC_COMPATIBILITY}) set (GLIBC_COMPATIBILITY TRUE) endif () -set (ENABLE_MONGODB TRUE CACHE BOOL "Set to TRUE to enable MongoDB support as source for external dictionaries") +option (ENABLE_MONGODB "Set to TRUE to enable MongoDB support as source for external dictionaries" ON) if (NOT $ENV{ENABLE_MONGODB}) set (ENABLE_MONGODB FALSE) endif () -set (ENABLE_LIBTCMALLOC TRUE CACHE BOOL "Set to TRUE to enable libtcmalloc.") +option (ENABLE_LIBTCMALLOC "Set to TRUE to enable libtcmalloc." ON) if (NOT $ENV{ENABLE_LIBTCMALLOC}) set (ENABLE_LIBTCMALLOC FALSE) endif () -set (DEBUG_LIBTCMALLOC FALSE CACHE BOOL "Set to TRUE to use debug version of libtcmalloc.") +option (DEBUG_LIBTCMALLOC "Set to TRUE to use debug version of libtcmalloc." OFF) if ($ENV{DEBUG_LIBTCMALLOC}) set (ENABLE_LIBTCMALLOC TRUE) endif () From 80cd00542c2bef1506c818076a792e5acd5edf18 Mon Sep 17 00:00:00 2001 From: Vitaliy Lyudvichenko Date: Mon, 12 Dec 2016 18:23:21 +0300 Subject: [PATCH 9/9] Improved benchmark and CMakeLists. --- CMakeLists.txt | 2 +- doc/benchmark.html | 42 ++++++++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9f86fc03ba9..838a9fb1209 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -71,7 +71,7 @@ endif () option (DEBUG_LIBTCMALLOC "Set to TRUE to use debug version of libtcmalloc." OFF) if ($ENV{DEBUG_LIBTCMALLOC}) - set (ENABLE_LIBTCMALLOC TRUE) + set (DEBUG_LIBTCMALLOC TRUE) endif () if (GLIBC_COMPATIBILITY) diff --git a/doc/benchmark.html b/doc/benchmark.html index 4333a80a7c2..20a74f3982a 100644 --- a/doc/benchmark.html +++ b/doc/benchmark.html @@ -1553,15 +1553,15 @@ function generate_comparison_table() { html += "
"; - for (current_run_idx in current_runs) { + for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var ratio = +$("#totals" + j + "_" + k).attr("data-ratio"); @@ -1749,6 +1749,8 @@ function generate_diagram() { html += "" + (total_ratio / min_total_ratio).toFixed(2) + "
"; html += ""; - html += ""; + html += ""; html += ""; - for (j = 0; j < filtered_results.length; j++) { + for (var j = 0; j < filtered_results.length; j++) { html += ""; } html += ""; - for (i = 0; i < queries.length; i++) { + for (var i = 0; i < queries.length; i++) { html += ""; html += ""; @@ -1571,8 +1571,8 @@ function generate_comparison_table() { // Вычислим максимальное и минимальное время выполнения по системам, для каждого из трёх прогонов. var minimums = [0, 0, 0], maximums = [0, 0, 0]; - for (j = 0; j < filtered_results.length; j++) { - for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + for (var j = 0; j < filtered_results.length; j++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var value = filtered_results[j].result[i][k]; @@ -1591,12 +1591,12 @@ function generate_comparison_table() { } } - for (j = 0; j < filtered_results.length; j++) { + for (var j = 0; j < filtered_results.length; j++) { if (!ratios[j]) { ratios[j] = []; } - for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var value = filtered_results[j].result[i][k]; @@ -1621,8 +1621,8 @@ function generate_comparison_table() { html += ""; html += ""; - for (j = 0; j < filtered_results.length; j++) { - for (k = 0; k < current_runs.length; k++) { + for (var j = 0; j < filtered_results.length; j++) { + for (var k = 0; k < current_runs.length; k++) { html += ""; } } @@ -1630,7 +1630,7 @@ function generate_comparison_table() { html += ""; html += ""; - for (j = 0; j < filtered_results.length; j++) { + for (var j = 0; j < filtered_results.length; j++) { html += ""; } @@ -1641,9 +1641,15 @@ function generate_comparison_table() { $('#comparison_table').html(html); - for (i = 0; i < queries.length; i++) { + for (var i = 0; i < queries.length; i++) { $('#query_checkbox' + i).click(function() { calculate_totals(); generate_diagram(); } ); } + $('#query_checkbox_toggler').click(function() { + for (var i = 0; i < queries.length; i++) { + var item = $('#query_checkbox' + i); + item.prop("checked", !item.prop("checked")); + } + }); calculate_totals(); } @@ -1657,8 +1663,8 @@ function calculate_totals() { var total_ratios = []; - for (j = 0; j < filtered_results.length; j++) { - for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + for (var j = 0; j < filtered_results.length; j++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var current_ratios = ratios[j][k].filter( @@ -1677,7 +1683,7 @@ function calculate_totals() { } } - for (j = 0; j < filtered_results.length; j++) { + for (var j = 0; j < filtered_results.length; j++) { var total_ratio = Math.pow(total_ratios[j], 1 / current_runs.length); $("#absolute_totals" + j).attr("data-ratio", total_ratio).html("x" + total_ratio.toFixed(2)); } @@ -1696,8 +1702,8 @@ function generate_diagram() { var max_total_ratio = 1; var min_total_ratio = 0; - for (j = 0; j < filtered_results.length; j++) { - for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + for (var j = 0; j < filtered_results.length; j++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var ratio = +$("#totals" + j + "_" + k).attr("data-ratio"); @@ -1723,7 +1729,7 @@ function generate_diagram() { html += "
Query" + filtered_results[j].system + (filtered_results[j].version ? " (" + filtered_results[j].version + ")" : "") + "
Geometric mean of ratios
"; - for (j = 0; j < filtered_results.length; j++) { + for (var j = 0; j < filtered_results.length; j++) { var total_ratio = +$("#absolute_totals" + j).attr("data-ratio"); html += ""; @@ -1732,7 +1738,7 @@ function generate_diagram() { html += "
"; - for (current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { + for (var current_run_idx = 0; current_run_idx < current_runs.length; current_run_idx++) { var k = current_runs[current_run_idx]; var ratio = +$("#totals" + j + "_" + k).attr("data-ratio");