From 12384551da5fd4d280eb95c356da449c31761d01 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 26 May 2020 16:19:24 +0300 Subject: [PATCH 01/38] Fix totals and extremes header for views. --- src/Processors/Pipe.h | 2 +- src/Processors/QueryPipeline.cpp | 13 ++++++++++- src/Processors/QueryPipeline.h | 3 ++- src/Storages/StorageView.cpp | 37 ++++++++++++++++++++++---------- 4 files changed, 41 insertions(+), 14 deletions(-) diff --git a/src/Processors/Pipe.h b/src/Processors/Pipe.h index 984fa7605c6..ec5514915a7 100644 --- a/src/Processors/Pipe.h +++ b/src/Processors/Pipe.h @@ -23,7 +23,7 @@ public: /// Will connect pipes outputs with transform inputs automatically. Pipe(Pipes && pipes, ProcessorPtr transform); /// Create pipe from output port. If pipe was created that way, it possibly will not have tree shape. - Pipe(OutputPort * port); + explicit Pipe(OutputPort * port); Pipe(const Pipe & other) = delete; Pipe(Pipe && other) = default; diff --git a/src/Processors/QueryPipeline.cpp b/src/Processors/QueryPipeline.cpp index 13787a3fd3b..26736672412 100644 --- a/src/Processors/QueryPipeline.cpp +++ b/src/Processors/QueryPipeline.cpp @@ -703,6 +703,11 @@ void QueryPipeline::initRowsBeforeLimit() Pipe QueryPipeline::getPipe() && { resize(1); + return std::move(std::move(*this).getPipes()[0]); +} + +Pipes QueryPipeline::getPipes() && +{ Pipe pipe(std::move(processors), streams.at(0), totals_having_port, extremes_port); pipe.max_parallel_streams = streams.maxParallelStreams(); @@ -721,7 +726,13 @@ Pipe QueryPipeline::getPipe() && if (extremes_port) pipe.setExtremesPort(extremes_port); - return pipe; + Pipes pipes; + pipes.emplace_back(std::move(pipe)); + + for (size_t i = 1; i < streams.size(); ++i) + pipes.emplace_back(Pipe(streams[i])); + + return pipes; } PipelineExecutorPtr QueryPipeline::execute() diff --git a/src/Processors/QueryPipeline.h b/src/Processors/QueryPipeline.h index 45e38ffa715..d248853131c 100644 --- a/src/Processors/QueryPipeline.h +++ b/src/Processors/QueryPipeline.h @@ -155,8 +155,9 @@ public: /// Set upper limit for the recommend number of threads void setMaxThreads(size_t max_threads_) { max_threads = max_threads_; } - /// Convert query pipeline to single pipe. + /// Convert query pipeline to single or several pipes. Pipe getPipe() &&; + Pipes getPipes() &&; private: /// Destruction order: processors, header, locks, temporary storages, local contexts diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 636c7f9d64d..01c85c5238a 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include namespace DB @@ -62,29 +64,42 @@ Pipes StorageView::read( if (context.getSettings().enable_optimize_predicate_expression) current_inner_query = getRuntimeViewQuery(*query_info.query->as(), context); - QueryPipeline pipeline; InterpreterSelectWithUnionQuery interpreter(current_inner_query, context, {}, column_names); /// FIXME res may implicitly use some objects owned be pipeline, but them will be destructed after return if (query_info.force_tree_shaped_pipeline) { + QueryPipeline pipeline; BlockInputStreams streams = interpreter.executeWithMultipleStreams(pipeline); + + for (auto & stream : streams) + { + stream = std::make_shared(stream); + stream = std::make_shared(stream, getSampleBlockForColumns(column_names), + ConvertingBlockInputStream::MatchColumnsMode::Name); + } + for (auto & stream : streams) pipes.emplace_back(std::make_shared(std::move(stream))); } else - /// TODO: support multiple streams here. Need more general interface than pipes. - pipes.emplace_back(interpreter.executeWithProcessors().getPipe()); - - /// It's expected that the columns read from storage are not constant. - /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. - for (auto & pipe : pipes) { - pipe.addSimpleTransform(std::make_shared(pipe.getHeader())); + auto pipeline = interpreter.executeWithProcessors(); + + /// It's expected that the columns read from storage are not constant. + /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery. + pipeline.addSimpleTransform([](const Block & header) + { + return std::make_shared(header); + }); /// And also convert to expected structure. - pipe.addSimpleTransform(std::make_shared( - pipe.getHeader(), getSampleBlockForColumns(column_names), - ConvertingTransform::MatchColumnsMode::Name)); + pipeline.addSimpleTransform([&](const Block & header) + { + return std::make_shared(header, getSampleBlockForColumns(column_names), + ConvertingTransform::MatchColumnsMode::Name); + }); + + pipes = std::move(pipeline).getPipes(); } return pipes; From 8074e2027fd76e03725598fbdf5e15565301441b Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 26 May 2020 16:32:44 +0300 Subject: [PATCH 02/38] Added test. --- .../queries/0_stateless/01284_view_and_extremes_bug.reference | 1 + tests/queries/0_stateless/01284_view_and_extremes_bug.sql | 4 ++++ 2 files changed, 5 insertions(+) create mode 100644 tests/queries/0_stateless/01284_view_and_extremes_bug.reference create mode 100644 tests/queries/0_stateless/01284_view_and_extremes_bug.sql diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.reference b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference new file mode 100644 index 00000000000..216e97ce082 --- /dev/null +++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.reference @@ -0,0 +1 @@ +World diff --git a/tests/queries/0_stateless/01284_view_and_extremes_bug.sql b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql new file mode 100644 index 00000000000..c444441a258 --- /dev/null +++ b/tests/queries/0_stateless/01284_view_and_extremes_bug.sql @@ -0,0 +1,4 @@ +drop table if exists view_bug_const; +CREATE VIEW view_bug_const AS SELECT 'World' AS hello FROM (SELECT number FROM system.numbers LIMIT 1) AS n1 JOIN (SELECT number FROM system.numbers LIMIT 1) AS n2 USING (number); +select * from view_bug_const; +drop table if exists view_bug_const; From b2057159c1d12fb016b7918b7a00b11ee73967fc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2020 17:24:20 +0300 Subject: [PATCH 03/38] Experiment on parsing backslash escapes in a more convenient way #10922 --- src/IO/ReadHelpers.cpp | 16 +++- src/IO/tests/CMakeLists.txt | 3 - src/IO/tests/mempbrk.cpp | 90 ------------------- ...escape_sequences_php_mysql_style.reference | 3 + ...01284_escape_sequences_php_mysql_style.sql | 6 ++ 5 files changed, 22 insertions(+), 96 deletions(-) delete mode 100644 src/IO/tests/mempbrk.cpp create mode 100644 tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference create mode 100644 tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index dccb413af2c..65c3cfab55a 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -283,7 +283,9 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf) if (buf.eof()) throw Exception("Cannot parse escape sequence", ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); - if (*buf.position() == 'x') + char char_after_backslash = *buf.position(); + + if (char_after_backslash == 'x') { ++buf.position(); /// escape sequence of the form \xAA @@ -291,7 +293,7 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf) readPODBinary(hex_code, buf); s.push_back(unhex2(hex_code)); } - else if (*buf.position() == 'N') + else if (char_after_backslash == 'N') { /// Support for NULLs: \N sequence must be parsed as empty string. ++buf.position(); @@ -299,7 +301,15 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf) else { /// The usual escape sequence of a single character. - s.push_back(parseEscapeSequence(*buf.position())); + char decoded_char = parseEscapeSequence(char_after_backslash); + + /// For convenience using LIKE and regular expressions, + /// we leave backslash when user write something like 'Hello 100\%': + /// it is parsed like Hello 100\% instead of Hello 100% + if (decoded_char != '\\' && !isControlASCII(decoded_char)) + s.push_back('\\'); + + s.push_back(decoded_char); ++buf.position(); } } diff --git a/src/IO/tests/CMakeLists.txt b/src/IO/tests/CMakeLists.txt index 2767ce6e271..dfbbfa77853 100644 --- a/src/IO/tests/CMakeLists.txt +++ b/src/IO/tests/CMakeLists.txt @@ -37,9 +37,6 @@ target_link_libraries (parse_int_perf2 PRIVATE clickhouse_common_io) add_executable (read_write_int read_write_int.cpp) target_link_libraries (read_write_int PRIVATE clickhouse_common_io) -add_executable (mempbrk mempbrk.cpp) -target_link_libraries (mempbrk PRIVATE clickhouse_common_io) - add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp) target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io) diff --git a/src/IO/tests/mempbrk.cpp b/src/IO/tests/mempbrk.cpp deleted file mode 100644 index 55097d989af..00000000000 --- a/src/IO/tests/mempbrk.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include -#include -#include - -#include - -#include -#include -#include -#include -#include - -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_PARSE_ESCAPE_SEQUENCE; -} -} - - -namespace test -{ -static void readEscapedString(DB::String & s, DB::ReadBuffer & buf) - { - s = ""; - while (!buf.eof()) - { - const char * next_pos = find_first_symbols<'\b', '\f', '\n', '\r', '\t', '\0', '\\'>(buf.position(), buf.buffer().end()); - - s.append(buf.position(), next_pos - buf.position()); - buf.position() += next_pos - buf.position(); - - if (!buf.hasPendingData()) - continue; - - if (*buf.position() == '\t' || *buf.position() == '\n') - return; - - if (*buf.position() == '\\') - { - ++buf.position(); - if (buf.eof()) - throw DB::Exception("Cannot parse escape sequence", DB::ErrorCodes::CANNOT_PARSE_ESCAPE_SEQUENCE); - s += DB::parseEscapeSequence(*buf.position()); - ++buf.position(); - } - } - } -} - - -int main(int, char **) -{ - try - { - DB::ReadBufferFromFileDescriptor in(STDIN_FILENO); -// DB::WriteBufferFromFileDescriptor out(STDOUT_FILENO); - std::string s; - size_t rows = 0; - - Stopwatch watch; - - while (!in.eof()) - { - test::readEscapedString(s, in); - in.ignore(); - - ++rows; - -/* DB::writeEscapedString(s, out); - DB::writeChar('\n', out);*/ - } - - watch.stop(); - std::cerr << std::fixed << std::setprecision(2) - << "Read " << rows << " rows (" << in.count() / 1000000.0 << " MB) in " << watch.elapsedSeconds() << " sec., " - << rows / watch.elapsedSeconds() << " rows/sec. (" << in.count() / watch.elapsedSeconds() / 1000000 << " MB/s.)" - << std::endl; - } - catch (const DB::Exception & e) - { - std::cerr << e.what() << ", " << e.displayText() << std::endl; - return 1; - } - - return 0; -} diff --git a/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference new file mode 100644 index 00000000000..19a5fc680ff --- /dev/null +++ b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.reference @@ -0,0 +1,3 @@ +a\\_\\c\\l\\i\\c\\k\\h\\o\\u\\s a\\_\\c\\l\\i\\c\\k\\h\\o\\u\\s\\e +1 0 1 1 +1 1 0 0 0 diff --git a/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql new file mode 100644 index 00000000000..36ad7b4506c --- /dev/null +++ b/tests/queries/0_stateless/01284_escape_sequences_php_mysql_style.sql @@ -0,0 +1,6 @@ +SELECT 'a\_\c\l\i\c\k\h\o\u\s\e', 'a\\_\\c\\l\\i\\c\\k\\h\\o\\u\\s\\e'; +select 'aXb' like 'a_b', 'aXb' like 'a\_b', 'a_b' like 'a\_b', 'a_b' like 'a\\_b'; +SELECT match('Hello', '\w+'), match('Hello', '\\w+'), match('Hello', '\\\w+'), match('Hello', '\w\+'), match('Hello', 'w+'); + +SELECT match('Hello', '\He\l\l\o'); -- { serverError 427 } +SELECT match('Hello', '\H\e\l\l\o'); -- { serverError 427 } From fb4089ab8e28c2e66ada5556344a225ffc629579 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 27 May 2020 00:48:29 +0300 Subject: [PATCH 04/38] Fix error --- src/IO/ReadHelpers.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 65c3cfab55a..0797e63cdd9 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -306,8 +306,15 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf) /// For convenience using LIKE and regular expressions, /// we leave backslash when user write something like 'Hello 100\%': /// it is parsed like Hello 100\% instead of Hello 100% - if (decoded_char != '\\' && !isControlASCII(decoded_char)) + if (decoded_char != '\\' + && decoded_char != '\'' + && decoded_char != '"' + && decoded_char != '`' /// MySQL style identifiers + && decoded_char != '/' /// JavaScript in HTML + && !isControlASCII(decoded_char)) + { s.push_back('\\'); + } s.push_back(decoded_char); ++buf.position(); From ec479839d93919f6f604cf5ca1fa4546353f7154 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 27 May 2020 22:44:22 +0300 Subject: [PATCH 05/38] fixed --- src/Dictionaries/CacheDictionary.inc.h | 48 +++++++-------- .../cache_strings_default_settings.xml | 35 +++++++++++ .../test_default_string.py | 61 +++++++++++++++++++ 3 files changed, 118 insertions(+), 26 deletions(-) create mode 100644 tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml create mode 100644 tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py diff --git a/src/Dictionaries/CacheDictionary.inc.h b/src/Dictionaries/CacheDictionary.inc.h index 71f3f1857ce..27064d113e6 100644 --- a/src/Dictionaries/CacheDictionary.inc.h +++ b/src/Dictionaries/CacheDictionary.inc.h @@ -302,37 +302,33 @@ void CacheDictionary::getItemsString( /// Request new values sync. /// We have request both cache_not_found_ids and cache_expired_ids. - if (!cache_not_found_ids.empty()) + std::vector required_ids; + required_ids.reserve(cache_not_found_ids.size() + cache_expired_ids.size()); + std::transform( + std::begin(cache_not_found_ids), std::end(cache_not_found_ids), + std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); + std::transform( + std::begin(cache_expired_ids), std::end(cache_expired_ids), + std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); + + auto on_cell_updated = [&] (const auto id, const auto cell_idx) { - std::vector required_ids; - required_ids.reserve(cache_not_found_ids.size() + cache_expired_ids.size()); - std::transform( - std::begin(cache_not_found_ids), std::end(cache_not_found_ids), - std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); - std::transform( - std::begin(cache_expired_ids), std::end(cache_expired_ids), - std::back_inserter(required_ids), [](auto & pair) { return pair.first; }); + const auto attribute_value = attribute_array[cell_idx]; - auto on_cell_updated = [&] (const auto id, const auto cell_idx) - { - const auto attribute_value = attribute_array[cell_idx]; + map[id] = String{attribute_value}; + total_length += (attribute_value.size + 1) * cache_not_found_ids[id].size(); + }; - map[id] = String{attribute_value}; - total_length += (attribute_value.size + 1) * cache_not_found_ids[id].size(); - }; + auto on_id_not_found = [&] (const auto id, const auto) + { + for (const auto row : cache_not_found_ids[id]) + total_length += get_default(row).size + 1; + }; - auto on_id_not_found = [&] (const auto id, const auto) - { - for (const auto row : cache_not_found_ids[id]) - total_length += get_default(row).size + 1; - }; - - auto update_unit_ptr = std::make_shared(required_ids, on_cell_updated, on_id_not_found); - - tryPushToUpdateQueueOrThrow(update_unit_ptr); - waitForCurrentUpdateFinish(update_unit_ptr); - } + auto update_unit_ptr = std::make_shared(required_ids, on_cell_updated, on_id_not_found); + tryPushToUpdateQueueOrThrow(update_unit_ptr); + waitForCurrentUpdateFinish(update_unit_ptr); out->getChars().reserve(total_length); for (const auto row : ext::range(0, ext::size(ids))) diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml b/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml new file mode 100644 index 00000000000..11807bc1ad6 --- /dev/null +++ b/tests/integration/test_dictionary_allow_read_expired_keys/configs/dictionaries/cache_strings_default_settings.xml @@ -0,0 +1,35 @@ + + + default_string + + + dictionary_node + 9000 + default + + test + strings
+ + + + 2 + 1 + + + + 1000 + 10000 + + + + + key + + + value + String + > + + + + \ No newline at end of file diff --git a/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py new file mode 100644 index 00000000000..7d762db2a6d --- /dev/null +++ b/tests/integration/test_dictionary_allow_read_expired_keys/test_default_string.py @@ -0,0 +1,61 @@ +from __future__ import print_function +import pytest +import os +import random +import string +import time + +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +cluster = ClickHouseCluster(__file__, base_configs_dir=os.path.join(SCRIPT_DIR, 'configs')) + +dictionary_node = cluster.add_instance('dictionary_node', stay_alive=True) +main_node = cluster.add_instance('main_node', main_configs=['configs/dictionaries/cache_strings_default_settings.xml']) + + +def get_random_string(string_length=8): + alphabet = string.ascii_letters + string.digits + return ''.join((random.choice(alphabet) for _ in range(string_length))) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + dictionary_node.query("CREATE DATABASE IF NOT EXISTS test;") + dictionary_node.query("DROP TABLE IF EXISTS test.strings;") + dictionary_node.query(""" + CREATE TABLE test.strings + (key UInt64, value String) + ENGINE = Memory; + """) + + values_to_insert = ", ".join(["({}, '{}')".format(1000000 + number, get_random_string()) for number in range(100)]) + dictionary_node.query("INSERT INTO test.strings VALUES {}".format(values_to_insert)) + + yield cluster + finally: + cluster.shutdown() + +# @pytest.mark.skip(reason="debugging") +def test_return_real_values(started_cluster): + assert None != dictionary_node.get_process_pid("clickhouse"), "ClickHouse must be alive" + + first_batch = """ + SELECT count(*) + FROM + ( + SELECT + arrayJoin(arrayMap(x -> (x + 1000000), range(100))) AS id, + dictGetString('default_string', 'value', toUInt64(id)) AS value + ) + WHERE value = ''; + """ + + assert TSV("0") == TSV(main_node.query(first_batch)) + + # Waiting for cache to become expired + time.sleep(5) + + assert TSV("0") == TSV(main_node.query(first_batch)) From 2432ea2c4b5a0307c175f0a5dcc7e95639e1da68 Mon Sep 17 00:00:00 2001 From: Ivan Lezhankin Date: Thu, 28 May 2020 00:58:46 +0300 Subject: [PATCH 06/38] Build universal binary using ya make --- programs/client/CMakeLists.txt | 11 ++--- programs/client/Client.cpp | 5 ++- programs/client/ConnectionParameters.cpp | 2 +- programs/client/config_client.h.in | 3 -- programs/client/readpassphrase/CMakeLists.txt | 12 ++---- .../{includes.h.in => includes.h} | 2 +- .../client/readpassphrase/readpassphrase.c | 16 +------ .../client/readpassphrase/readpassphrase.h | 39 +++++------------ programs/client/readpassphrase/ya.make | 7 +++ programs/main.cpp | 43 +++++++++---------- programs/ya.make | 28 +++++++++++- src/Common/ya.make | 3 ++ 12 files changed, 80 insertions(+), 91 deletions(-) delete mode 100644 programs/client/config_client.h.in rename programs/client/readpassphrase/{includes.h.in => includes.h} (79%) create mode 100644 programs/client/readpassphrase/ya.make diff --git a/programs/client/CMakeLists.txt b/programs/client/CMakeLists.txt index 11ade559a8d..e273123afe0 100644 --- a/programs/client/CMakeLists.txt +++ b/programs/client/CMakeLists.txt @@ -6,14 +6,9 @@ set(CLICKHOUSE_CLIENT_SOURCES set(CLICKHOUSE_CLIENT_LINK PRIVATE clickhouse_common_config clickhouse_functions clickhouse_aggregate_functions clickhouse_common_io clickhouse_parsers string_utils ${Boost_PROGRAM_OPTIONS_LIBRARY}) -include(CheckSymbolExists) -check_symbol_exists(readpassphrase readpassphrase.h HAVE_READPASSPHRASE) -configure_file(config_client.h.in ${ConfigIncludePath}/config_client.h) - -if(NOT HAVE_READPASSPHRASE) - add_subdirectory(readpassphrase) - list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase) -endif() +# Always use internal readpassphrase +add_subdirectory(readpassphrase) +list(APPEND CLICKHOUSE_CLIENT_LINK PRIVATE readpassphrase) clickhouse_program_add(client) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index d6cac7a7b02..afc8f9a72b1 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -39,7 +39,6 @@ #include #include #include -#include #include #include #include @@ -77,6 +76,10 @@ #include #include +#if !defined(ARCADIA_BUILD) +# include +#endif + #ifndef __clang__ #pragma GCC optimize("-fno-var-tracking-assignments") #endif diff --git a/programs/client/ConnectionParameters.cpp b/programs/client/ConnectionParameters.cpp index f0ef3ae5694..d8b4d0f1add 100644 --- a/programs/client/ConnectionParameters.cpp +++ b/programs/client/ConnectionParameters.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include "readpassphrase/readpassphrase.h" namespace DB { diff --git a/programs/client/config_client.h.in b/programs/client/config_client.h.in deleted file mode 100644 index 5ad788ff54c..00000000000 --- a/programs/client/config_client.h.in +++ /dev/null @@ -1,3 +0,0 @@ -#pragma once - -#cmakedefine HAVE_READPASSPHRASE diff --git a/programs/client/readpassphrase/CMakeLists.txt b/programs/client/readpassphrase/CMakeLists.txt index a10b54c377d..dd1bf2c91b9 100644 --- a/programs/client/readpassphrase/CMakeLists.txt +++ b/programs/client/readpassphrase/CMakeLists.txt @@ -1,13 +1,7 @@ - # wget https://raw.githubusercontent.com/openssh/openssh-portable/master/openbsd-compat/readpassphrase.c # wget https://raw.githubusercontent.com/openssh/openssh-portable/master/openbsd-compat/readpassphrase.h -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-unused-result -Wno-reserved-id-macro") +add_library(readpassphrase readpassphrase.c) -configure_file(includes.h.in ${CMAKE_CURRENT_BINARY_DIR}/include/includes.h) -add_library(readpassphrase ${CMAKE_CURRENT_SOURCE_DIR}/readpassphrase.c) -set_target_properties(readpassphrase - PROPERTIES LINKER_LANGUAGE C - ) -# . to allow #include -target_include_directories(readpassphrase PUBLIC . ${CMAKE_CURRENT_BINARY_DIR}/include) +set_target_properties(readpassphrase PROPERTIES LINKER_LANGUAGE C) +target_compile_options(readpassphrase PRIVATE -Wno-unused-result -Wno-reserved-id-macro) diff --git a/programs/client/readpassphrase/includes.h.in b/programs/client/readpassphrase/includes.h similarity index 79% rename from programs/client/readpassphrase/includes.h.in rename to programs/client/readpassphrase/includes.h index 44580d1ed95..3ca5eb2bff8 100644 --- a/programs/client/readpassphrase/includes.h.in +++ b/programs/client/readpassphrase/includes.h @@ -1,6 +1,6 @@ #pragma once -#cmakedefine HAVE_READPASSPHRASE +/* #undef HAVE_READPASSPHRASE */ #if !defined(HAVE_READPASSPHRASE) # ifndef _PATH_TTY diff --git a/programs/client/readpassphrase/readpassphrase.c b/programs/client/readpassphrase/readpassphrase.c index 243701239bf..21b48e7efc3 100644 --- a/programs/client/readpassphrase/readpassphrase.c +++ b/programs/client/readpassphrase/readpassphrase.c @@ -25,13 +25,11 @@ #include "includes.h" -#ifndef HAVE_READPASSPHRASE - #include #include #include #include -#include +#include "readpassphrase.h" #include #include #include @@ -193,19 +191,7 @@ restart: } //DEF_WEAK(readpassphrase); -#if 0 -char * -getpass(const char *prompt) -{ - static char buf[_PASSWORD_LEN + 1]; - - return(readpassphrase(prompt, buf, sizeof(buf), RPP_ECHO_OFF)); -} -#endif - static void handler(int s) { - signo[s] = 1; } -#endif /* HAVE_READPASSPHRASE */ diff --git a/programs/client/readpassphrase/readpassphrase.h b/programs/client/readpassphrase/readpassphrase.h index 0782a1773ea..399eb7b062c 100644 --- a/programs/client/readpassphrase/readpassphrase.h +++ b/programs/client/readpassphrase/readpassphrase.h @@ -23,39 +23,22 @@ /* OPENBSD ORIGINAL: include/readpassphrase.h */ #pragma once -// #ifndef _READPASSPHRASE_H_ -// #define _READPASSPHRASE_H_ -//#include "includes.h" -#include "config_client.h" - -// Should not be included on BSD systems, but if it happen... -#ifdef HAVE_READPASSPHRASE -# include_next +#if defined(__cplusplus) +extern "C" { #endif -#ifndef HAVE_READPASSPHRASE -# ifdef __cplusplus -extern "C" { -# endif - - -# define RPP_ECHO_OFF 0x00 /* Turn off echo (default). */ -# define RPP_ECHO_ON 0x01 /* Leave echo on. */ -# define RPP_REQUIRE_TTY 0x02 /* Fail if there is no tty. */ -# define RPP_FORCELOWER 0x04 /* Force input to lower case. */ -# define RPP_FORCEUPPER 0x08 /* Force input to upper case. */ -# define RPP_SEVENBIT 0x10 /* Strip the high bit from input. */ -# define RPP_STDIN 0x20 /* Read from stdin, not /dev/tty */ +#define RPP_ECHO_OFF 0x00 /* Turn off echo (default). */ +#define RPP_ECHO_ON 0x01 /* Leave echo on. */ +#define RPP_REQUIRE_TTY 0x02 /* Fail if there is no tty. */ +#define RPP_FORCELOWER 0x04 /* Force input to lower case. */ +#define RPP_FORCEUPPER 0x08 /* Force input to upper case. */ +#define RPP_SEVENBIT 0x10 /* Strip the high bit from input. */ +#define RPP_STDIN 0x20 /* Read from stdin, not /dev/tty */ char * readpassphrase(const char *, char *, size_t, int); -# ifdef __cplusplus +#if defined(__cplusplus) } -# endif - - -#endif /* HAVE_READPASSPHRASE */ - -// #endif /* !_READPASSPHRASE_H_ */ +#endif diff --git a/programs/client/readpassphrase/ya.make b/programs/client/readpassphrase/ya.make new file mode 100644 index 00000000000..80ad197e5d4 --- /dev/null +++ b/programs/client/readpassphrase/ya.make @@ -0,0 +1,7 @@ +LIBRARY() + +SRCS( + readpassphrase.c +) + +END() diff --git a/programs/main.cpp b/programs/main.cpp index 2eb226d3c00..6b34504ce37 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -8,11 +8,8 @@ #include #include /// pair -#if __has_include("config_tools.h") -#include "config_tools.h" -#endif -#if __has_include("config_core.h") -#include "config_core.h" +#if !defined(ARCADIA_BUILD) +# include "config_tools.h" #endif #include @@ -22,31 +19,31 @@ /// Universal executable for various clickhouse applications -#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER) +#if ENABLE_CLICKHOUSE_SERVER int mainEntryClickHouseServer(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT) +#if ENABLE_CLICKHOUSE_CLIENT int mainEntryClickHouseClient(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL) +#if ENABLE_CLICKHOUSE_LOCAL int mainEntryClickHouseLocal(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK) +#if ENABLE_CLICKHOUSE_BENCHMARK int mainEntryClickHouseBenchmark(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG int mainEntryClickHouseExtractFromConfig(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR) +#if ENABLE_CLICKHOUSE_COMPRESSOR int mainEntryClickHouseCompressor(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT) +#if ENABLE_CLICKHOUSE_FORMAT int mainEntryClickHouseFormat(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER) +#if ENABLE_CLICKHOUSE_COPIER int mainEntryClickHouseClusterCopier(int argc, char ** argv); #endif -#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR) +#if ENABLE_CLICKHOUSE_OBFUSCATOR int mainEntryClickHouseObfuscator(int argc, char ** argv); #endif @@ -60,31 +57,31 @@ using MainFunc = int (*)(int, char**); /// Add an item here to register new application std::pair clickhouse_applications[] = { -#if ENABLE_CLICKHOUSE_LOCAL || !defined(ENABLE_CLICKHOUSE_LOCAL) +#if ENABLE_CLICKHOUSE_LOCAL {"local", mainEntryClickHouseLocal}, #endif -#if ENABLE_CLICKHOUSE_CLIENT || !defined(ENABLE_CLICKHOUSE_CLIENT) +#if ENABLE_CLICKHOUSE_CLIENT {"client", mainEntryClickHouseClient}, #endif -#if ENABLE_CLICKHOUSE_BENCHMARK || !defined(ENABLE_CLICKHOUSE_BENCHMARK) +#if ENABLE_CLICKHOUSE_BENCHMARK {"benchmark", mainEntryClickHouseBenchmark}, #endif -#if ENABLE_CLICKHOUSE_SERVER || !defined(ENABLE_CLICKHOUSE_SERVER) +#if ENABLE_CLICKHOUSE_SERVER {"server", mainEntryClickHouseServer}, #endif -#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG || !defined(ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG) +#if ENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG {"extract-from-config", mainEntryClickHouseExtractFromConfig}, #endif -#if ENABLE_CLICKHOUSE_COMPRESSOR || !defined(ENABLE_CLICKHOUSE_COMPRESSOR) +#if ENABLE_CLICKHOUSE_COMPRESSOR {"compressor", mainEntryClickHouseCompressor}, #endif -#if ENABLE_CLICKHOUSE_FORMAT || !defined(ENABLE_CLICKHOUSE_FORMAT) +#if ENABLE_CLICKHOUSE_FORMAT {"format", mainEntryClickHouseFormat}, #endif -#if ENABLE_CLICKHOUSE_COPIER || !defined(ENABLE_CLICKHOUSE_COPIER) +#if ENABLE_CLICKHOUSE_COPIER {"copier", mainEntryClickHouseClusterCopier}, #endif -#if ENABLE_CLICKHOUSE_OBFUSCATOR || !defined(ENABLE_CLICKHOUSE_OBFUSCATOR) +#if ENABLE_CLICKHOUSE_OBFUSCATOR {"obfuscator", mainEntryClickHouseObfuscator}, #endif }; diff --git a/programs/ya.make b/programs/ya.make index 6c773c312b8..f4a61850212 100644 --- a/programs/ya.make +++ b/programs/ya.make @@ -1,3 +1,27 @@ -RECURSE( - server +PROGRAM(clickhouse) + +CFLAGS( + -DENABLE_CLICKHOUSE_CLIENT + -DENABLE_CLICKHOUSE_EXTRACT_FROM_CONFIG + -DENABLE_CLICKHOUSE_SERVER ) + +PEERDIR( + clickhouse/base/daemon + clickhouse/base/loggers + clickhouse/programs/client/readpassphrase + clickhouse/src +) + +SRCS( + main.cpp + + client/Client.cpp + client/ConnectionParameters.cpp + client/Suggest.cpp + extract-from-config/ExtractFromConfig.cpp + server/Server.cpp + server/MetricsTransmitter.cpp +) + +END() diff --git a/src/Common/ya.make b/src/Common/ya.make index c0178f3d310..99659aa2aa0 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -25,6 +25,9 @@ CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0) CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0) CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\") CFLAGS (GLOBAL -DVERSION_INTEGER=0) +CFLAGS (GLOBAL -DVERSION_MAJOR=0) +CFLAGS (GLOBAL -DVERSION_MINOR=0) +CFLAGS (GLOBAL -DVERSION_PATCH=0) CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\") CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\") CFLAGS (GLOBAL -DVERSION_REVISION=0) From 98172deffc81b1bd962ae5213aa3e62dbf031e07 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Wed, 20 May 2020 23:31:55 +0300 Subject: [PATCH 07/38] Show authentication type in table system.users and while executing SHOW CREATE USER query. --- src/Access/Authentication.cpp | 12 +- src/Access/Authentication.h | 61 +++++++- ...InterpreterShowCreateAccessEntityQuery.cpp | 4 +- src/Parsers/ASTCreateUserQuery.cpp | 68 ++++++--- src/Parsers/ASTCreateUserQuery.h | 5 +- src/Parsers/ParserCreateUserQuery.cpp | 138 +++++++----------- src/Parsers/ParserCreateUserQuery.h | 4 +- src/Storages/System/StorageSystemUsers.cpp | 23 ++- .../test_disk_access_storage/test.py | 4 +- .../integration/test_grant_and_revoke/test.py | 6 +- 10 files changed, 207 insertions(+), 118 deletions(-) diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp index f435d6e6336..cb2c9e7a256 100644 --- a/src/Access/Authentication.cpp +++ b/src/Access/Authentication.cpp @@ -7,8 +7,8 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; } @@ -36,8 +36,11 @@ Authentication::Digest Authentication::getPasswordDoubleSHA1() const case DOUBLE_SHA1_PASSWORD: return password_hash; + + case MAX_TYPE: + break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("getPasswordDoubleSHA1(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } @@ -71,8 +74,11 @@ bool Authentication::isCorrectPassword(const String & password_) const return encodeSHA1(first_sha1) == password_hash; } + + case MAX_TYPE: + break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("Cannot check if the password is correct for authentication type " + toString(type), ErrorCodes::NOT_IMPLEMENTED); } } diff --git a/src/Access/Authentication.h b/src/Access/Authentication.h index 3f16dc56de3..c410a101cdd 100644 --- a/src/Access/Authentication.h +++ b/src/Access/Authentication.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -14,6 +15,7 @@ namespace ErrorCodes extern const int SUPPORT_IS_DISABLED; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; } @@ -35,6 +37,15 @@ public: /// SHA1(SHA1(password)). /// This kind of hash is used by the `mysql_native_password` authentication plugin. DOUBLE_SHA1_PASSWORD, + + MAX_TYPE, + }; + + struct TypeInfo + { + const char * const raw_name; + const String name; /// Lowercased with underscores, e.g. "sha256_password". + static const TypeInfo & get(Type type_); }; using Digest = std::vector; @@ -85,6 +96,48 @@ private: }; +inline const Authentication::TypeInfo & Authentication::TypeInfo::get(Type type_) +{ + static constexpr auto make_info = [](const char * raw_name_) + { + String init_name = raw_name_; + boost::to_lower(init_name); + return TypeInfo{raw_name_, std::move(init_name)}; + }; + + switch (type_) + { + case NO_PASSWORD: + { + static const auto info = make_info("NO_PASSWORD"); + return info; + } + case PLAINTEXT_PASSWORD: + { + static const auto info = make_info("PLAINTEXT_PASSWORD"); + return info; + } + case SHA256_PASSWORD: + { + static const auto info = make_info("SHA256_PASSWORD"); + return info; + } + case DOUBLE_SHA1_PASSWORD: + { + static const auto info = make_info("DOUBLE_SHA1_PASSWORD"); + return info; + } + case MAX_TYPE: break; + } + throw Exception("Unknown authentication type: " + std::to_string(static_cast(type_)), ErrorCodes::LOGICAL_ERROR); +} + +inline String toString(Authentication::Type type_) +{ + return Authentication::TypeInfo::get(type_).raw_name; +} + + inline Authentication::Digest Authentication::encodeSHA256(const std::string_view & text [[maybe_unused]]) { #if USE_SSL @@ -122,8 +175,10 @@ inline void Authentication::setPassword(const String & password_) case DOUBLE_SHA1_PASSWORD: return setPasswordHashBinary(encodeDoubleSHA1(password_)); + + case MAX_TYPE: break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("setPassword(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } @@ -186,8 +241,10 @@ inline void Authentication::setPasswordHashBinary(const Digest & hash) password_hash = hash; return; } + + case MAX_TYPE: break; } - throw Exception("Unknown authentication type: " + std::to_string(static_cast(type)), ErrorCodes::LOGICAL_ERROR); + throw Exception("setPasswordHashBinary(): authentication type " + toString(type) + " not supported", ErrorCodes::NOT_IMPLEMENTED); } } diff --git a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp index 87e1265f793..e37c31aab22 100644 --- a/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/InterpreterShowCreateAccessEntityQuery.cpp @@ -56,10 +56,10 @@ namespace query->default_roles = user.default_roles.toASTWithNames(*manager); } - if (attach_mode && (user.authentication.getType() != Authentication::NO_PASSWORD)) + if (user.authentication.getType() != Authentication::NO_PASSWORD) { - /// We don't show password unless it's an ATTACH statement. query->authentication = user.authentication; + query->show_password = attach_mode; /// We don't show password unless it's an ATTACH statement. } if (!user.settings.empty()) diff --git a/src/Parsers/ASTCreateUserQuery.cpp b/src/Parsers/ASTCreateUserQuery.cpp index c8e2a76dfa2..e5c1178285b 100644 --- a/src/Parsers/ASTCreateUserQuery.cpp +++ b/src/Parsers/ASTCreateUserQuery.cpp @@ -6,6 +6,12 @@ namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + + namespace { void formatRenameTo(const String & new_name, const IAST::FormatSettings & settings) @@ -15,27 +21,51 @@ namespace } - void formatAuthentication(const Authentication & authentication, const IAST::FormatSettings & settings) + void formatAuthentication(const Authentication & authentication, bool show_password, const IAST::FormatSettings & settings) { - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << (settings.hilite ? IAST::hilite_none : ""); - switch (authentication.getType()) + auto authentication_type = authentication.getType(); + if (authentication_type == Authentication::NO_PASSWORD) { - case Authentication::Type::NO_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "no_password" << (settings.hilite ? IAST::hilite_none : ""); - break; - case Authentication::Type::PLAINTEXT_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "plaintext_password BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(authentication.getPassword()); - break; - case Authentication::Type::SHA256_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "sha256_hash BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(authentication.getPasswordHashHex()); - break; - case Authentication::Type::DOUBLE_SHA1_PASSWORD: - settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << "double_sha1_hash BY " << (settings.hilite ? IAST::hilite_none : "") - << quoteString(authentication.getPasswordHashHex()); - break; + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " NOT IDENTIFIED" + << (settings.hilite ? IAST::hilite_none : ""); + return; } + + String authentication_type_name = Authentication::TypeInfo::get(authentication_type).name; + std::optional password; + + if (show_password) + { + switch (authentication_type) + { + case Authentication::PLAINTEXT_PASSWORD: + { + password = authentication.getPassword(); + break; + } + case Authentication::SHA256_PASSWORD: + { + authentication_type_name = "sha256_hash"; + password = authentication.getPasswordHashHex(); + break; + } + case Authentication::DOUBLE_SHA1_PASSWORD: + { + authentication_type_name = "double_sha1_hash"; + password = authentication.getPasswordHashHex(); + break; + } + + case Authentication::NO_PASSWORD: [[fallthrough]]; + case Authentication::MAX_TYPE: + throw Exception("AST: Unexpected authentication type " + toString(authentication_type), ErrorCodes::LOGICAL_ERROR); + } + } + + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " IDENTIFIED WITH " << authentication_type_name + << (settings.hilite ? IAST::hilite_none : ""); + if (password) + settings.ostr << (settings.hilite ? IAST::hilite_keyword : "") << " BY " << quoteString(*password); } @@ -190,7 +220,7 @@ void ASTCreateUserQuery::formatImpl(const FormatSettings & format, FormatState & formatRenameTo(new_name, format); if (authentication) - formatAuthentication(*authentication, format); + formatAuthentication(*authentication, show_password, format); if (hosts) formatHosts(nullptr, *hosts, format); diff --git a/src/Parsers/ASTCreateUserQuery.h b/src/Parsers/ASTCreateUserQuery.h index 54dc51d783b..28ef6c059da 100644 --- a/src/Parsers/ASTCreateUserQuery.h +++ b/src/Parsers/ASTCreateUserQuery.h @@ -12,14 +12,14 @@ class ASTExtendedRoleSet; class ASTSettingsProfileElements; /** CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...]] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] @@ -38,6 +38,7 @@ public: String new_name; std::optional authentication; + bool show_password = true; /// formatImpl() will show the password or hash. std::optional hosts; std::optional add_hosts; diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index 76a06a0282f..2f73e7fa75a 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -35,100 +35,74 @@ namespace } - bool parseByPassword(IParserBase::Pos & pos, Expected & expected, String & password) - { - return IParserBase::wrapParseImpl(pos, [&] - { - if (!ParserKeyword{"BY"}.ignore(pos, expected)) - return false; - - ASTPtr ast; - if (!ParserStringLiteral{}.parse(pos, ast, expected)) - return false; - - password = ast->as().value.safeGet(); - return true; - }); - } - - bool parseAuthentication(IParserBase::Pos & pos, Expected & expected, std::optional & authentication) { return IParserBase::wrapParseImpl(pos, [&] { + if (ParserKeyword{"NOT IDENTIFIED"}.ignore(pos, expected)) + { + authentication = Authentication{Authentication::NO_PASSWORD}; + return true; + } + if (!ParserKeyword{"IDENTIFIED"}.ignore(pos, expected)) return false; - if (!ParserKeyword{"WITH"}.ignore(pos, expected)) + std::optional type; + bool expect_password = false; + bool expect_hash = false; + + if (ParserKeyword{"WITH"}.ignore(pos, expected)) { - String password; - if (!parseByPassword(pos, expected, password)) + for (auto check_type : ext::range(Authentication::MAX_TYPE)) + { + if (ParserKeyword{Authentication::TypeInfo::get(check_type).raw_name}.ignore(pos, expected)) + { + type = check_type; + expect_password = (check_type != Authentication::NO_PASSWORD); + break; + } + } + + if (!type) + { + if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected)) + { + type = Authentication::SHA256_PASSWORD; + expect_hash = true; + } + else if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected)) + { + type = Authentication::DOUBLE_SHA1_PASSWORD; + expect_hash = true; + } + else + return false; + } + } + + if (!type) + { + type = Authentication::SHA256_PASSWORD; + expect_password = true; + } + + String password; + if (expect_password || expect_hash) + { + ASTPtr ast; + if (!ParserKeyword{"BY"}.ignore(pos, expected) || !ParserStringLiteral{}.parse(pos, ast, expected)) return false; - authentication = Authentication{Authentication::SHA256_PASSWORD}; + password = ast->as().value.safeGet(); + } + + authentication = Authentication{*type}; + if (expect_password) authentication->setPassword(password); - return true; - } + else if (expect_hash) + authentication->setPasswordHashHex(password); - if (ParserKeyword{"PLAINTEXT_PASSWORD"}.ignore(pos, expected)) - { - String password; - if (!parseByPassword(pos, expected, password)) - return false; - - authentication = Authentication{Authentication::PLAINTEXT_PASSWORD}; - authentication->setPassword(password); - return true; - } - - if (ParserKeyword{"SHA256_PASSWORD"}.ignore(pos, expected)) - { - String password; - if (!parseByPassword(pos, expected, password)) - return false; - - authentication = Authentication{Authentication::SHA256_PASSWORD}; - authentication->setPassword(password); - return true; - } - - if (ParserKeyword{"SHA256_HASH"}.ignore(pos, expected)) - { - String hash; - if (!parseByPassword(pos, expected, hash)) - return false; - - authentication = Authentication{Authentication::SHA256_PASSWORD}; - authentication->setPasswordHashHex(hash); - return true; - } - - if (ParserKeyword{"DOUBLE_SHA1_PASSWORD"}.ignore(pos, expected)) - { - String password; - if (!parseByPassword(pos, expected, password)) - return false; - - authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD}; - authentication->setPassword(password); - return true; - } - - if (ParserKeyword{"DOUBLE_SHA1_HASH"}.ignore(pos, expected)) - { - String hash; - if (!parseByPassword(pos, expected, hash)) - return false; - - authentication = Authentication{Authentication::DOUBLE_SHA1_PASSWORD}; - authentication->setPasswordHashHex(hash); - return true; - } - - if (!ParserKeyword{"NO_PASSWORD"}.ignore(pos, expected)) - return false; - - authentication = Authentication{Authentication::NO_PASSWORD}; return true; }); } diff --git a/src/Parsers/ParserCreateUserQuery.h b/src/Parsers/ParserCreateUserQuery.h index d609894a7ec..2a890f41060 100644 --- a/src/Parsers/ParserCreateUserQuery.h +++ b/src/Parsers/ParserCreateUserQuery.h @@ -7,13 +7,13 @@ namespace DB { /** Parses queries like * CREATE USER [IF NOT EXISTS | OR REPLACE] name - * [IDENTIFIED [WITH {NO_PASSWORD|PLAINTEXT_PASSWORD|SHA256_PASSWORD|SHA256_HASH|DOUBLE_SHA1_PASSWORD|DOUBLE_SHA1_HASH}] BY {'password'|'hash'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}] * [HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] * * ALTER USER [IF EXISTS] name * [RENAME TO new_name] - * [IDENTIFIED [WITH {PLAINTEXT_PASSWORD|SHA256_PASSWORD|DOUBLE_SHA1_PASSWORD}] BY {'password'|'hash'}] + * [NOT IDENTIFIED | IDENTIFIED [WITH {no_password|plaintext_password|sha256_password|sha256_hash|double_sha1_password|double_sha1_hash}] BY {'password'|'hash'}] * [[ADD|DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE] * [SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY|WRITABLE] | PROFILE 'profile_name'] [,...] */ diff --git a/src/Storages/System/StorageSystemUsers.cpp b/src/Storages/System/StorageSystemUsers.cpp index d0e042d054f..e0755fe59ab 100644 --- a/src/Storages/System/StorageSystemUsers.cpp +++ b/src/Storages/System/StorageSystemUsers.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -15,12 +16,26 @@ namespace DB { +namespace +{ + DataTypeEnum8::Values getAuthenticationTypeEnumValues() + { + DataTypeEnum8::Values enum_values; + for (auto type : ext::range(Authentication::MAX_TYPE)) + enum_values.emplace_back(Authentication::TypeInfo::get(type).name, static_cast(type)); + return enum_values; + } +} + + NamesAndTypesList StorageSystemUsers::getNamesAndTypes() { NamesAndTypesList names_and_types{ {"name", std::make_shared()}, {"id", std::make_shared()}, {"storage", std::make_shared()}, + {"auth_type", std::make_shared(getAuthenticationTypeEnumValues())}, + {"auth_params", std::make_shared(std::make_shared())}, {"host_ip", std::make_shared(std::make_shared())}, {"host_names", std::make_shared(std::make_shared())}, {"host_names_regexp", std::make_shared(std::make_shared())}, @@ -43,6 +58,9 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & auto & column_name = assert_cast(*res_columns[column_index++]); auto & column_id = assert_cast(*res_columns[column_index++]).getData(); auto & column_storage = assert_cast(*res_columns[column_index++]); + auto & column_auth_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_auth_params = assert_cast(assert_cast(*res_columns[column_index]).getData()); + auto & column_auth_params_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_ip = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_host_ip_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_host_names = assert_cast(assert_cast(*res_columns[column_index]).getData()); @@ -60,12 +78,15 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & auto add_row = [&](const String & name, const UUID & id, const String & storage_name, + const Authentication & authentication, const AllowedClientHosts & allowed_hosts, const ExtendedRoleSet & default_roles) { column_name.insertData(name.data(), name.length()); column_id.push_back(id); column_storage.insertData(storage_name.data(), storage_name.length()); + column_auth_type.push_back(static_cast(authentication.getType())); + column_auth_params_offsets.push_back(column_auth_params.size()); if (allowed_hosts.containsAnyHost()) { @@ -128,7 +149,7 @@ void StorageSystemUsers::fillData(MutableColumns & res_columns, const Context & if (!storage) continue; - add_row(user->getName(), id, storage->getStorageName(), user->allowed_client_hosts, user->default_roles); + add_row(user->getName(), id, storage->getStorageName(), user->authentication, user->allowed_client_hosts, user->default_roles); } } diff --git a/tests/integration/test_disk_access_storage/test.py b/tests/integration/test_disk_access_storage/test.py index 315440b4358..a47af5ad5b8 100644 --- a/tests/integration/test_disk_access_storage/test.py +++ b/tests/integration/test_disk_access_storage/test.py @@ -39,7 +39,7 @@ def test_create(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE rx\n" + assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE rx\n" assert instance.query("SHOW CREATE ROW POLICY p ON mydb.mytable") == "CREATE ROW POLICY p ON mydb.mytable FOR SELECT USING a < 1000 TO u1, u2\n" assert instance.query("SHOW CREATE QUOTA q") == "CREATE QUOTA q KEYED BY \\'none\\' FOR INTERVAL 1 HOUR MAX QUERIES 100 TO ALL EXCEPT rx\n" assert instance.query("SHOW GRANTS FOR u1") == "" @@ -69,7 +69,7 @@ def test_alter(): def check(): assert instance.query("SHOW CREATE USER u1") == "CREATE USER u1 SETTINGS PROFILE s1\n" - assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 HOST LOCAL DEFAULT ROLE ry\n" + assert instance.query("SHOW CREATE USER u2") == "CREATE USER u2 IDENTIFIED WITH sha256_password HOST LOCAL DEFAULT ROLE ry\n" assert instance.query("SHOW GRANTS FOR u1") == "GRANT SELECT ON mydb.mytable TO u1\n" assert instance.query("SHOW GRANTS FOR u2") == "GRANT rx, ry TO u2\n" assert instance.query("SHOW CREATE ROLE rx") == "CREATE ROLE rx SETTINGS PROFILE s2\n" diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index 8df68547f38..7054ce28e59 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -155,9 +155,9 @@ def test_introspection(): assert instance.query("SHOW ENABLED ROLES", user='A') == TSV([[ "R1", 0, 1, 1 ]]) assert instance.query("SHOW ENABLED ROLES", user='B') == TSV([[ "R2", 1, 1, 1 ]]) - assert instance.query("SELECT name, storage, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ - TSV([[ "A", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], - [ "B", "disk", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) + assert instance.query("SELECT name, storage, auth_type, auth_params, host_ip, host_names, host_names_regexp, host_names_like, default_roles_all, default_roles_list, default_roles_except from system.users WHERE name IN ('A', 'B') ORDER BY name") ==\ + TSV([[ "A", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ], + [ "B", "disk", "no_password", "[]", "['::/0']", "[]", "[]", "[]", 1, "[]", "[]" ]]) assert instance.query("SELECT name, storage from system.roles WHERE name IN ('R1', 'R2') ORDER BY name") ==\ TSV([[ "R1", "disk" ], From bc773be0e4dab18c45e67e053f82457e5c107534 Mon Sep 17 00:00:00 2001 From: Ilya Date: Thu, 28 May 2020 13:30:29 +0300 Subject: [PATCH 08/38] Fix package name in docker file: tzata -> tzdata --- docker/server/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 3ad20a9479c..93f192c3f3c 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -21,7 +21,7 @@ RUN apt-get update \ locales \ ca-certificates \ wget \ - tzata \ + tzdata \ && rm -rf \ /var/lib/apt/lists/* \ /var/cache/debconf \ From 4f6b12457b8b390823765912d43fad98846e9506 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Thu, 28 May 2020 13:53:40 +0300 Subject: [PATCH 09/38] trigger From b71d520a3f85f1d6474d5cec88b57404f03521c2 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 28 May 2020 15:30:42 +0300 Subject: [PATCH 10/38] Fix race for description in MemoryTracker. --- src/Common/MemoryTracker.cpp | 6 ++++-- src/Common/MemoryTracker.h | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index 082b78682a2..16d166c191b 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -49,11 +49,13 @@ MemoryTracker::~MemoryTracker() void MemoryTracker::logPeakMemoryUsage() const { + const auto * description = description_ptr.load(std::memory_order_relaxed); LOG_DEBUG(&Logger::get("MemoryTracker"), "Peak memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(peak)); } void MemoryTracker::logMemoryUsage(Int64 current) const { + const auto * description = description_ptr.load(std::memory_order_relaxed); LOG_DEBUG(&Logger::get("MemoryTracker"), "Current memory usage{}: {}.", (description ? " " + std::string(description) : ""), formatReadableSizeWithBinarySuffix(current)); } @@ -85,7 +87,7 @@ void MemoryTracker::alloc(Int64 size) std::stringstream message; message << "Memory tracker"; - if (description) + if (const auto * description = description_ptr.load(std::memory_order_relaxed)) message << " " << description; message << ": fault injected. Would use " << formatReadableSizeWithBinarySuffix(will_be) << " (attempt to allocate chunk of " << size << " bytes)" @@ -117,7 +119,7 @@ void MemoryTracker::alloc(Int64 size) std::stringstream message; message << "Memory limit"; - if (description) + if (const auto * description = description_ptr.load(std::memory_order_relaxed)) message << " " << description; message << " exceeded: would use " << formatReadableSizeWithBinarySuffix(will_be) << " (attempt to allocate chunk of " << size << " bytes)" diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index 23f32b051b2..8af683ae790 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -35,7 +35,7 @@ private: CurrentMetrics::Metric metric = CurrentMetrics::end(); /// This description will be used as prefix into log messages (if isn't nullptr) - const char * description = nullptr; + std::atomic description_ptr = nullptr; void updatePeak(Int64 will_be); void logMemoryUsage(Int64 current) const; @@ -114,9 +114,9 @@ public: metric = metric_; } - void setDescription(const char * description_) + void setDescription(const char * description) { - description = description_; + description_ptr.store(description, std::memory_order_relaxed); } /// Reset the accumulated data From f5b1ab8e4a7556644f3d2598ef108ecd635abe37 Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Thu, 28 May 2020 16:46:56 +0300 Subject: [PATCH 11/38] Use proper versions in Arcadia build. --- cmake/autogenerated_versions.txt | 9 +++++++++ cmake/version.cmake | 10 +--------- cmake/yandex/ya.make.versions.inc | 22 ++++++++++++++++++++++ src/Common/ya.make | 15 +-------------- utils/release/release_lib.sh | 28 ++++++++++++++-------------- 5 files changed, 47 insertions(+), 37 deletions(-) create mode 100644 cmake/autogenerated_versions.txt create mode 100644 cmake/yandex/ya.make.versions.inc diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt new file mode 100644 index 00000000000..72959d3c1d8 --- /dev/null +++ b/cmake/autogenerated_versions.txt @@ -0,0 +1,9 @@ +# This strings autochanged from release_lib.sh: +SET(VERSION_REVISION 54435) +SET(VERSION_MAJOR 20) +SET(VERSION_MINOR 5) +SET(VERSION_PATCH 1) +SET(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f) +SET(VERSION_DESCRIBE v20.5.1.1-prestable) +SET(VERSION_STRING 20.5.1.1) +# end of autochange diff --git a/cmake/version.cmake b/cmake/version.cmake index 3f51f59cf0f..eea17f68c47 100644 --- a/cmake/version.cmake +++ b/cmake/version.cmake @@ -1,12 +1,4 @@ -# This strings autochanged from release_lib.sh: -set(VERSION_REVISION 54435) -set(VERSION_MAJOR 20) -set(VERSION_MINOR 5) -set(VERSION_PATCH 1) -set(VERSION_GITHASH 91df18a906dcffdbee6816e5389df6c65f86e35f) -set(VERSION_DESCRIBE v20.5.1.1-prestable) -set(VERSION_STRING 20.5.1.1) -# end of autochange +include(${CMAKE_SOURCE_DIR}/cmake/autogenerated_versions.txt) set(VERSION_EXTRA "" CACHE STRING "") set(VERSION_TWEAK "" CACHE STRING "") diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc new file mode 100644 index 00000000000..5c3c5eabb60 --- /dev/null +++ b/cmake/yandex/ya.make.versions.inc @@ -0,0 +1,22 @@ +INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/autogenerated_versions.txt) + +# TODO: not sure if this is customizable per-binary +SET(VERSION_NAME "ClickHouse") + +# TODO: not quite sure how to replace dash with space in ya.make +SET(VERSION_FULL "${VERSION_NAME}-${VERSION_STRING}") + +CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\") +CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR}) +CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR}) +CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH}) +CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\") + +# TODO: not supported yet, not sure if ya.make supports arithmetics. +CFLAGS (GLOBAL -DVERSION_INTEGER=0) + +CFLAGS (GLOBAL -DVERSION_NAME=\"\\\"${VERSION_NAME}\\\"\") +CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"-arcadia\") +CFLAGS (GLOBAL -DVERSION_REVISION=${VERSION_REVISION}) +CFLAGS (GLOBAL -DVERSION_STRING=\"\\\"${VERSION_STRING}\\\"\") + diff --git a/src/Common/ya.make b/src/Common/ya.make index 99659aa2aa0..83a419212bd 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -18,20 +18,7 @@ PEERDIR( contrib/restricted/ryu ) -# TODO: stub for config_version.h -CFLAGS (GLOBAL -DDBMS_NAME=\"ClickHouse\") -CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=0) -CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=0) -CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=0) -CFLAGS (GLOBAL -DVERSION_FULL=\"ClickHouse\") -CFLAGS (GLOBAL -DVERSION_INTEGER=0) -CFLAGS (GLOBAL -DVERSION_MAJOR=0) -CFLAGS (GLOBAL -DVERSION_MINOR=0) -CFLAGS (GLOBAL -DVERSION_PATCH=0) -CFLAGS (GLOBAL -DVERSION_NAME=\"ClickHouse\") -CFLAGS (GLOBAL -DVERSION_OFFICIAL=\"\\\(arcadia\\\)\") -CFLAGS (GLOBAL -DVERSION_REVISION=0) -CFLAGS (GLOBAL -DVERSION_STRING=\"Unknown\") +INCLUDE(${ARCADIA_ROOT}/clickhouse/cmake/yandex/ya.make.versions.inc) SRCS( ActionLock.cpp diff --git a/utils/release/release_lib.sh b/utils/release/release_lib.sh index 2e700593523..569e1dfb9f2 100644 --- a/utils/release/release_lib.sh +++ b/utils/release/release_lib.sh @@ -12,10 +12,10 @@ function gen_version_string { function get_version { if [ -z "$VERSION_MAJOR" ] && [ -z "$VERSION_MINOR" ] && [ -z "$VERSION_PATCH" ]; then BASEDIR=$(dirname "${BASH_SOURCE[0]}")/../../ - VERSION_REVISION=`grep "set(VERSION_REVISION" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'` - VERSION_MAJOR=`grep "set(VERSION_MAJOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'` - VERSION_MINOR=`grep "set(VERSION_MINOR" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'` - VERSION_PATCH=`grep "set(VERSION_PATCH" ${BASEDIR}/cmake/version.cmake | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_REVISION=`grep "SET(VERSION_REVISION" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_REVISION \(.*\)$/\1/' | sed 's/[) ].*//'` + VERSION_MAJOR=`grep "SET(VERSION_MAJOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MAJOR \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_MINOR=`grep "SET(VERSION_MINOR" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_MINOR \(.*\)/\1/' | sed 's/[) ].*//'` + VERSION_PATCH=`grep "SET(VERSION_PATCH" ${BASEDIR}/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_PATCH \(.*\)/\1/' | sed 's/[) ].*//'` fi VERSION_PREFIX="${VERSION_PREFIX:-v}" VERSION_POSTFIX_TAG="${VERSION_POSTFIX:--testing}" @@ -90,28 +90,28 @@ function gen_revision_author { git_describe=`git describe` git_hash=`git rev-parse HEAD` - sed -i -e "s/set(VERSION_REVISION [^) ]*/set(VERSION_REVISION $VERSION_REVISION/g;" \ - -e "s/set(VERSION_DESCRIBE [^) ]*/set(VERSION_DESCRIBE $git_describe/g;" \ - -e "s/set(VERSION_GITHASH [^) ]*/set(VERSION_GITHASH $git_hash/g;" \ - -e "s/set(VERSION_MAJOR [^) ]*/set(VERSION_MAJOR $VERSION_MAJOR/g;" \ - -e "s/set(VERSION_MINOR [^) ]*/set(VERSION_MINOR $VERSION_MINOR/g;" \ - -e "s/set(VERSION_PATCH [^) ]*/set(VERSION_PATCH $VERSION_PATCH/g;" \ - -e "s/set(VERSION_STRING [^) ]*/set(VERSION_STRING $VERSION_STRING/g;" \ - cmake/version.cmake + sed -i -e "s/SET(VERSION_REVISION [^) ]*/SET(VERSION_REVISION $VERSION_REVISION/g;" \ + -e "s/SET(VERSION_DESCRIBE [^) ]*/SET(VERSION_DESCRIBE $git_describe/g;" \ + -e "s/SET(VERSION_GITHASH [^) ]*/SET(VERSION_GITHASH $git_hash/g;" \ + -e "s/SET(VERSION_MAJOR [^) ]*/SET(VERSION_MAJOR $VERSION_MAJOR/g;" \ + -e "s/SET(VERSION_MINOR [^) ]*/SET(VERSION_MINOR $VERSION_MINOR/g;" \ + -e "s/SET(VERSION_PATCH [^) ]*/SET(VERSION_PATCH $VERSION_PATCH/g;" \ + -e "s/SET(VERSION_STRING [^) ]*/SET(VERSION_STRING $VERSION_STRING/g;" \ + cmake/autogenerated_versions.txt gen_changelog "$VERSION_STRING" "" "$AUTHOR" "" gen_dockerfiles "$VERSION_STRING" src/Storages/System/StorageSystemContributors.sh ||: utils/list-versions/list-versions.sh > utils/list-versions/version_date.tsv - git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/version.cmake debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv + git commit -m "$auto_message [$VERSION_STRING] [$VERSION_REVISION]" cmake/autogenerated_versions.txt debian/changelog docker/*/Dockerfile src/Storages/System/StorageSystemContributors.generated.cpp utils/list-versions/version_date.tsv if [ -z $NO_PUSH ]; then git push fi echo "Generated version: ${VERSION_STRING}, revision: ${VERSION_REVISION}." - # Second tag for correct version information in version.cmake inside tag + # Second tag for correct version information in autogenerated_versions.txt inside tag if git tag --force -a "$tag" -m "$tag" then if [ -z $NO_PUSH ]; then From 7853722519cf594a6a940e61151689fea967b706 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 May 2020 17:16:56 +0300 Subject: [PATCH 12/38] Update tests --- .../queries/0_stateless/00342_escape_sequences.reference | 1 - tests/queries/0_stateless/00926_multimatch.sql | 2 +- .../0_stateless/01075_allowed_client_hosts.reference | 8 ++++---- 3 files changed, 5 insertions(+), 6 deletions(-) delete mode 100644 tests/queries/0_stateless/00342_escape_sequences.reference diff --git a/tests/queries/0_stateless/00342_escape_sequences.reference b/tests/queries/0_stateless/00342_escape_sequences.reference deleted file mode 100644 index b7ecd06be78..00000000000 --- a/tests/queries/0_stateless/00342_escape_sequences.reference +++ /dev/null @@ -1 +0,0 @@ -07080C0A0D090B5C27223FAA diff --git a/tests/queries/0_stateless/00926_multimatch.sql b/tests/queries/0_stateless/00926_multimatch.sql index d54e4fd2280..bf67533bb6c 100644 --- a/tests/queries/0_stateless/00926_multimatch.sql +++ b/tests/queries/0_stateless/00926_multimatch.sql @@ -89,4 +89,4 @@ SELECT [1, 2, 3, 11] = arraySort(multiMatchAllIndices('фабрикант', ['', SELECT [1] = multiMatchAllIndices(materialize('/odezhda-dlya-bega/'), ['/odezhda-dlya-bega/', 'kurtki-i-vetrovki-dlya-bega', 'futbolki-i-mayki-dlya-bega']); SELECT [] = multiMatchAllIndices(materialize('aaaa'), ['.*aa.*aaa.*', 'aaaaaa{2}', '\(aa\){3}']); SELECT 'All tests above must return 1, all tests below return something.'; -SELECT arraySort(multiMatchAllIndices(arrayJoin(['aaaa', 'aaaaaa', 'bbbb', 'aaaaaaaaaaaaaa']), ['.*aa.*aaa.*', 'aaaaaa{2}', '\(aa\){3}'])); +SELECT arraySort(multiMatchAllIndices(arrayJoin(['aaaa', 'aaaaaa', 'bbbb', 'aaaaaaaaaaaaaa']), ['.*aa.*aaa.*', 'aaaaaa{2}', '(aa){3}'])); diff --git a/tests/queries/0_stateless/01075_allowed_client_hosts.reference b/tests/queries/0_stateless/01075_allowed_client_hosts.reference index 73f54c6027a..3fdea9d1cda 100644 --- a/tests/queries/0_stateless/01075_allowed_client_hosts.reference +++ b/tests/queries/0_stateless/01075_allowed_client_hosts.reference @@ -8,10 +8,10 @@ CREATE USER test_user_01075 HOST LOCAL, IP \'2001:db8:11a3:9d7:1f34:8a2e:7a0:765 CREATE USER test_user_01075 HOST LOCAL CREATE USER test_user_01075 HOST NONE CREATE USER test_user_01075 HOST LIKE \'@.somesite.com\' -CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite.com\' -CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite.com\', \'.*.anothersite.org\' -CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite2.com\', \'.*.anothersite2.org\' -CREATE USER test_user_01075 HOST REGEXP \'.*.anothersite3.com\', \'.*.anothersite3.org\' +CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\' +CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite\\\\.com\', \'.*\\\\.anothersite\\\\.org\' +CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite2\\\\.com\', \'.*\\\\.anothersite2\\\\.org\' +CREATE USER test_user_01075 HOST REGEXP \'.*\\\\.anothersite3\\\\.com\', \'.*\\\\.anothersite3\\\\.org\' CREATE USER `test_user_01075_x@localhost` HOST LOCAL CREATE USER test_user_01075_x CREATE USER `test_user_01075_x@192.168.23.15` HOST LIKE \'192.168.23.15\' From a452d037efe71a7b9d200ad42c06267279176d99 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 28 May 2020 18:35:35 +0300 Subject: [PATCH 13/38] Fix handling bad data while reading in Protobuf format. --- src/Formats/ProtobufReader.cpp | 29 ++++++++++++++++++++--------- src/Formats/ProtobufReader.h | 10 +++++----- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/Formats/ProtobufReader.cpp b/src/Formats/ProtobufReader.cpp index d8302771002..d76b048c1e9 100644 --- a/src/Formats/ProtobufReader.cpp +++ b/src/Formats/ProtobufReader.cpp @@ -32,11 +32,12 @@ namespace BITS32 = 5, }; - // The following condition must always be true: - // any_cursor_position < min(END_OF_VARINT, END_OF_GROUP) - // This inequation helps to check conditions in SimpleReader. - constexpr UInt64 END_OF_VARINT = static_cast(-1); - constexpr UInt64 END_OF_GROUP = static_cast(-2); + // The following conditions must always be true: + // any_cursor_position > END_OF_VARINT + // any_cursor_position > END_OF_GROUP + // Those inequations helps checking conditions in ProtobufReader::SimpleReader. + constexpr Int64 END_OF_VARINT = -1; + constexpr Int64 END_OF_GROUP = -2; Int64 decodeZigZag(UInt64 n) { return static_cast((n >> 1) ^ (~(n & 1) + 1)); } @@ -77,7 +78,7 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors) if (!current_message_level) return; - UInt64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front(); + Int64 root_message_end = (current_message_level == 1) ? current_message_end : parent_message_ends.front(); if (cursor != root_message_end) { if (cursor < root_message_end) @@ -95,6 +96,9 @@ void ProtobufReader::SimpleReader::endMessage(bool ignore_errors) void ProtobufReader::SimpleReader::startNestedMessage() { assert(current_message_level >= 1); + if ((cursor > field_end) && (field_end != END_OF_GROUP)) + throwUnknownFormat(); + // Start reading a nested message which is located inside a length-delimited field // of another message. parent_message_ends.emplace_back(current_message_end); @@ -146,7 +150,7 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) throwUnknownFormat(); } - if (cursor >= current_message_end) + if ((cursor >= current_message_end) && (current_message_end != END_OF_GROUP)) return false; UInt64 varint = readVarint(); @@ -196,11 +200,17 @@ bool ProtobufReader::SimpleReader::readFieldNumber(UInt32 & field_number) bool ProtobufReader::SimpleReader::readUInt(UInt64 & value) { + if (field_end == END_OF_VARINT) + { + value = readVarint(); + field_end = cursor; + return true; + } + if (unlikely(cursor >= field_end)) return false; + value = readVarint(); - if (field_end == END_OF_VARINT) - field_end = cursor; return true; } @@ -227,6 +237,7 @@ bool ProtobufReader::SimpleReader::readFixed(T & value) { if (unlikely(cursor >= field_end)) return false; + readBinary(&value, sizeof(T)); return true; } diff --git a/src/Formats/ProtobufReader.h b/src/Formats/ProtobufReader.h index a50c2f202f0..7c3c19ba376 100644 --- a/src/Formats/ProtobufReader.h +++ b/src/Formats/ProtobufReader.h @@ -124,12 +124,12 @@ private: void ignoreGroup(); ReadBuffer & in; - UInt64 cursor; + Int64 cursor; size_t current_message_level; - UInt64 current_message_end; - std::vector parent_message_ends; - UInt64 field_end; - UInt64 last_string_pos; + Int64 current_message_end; + std::vector parent_message_ends; + Int64 field_end; + Int64 last_string_pos; }; class IConverter From 8e63fd34b91c608feccc9ea2f803043deb73d72e Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Thu, 28 May 2020 18:59:19 +0300 Subject: [PATCH 14/38] Bump build. From ce52a2c820b60d8a44476b42d261248e809c7c62 Mon Sep 17 00:00:00 2001 From: Maxim Akhmedov Date: Thu, 28 May 2020 19:44:30 +0300 Subject: [PATCH 15/38] Add lost VERSION_* defines to ya.make.versions.inc --- cmake/yandex/ya.make.versions.inc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmake/yandex/ya.make.versions.inc b/cmake/yandex/ya.make.versions.inc index 5c3c5eabb60..3ac401cb108 100644 --- a/cmake/yandex/ya.make.versions.inc +++ b/cmake/yandex/ya.make.versions.inc @@ -11,6 +11,9 @@ CFLAGS (GLOBAL -DDBMS_VERSION_MAJOR=${VERSION_MAJOR}) CFLAGS (GLOBAL -DDBMS_VERSION_MINOR=${VERSION_MINOR}) CFLAGS (GLOBAL -DDBMS_VERSION_PATCH=${VERSION_PATCH}) CFLAGS (GLOBAL -DVERSION_FULL=\"\\\"${VERSION_FULL}\\\"\") +CFLAGS (GLOBAL -DVERSION_MAJOR=${VERSION_MAJOR}) +CFLAGS (GLOBAL -DVERSION_MINOR=${VERSION_MINOR}) +CFLAGS (GLOBAL -DVERSION_PATCH=${VERSION_PATCH}) # TODO: not supported yet, not sure if ya.make supports arithmetics. CFLAGS (GLOBAL -DVERSION_INTEGER=0) From 99626a5bd3fa3ba0470142793ce93faec1a0efc5 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 28 May 2020 20:07:47 +0300 Subject: [PATCH 16/38] Add test for handling malformed protobuf data. --- .../0_stateless/00825_protobuf_format_input.reference | 1 + tests/queries/0_stateless/00825_protobuf_format_input.sh | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.reference b/tests/queries/0_stateless/00825_protobuf_format_input.reference index 884cc74c4e5..0c56bc4ebf0 100644 --- a/tests/queries/0_stateless/00825_protobuf_format_input.reference +++ b/tests/queries/0_stateless/00825_protobuf_format_input.reference @@ -8,3 +8,4 @@ a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +7495123456 0 0 2 4 3 9 +ok diff --git a/tests/queries/0_stateless/00825_protobuf_format_input.sh b/tests/queries/0_stateless/00825_protobuf_format_input.sh index d28b70bb002..1c915bc3f24 100755 --- a/tests/queries/0_stateless/00825_protobuf_format_input.sh +++ b/tests/queries/0_stateless/00825_protobuf_format_input.sh @@ -3,7 +3,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . $CURDIR/../shell_config.sh -set -e -o pipefail +set -eo pipefail # Run the client. $CLICKHOUSE_CLIENT --multiquery <<'EOF' @@ -48,5 +48,12 @@ source $CURDIR/00825_protobuf_format_input.insh $CLICKHOUSE_CLIENT --query "SELECT * FROM in_persons_00825 ORDER BY uuid;" $CLICKHOUSE_CLIENT --query "SELECT * FROM in_squares_00825 ORDER BY number;" +# Try to input malformed data. +set +eo pipefail +echo -ne '\xe0\x80\x3f\x0b' \ + | $CLICKHOUSE_CLIENT --query="INSERT INTO in_persons_00825 FORMAT Protobuf SETTINGS format_schema = '$CURDIR/00825_protobuf_format:Person'" 2>&1 \ + | grep -qF "Protobuf messages are corrupted" && echo "ok" || echo "fail" +set -eo pipefail + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_persons_00825;" $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS in_squares_00825;" From a95469020adf75ca6e07e2294113539588a81989 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 28 May 2020 21:02:41 +0300 Subject: [PATCH 17/38] Better error codes --- src/Common/ErrorCodes.cpp | 1 + src/DataStreams/CheckSortedBlockInputStream.cpp | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index cb4c591041c..5898e956400 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -498,6 +498,7 @@ namespace ErrorCodes extern const int ALTER_OF_COLUMN_IS_FORBIDDEN = 524; extern const int INCORRECT_DISK_INDEX = 525; extern const int UNKNOWN_VOLUME_TYPE = 526; + extern const int SORT_ORDER_VIOLATED = 527; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/DataStreams/CheckSortedBlockInputStream.cpp b/src/DataStreams/CheckSortedBlockInputStream.cpp index a56738a9a9f..db4794b917f 100644 --- a/src/DataStreams/CheckSortedBlockInputStream.cpp +++ b/src/DataStreams/CheckSortedBlockInputStream.cpp @@ -6,7 +6,7 @@ namespace DB namespace ErrorCodes { -extern const int LOGICAL_ERROR; +extern const int SORT_ORDER_VIOLATED; } CheckSortedBlockInputStream::CheckSortedBlockInputStream( @@ -72,12 +72,12 @@ Block CheckSortedBlockInputStream::readImpl() auto block_columns = block.getColumns(); if (!last_row.empty() && !less(last_row, 0, block_columns, 0)) - throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR); + throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED); size_t rows = block.rows(); for (size_t i = 1; i < rows; ++i) if (!less(block_columns, i - 1, block_columns, i)) - throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR); + throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED); last_row.clear(); for (size_t i = 0; i < block.columns(); ++i) From 93801a1a9dcfc0c8f801d465ce9611b30ace5976 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 28 May 2020 21:52:08 +0300 Subject: [PATCH 18/38] Fix unit tests --- src/Common/tests/gtest_global_context.h | 9 +++--- src/Storages/tests/gtest_storage_log.cpp | 6 ++-- ..._transform_query_for_external_database.cpp | 28 +++++++++---------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h index 05f60e01774..959d51c6e0b 100644 --- a/src/Common/tests/gtest_global_context.h +++ b/src/Common/tests/gtest_global_context.h @@ -1,6 +1,7 @@ #pragma once #include +#include struct ContextHolder { @@ -11,15 +12,15 @@ struct ContextHolder : shared_context(DB::Context::createShared()) , context(DB::Context::createGlobal(shared_context.get())) { + context.makeGlobalContext(); + context.setPath("./"); } ContextHolder(ContextHolder &&) = default; }; -inline ContextHolder getContext() +inline const ContextHolder & getContext() { - ContextHolder holder; - holder.context.makeGlobalContext(); - holder.context.setPath("./"); + static ContextHolder holder; return holder; } diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 203104801aa..fff352210e7 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -68,7 +68,7 @@ using DiskImplementations = testing::Types; TYPED_TEST_SUITE(StorageLogTest, DiskImplementations); // Returns data written to table in Values format. -std::string writeData(int rows, DB::StoragePtr & table, DB::Context & context) +std::string writeData(int rows, DB::StoragePtr & table, const DB::Context & context) { using namespace DB; @@ -104,7 +104,7 @@ std::string writeData(int rows, DB::StoragePtr & table, DB::Context & context) } // Returns all table data in Values format. -std::string readData(DB::StoragePtr & table, DB::Context & context) +std::string readData(DB::StoragePtr & table, const DB::Context & context) { using namespace DB; @@ -136,7 +136,7 @@ std::string readData(DB::StoragePtr & table, DB::Context & context) TYPED_TEST(StorageLogTest, testReadWrite) { using namespace DB; - auto context_holder = getContext(); + const auto & context_holder = getContext(); std::string data; diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index df6ee2bd0a5..f51e38d32a9 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include @@ -18,7 +17,7 @@ using namespace DB; /// NOTE How to do better? struct State { - Context & context; + Context context; NamesAndTypesList columns{ {"column", std::make_shared()}, {"apply_id", std::make_shared()}, @@ -27,9 +26,10 @@ struct State {"create_time", std::make_shared()}, }; - explicit State(Context & context_) : context(context_) + explicit State() + : context(getContext().context) { - registerFunctions(); + DB::registerFunctions(); DatabasePtr database = std::make_shared("test"); database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); context.makeGlobalContext(); @@ -38,6 +38,11 @@ struct State } }; +State getState() +{ + static State state; + return state; +} static void check(const std::string & query, const std::string & expected, const Context & context, const NamesAndTypesList & columns) { @@ -54,8 +59,7 @@ static void check(const std::string & query, const std::string & expected, const TEST(TransformQueryForExternalDatabase, InWithSingleElement) { - auto context_holder = getContext(); - State state(context_holder.context); + const State & state = getState(); check("SELECT column FROM test.table WHERE 1 IN (1)", R"(SELECT "column" FROM "test"."table" WHERE 1)", @@ -70,8 +74,7 @@ TEST(TransformQueryForExternalDatabase, InWithSingleElement) TEST(TransformQueryForExternalDatabase, Like) { - auto context_holder = getContext(); - State state(context_holder.context); + const State & state = getState(); check("SELECT column FROM test.table WHERE column LIKE '%hello%'", R"(SELECT "column" FROM "test"."table" WHERE "column" LIKE '%hello%')", @@ -83,8 +86,7 @@ TEST(TransformQueryForExternalDatabase, Like) TEST(TransformQueryForExternalDatabase, Substring) { - auto context_holder = getContext(); - State state(context_holder.context); + const State & state = getState(); check("SELECT column FROM test.table WHERE left(column, 10) = RIGHT(column, 10) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello'", R"(SELECT "column" FROM "test"."table")", @@ -93,8 +95,7 @@ TEST(TransformQueryForExternalDatabase, Substring) TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) { - auto context_holder = getContext(); - State state(context_holder.context); + const State & state = getState(); check("SELECT column FROM test.table WHERE 1 = 1 AND toString(column) = '42' AND column = 42 AND left(column, 10) = RIGHT(column, 10) AND column IN (1, 42) AND SUBSTRING(column FROM 1 FOR 2) = 'Hello' AND column != 4", R"(SELECT "column" FROM "test"."table" WHERE 1 AND ("column" = 42) AND ("column" IN (1, 42)) AND ("column" != 4))", @@ -106,8 +107,7 @@ TEST(TransformQueryForExternalDatabase, MultipleAndSubqueries) TEST(TransformQueryForExternalDatabase, Issue7245) { - auto context_holder = getContext(); - State state(context_holder.context); + const State & state = getState(); check("select apply_id from test.table where apply_type = 2 and create_time > addDays(toDateTime('2019-01-01 01:02:03'),-7) and apply_status in (3,4)", R"(SELECT "apply_id", "apply_type", "apply_status", "create_time" FROM "test"."table" WHERE ("apply_type" = 2) AND ("create_time" > '2018-12-25 01:02:03') AND ("apply_status" IN (3, 4)))", From e9c1e1a7e4913fafb1164b8633b79bf76a4fd48a Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 28 May 2020 21:54:42 +0300 Subject: [PATCH 19/38] Remove redundant header --- src/Common/tests/gtest_global_context.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/tests/gtest_global_context.h b/src/Common/tests/gtest_global_context.h index 959d51c6e0b..b6529f09b46 100644 --- a/src/Common/tests/gtest_global_context.h +++ b/src/Common/tests/gtest_global_context.h @@ -1,7 +1,6 @@ #pragma once #include -#include struct ContextHolder { From 026244b07f796a5cd36c80ea20f3484b629b68ee Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 28 May 2020 21:57:09 +0300 Subject: [PATCH 20/38] Better --- .../tests/gtest_transform_query_for_external_database.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/tests/gtest_transform_query_for_external_database.cpp b/src/Storages/tests/gtest_transform_query_for_external_database.cpp index f51e38d32a9..6caf759da42 100644 --- a/src/Storages/tests/gtest_transform_query_for_external_database.cpp +++ b/src/Storages/tests/gtest_transform_query_for_external_database.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -29,7 +30,7 @@ struct State explicit State() : context(getContext().context) { - DB::registerFunctions(); + registerFunctions(); DatabasePtr database = std::make_shared("test"); database->attachTable("table", StorageMemory::create(StorageID("test", "table"), ColumnsDescription{columns}, ConstraintsDescription{})); context.makeGlobalContext(); From db72f3959489ef54261071390e20703dea5caf51 Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 28 May 2020 22:07:20 +0300 Subject: [PATCH 21/38] Remove error code --- src/Common/ErrorCodes.cpp | 1 - src/DataStreams/CheckSortedBlockInputStream.cpp | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 5898e956400..cb4c591041c 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -498,7 +498,6 @@ namespace ErrorCodes extern const int ALTER_OF_COLUMN_IS_FORBIDDEN = 524; extern const int INCORRECT_DISK_INDEX = 525; extern const int UNKNOWN_VOLUME_TYPE = 526; - extern const int SORT_ORDER_VIOLATED = 527; extern const int KEEPER_EXCEPTION = 999; extern const int POCO_EXCEPTION = 1000; diff --git a/src/DataStreams/CheckSortedBlockInputStream.cpp b/src/DataStreams/CheckSortedBlockInputStream.cpp index db4794b917f..a56738a9a9f 100644 --- a/src/DataStreams/CheckSortedBlockInputStream.cpp +++ b/src/DataStreams/CheckSortedBlockInputStream.cpp @@ -6,7 +6,7 @@ namespace DB namespace ErrorCodes { -extern const int SORT_ORDER_VIOLATED; +extern const int LOGICAL_ERROR; } CheckSortedBlockInputStream::CheckSortedBlockInputStream( @@ -72,12 +72,12 @@ Block CheckSortedBlockInputStream::readImpl() auto block_columns = block.getColumns(); if (!last_row.empty() && !less(last_row, 0, block_columns, 0)) - throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED); + throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR); size_t rows = block.rows(); for (size_t i = 1; i < rows; ++i) if (!less(block_columns, i - 1, block_columns, i)) - throw Exception("Sort order of blocks violated", ErrorCodes::SORT_ORDER_VIOLATED); + throw Exception("Sort order of blocks violated", ErrorCodes::LOGICAL_ERROR); last_row.clear(); for (size_t i = 0; i < block.columns(); ++i) From d7ec509f337a88b0125b5af48d981bb4b98ba604 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 28 May 2020 22:26:20 +0300 Subject: [PATCH 22/38] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index b7756f7fb79..9b92dbfa717 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -203,10 +203,11 @@ Upd. SharedContext вынесен из Context. Upd. В очереди. Иван Лежанкин. -### 2.9. Логгировние в format-стиле {#loggirovnie-v-format-stile} +### 2.9. + Логгировние в format-стиле {#loggirovnie-v-format-stile} -Делает [Иван Лежанкин](https://github.com/abyss7). Низкий приоритет. -[\#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998) +[#6049](https://github.com/ClickHouse/ClickHouse/issues/6049#issuecomment-570836998) + +Сделано. ### 2.10. Запрашивать у таблиц не столбцы, а срезы {#zaprashivat-u-tablits-ne-stolbtsy-a-srezy} From d8049be204ed7325c5725bdb0b2a49fef9cd8464 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 28 May 2020 22:27:31 +0300 Subject: [PATCH 23/38] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index 9b92dbfa717..0bedfda57b1 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -73,10 +73,10 @@ Upd. Включено для системных таблиц. Q1. Закоммичено, но есть технический долг, который исправляется сейчас. Готово. Нет, не готово - там всё ещё технический долг. -### 1.9. Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh} +### 1.9. + Использование TTL для прореживания данных {#ispolzovanie-ttl-dlia-prorezhivaniia-dannykh} Будет делать Сорокин Николай, ВШЭ и Яндекс. -Upd. Есть pull request. +Upd. Есть pull request. Upd. Сделано. Сейчас пользователь может задать в таблице выражение, которое определяет, сколько времени хранятся данные. Обычно это выражение задаётся относительно значения столбца с датой - например: удалять данные через три месяца. https://clickhouse.tech/docs/ru/operations/table_engines/mergetree/\#table_engine-mergetree-ttl From 24fb6266daa54d4aa8a7ad9e7bcfa98da9afe34b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 May 2020 22:28:44 +0300 Subject: [PATCH 24/38] Fix wrong markup --- website/css/docs.css | 1 - 1 file changed, 1 deletion(-) diff --git a/website/css/docs.css b/website/css/docs.css index 18bf40de13b..2c175b18db6 100644 --- a/website/css/docs.css +++ b/website/css/docs.css @@ -69,7 +69,6 @@ summary { #content code { color: #111; background: #eee; - padding: 2px; } From 05642d2a84bae35313be9050070425158e8b596e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 28 May 2020 23:53:13 +0300 Subject: [PATCH 25/38] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index 0bedfda57b1..091e7f37c66 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -124,7 +124,7 @@ Q2. Upd. Олег будет делать только часть про HDFS. Upd. Реализация поверх S3 является рабочей на уровне PoC. -### 1.13. Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final} +### 1.13. + Ускорение запросов с FINAL {#uskorenie-zaprosov-s-final} Требует 2.1. Делает [Николай Кочетов](https://github.com/KochetovNicolai). Нужно для Яндекс.Метрики. Q2. Upd: PR [#10463](https://github.com/ClickHouse/ClickHouse/pull/10463) From 2918677a9e3ece71b5403e4d72b9275bbcba71ff Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 28 May 2020 23:58:25 +0300 Subject: [PATCH 26/38] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index 091e7f37c66..895138e1a33 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -283,24 +283,20 @@ Upd. Сейчас обсуждается, как сделать другую з ### 4.3. Ограничение числа одновременных скачиваний с реплик {#ogranichenie-chisla-odnovremennykh-skachivanii-s-replik} -Дмитрий Григорьев, ВШЭ. Изначально делал Олег Алексеенков, но пока решение не готово, хотя там не так уж много доделывать. ### 4.4. Ограничение сетевой полосы при репликации {#ogranichenie-setevoi-polosy-pri-replikatsii} -Дмитрий Григорьев, ВШЭ. Нужно для Метрики. +Нужно для Метрики. ### 4.5. Возможность продолжить передачу куска данных при репликации после сбоя {#vozmozhnost-prodolzhit-peredachu-kuska-dannykh-pri-replikatsii-posle-sboia} -Дмитрий Григорьев, ВШЭ. - ### 4.6. p2p передача для GLOBAL подзапросов {#p2p-peredacha-dlia-global-podzaprosov} ### 4.7. Ленивая загрузка множеств для IN и JOIN с помощью k/v запросов {#lenivaia-zagruzka-mnozhestv-dlia-in-i-join-s-pomoshchiu-kv-zaprosov} ### 4.8. Разделить background pool для fetch и merge {#razdelit-background-pool-dlia-fetch-i-merge} -Дмитрий Григорьев, ВШЭ. В очереди. Исправить проблему, что восстанавливающаяся реплика перестаёт мержить. Частично компенсируется 4.3. @@ -330,6 +326,7 @@ Upd. Сделано. Эффективность работы под вопрос Метрика, БК, Маркет, Altinity уже используют более свежие версии чем LTS. Upd. Появилась вторая версия LTS - 20.3. + ## 6. Инструментирование {#instrumentirovanie} ### 6.1. + Исправления сэмплирующего профайлера запросов {#ispravleniia-sempliruiushchego-profailera-zaprosov} @@ -426,11 +423,11 @@ Upd. Рассмотрели все проверки подряд. UBSan включен в функциональных тестах, но не включен в интеграционных тестах. Требует 7.7. -### 7.11. Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh} +### 7.11. + Включение \*San в unit тестах {#vkliuchenie-san-v-unit-testakh} У нас мало unit тестов по сравнению с функциональными тестами и их использование не обязательно. Но они всё-равно важны и нет причин не запускать их под всеми видами sanitizers. -Илья Яцишин. +Илья Яцишин. Сделано. ### 7.12. Показывать тестовое покрытие нового кода в PR {#pokazyvat-testovoe-pokrytie-novogo-koda-v-pr} @@ -529,6 +526,8 @@ Upd. Есть сборки, [пример](https://clickhouse-builds.s3.yandex.n Дарья Петрова, УрФУ. +Рабочий прототип: https://pulls-dashboard-demo.herokuapp.com/dashboard/ClickHouse/ClickHouse + Над ClickHouse одновременно работает большое количество разработчиков, которые оформляют свои изменения в виде pull requests. Когда непомерженных pull requests много, то возникает сложность с организацией работы - непонятно, на какой pull request смотреть в первую очередь. Предлагается реализовать простое одностраничное веб-приложение, в котором отображается список pull requests со следующей информацией: From f0f7b7cc7cafba1376e2ac33f142f973686ea136 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 29 May 2020 00:21:03 +0300 Subject: [PATCH 27/38] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index 895138e1a33..df4bbb4ca57 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -627,6 +627,7 @@ Upd. Готово (все директории кроме contrib). ### 7.32. Обфускация продакшен запросов {#obfuskatsiia-prodakshen-zaprosov} Роман Ильговский. Нужно для Яндекс.Метрики. +Есть pull request, почти готово: https://github.com/ClickHouse/ClickHouse/pull/10973 Имея SQL запрос, требуется вывести структуру таблиц, на которых этот запрос будет выполнен, и заполнить эти таблицы случайными данными, такими, что результат этого запроса зависит от выбора подмножества данных. From 9d5b1b69f7c3d6279e09ab593c22f6b1ec3ec7bf Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Fri, 29 May 2020 00:22:18 +0300 Subject: [PATCH 28/38] Update extended-roadmap.md --- docs/ru/whats-new/extended-roadmap.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/whats-new/extended-roadmap.md b/docs/ru/whats-new/extended-roadmap.md index df4bbb4ca57..20ebe28fe5b 100644 --- a/docs/ru/whats-new/extended-roadmap.md +++ b/docs/ru/whats-new/extended-roadmap.md @@ -1398,11 +1398,11 @@ Constraints позволяют задать выражение, истиннос Василий Морозов, Арслан Гумеров, Альберт Кидрачев, ВШЭ. В прошлом году задачу начинал делать другой человек, но не добился достаточного прогресса. -1. Оптимизация top sort. ++ 1. Оптимизация top sort. В ClickHouse используется неоптимальный вариант top sort. Суть его в том, что из каждого блока достаётся top N записей, а затем, все блоки мержатся. Но доставание top N записей у каждого следующего блока бессмысленно, если мы знаем, что из них в глобальный top N войдёт меньше. Конечно нужно реализовать вариацию на тему priority queue (heap) с быстрым пропуском целых блоков, если ни одна строка не попадёт в накопленный top. -2. Рекурсивный вариант сортировки по кортежам. ++ 2. Рекурсивный вариант сортировки по кортежам. Для сортировки по кортежам используется обычная сортировка с компаратором, который в цикле по элементам кортежа делает виртуальные вызовы `IColumn::compareAt`. Это неоптимально - как из-за короткого цикла по неизвестному в compile-time количеству элементов, так и из-за виртуальных вызовов. Чтобы обойтись без виртуальных вызовов, есть метод `IColumn::getPermutation`. Он используется в случае сортировки по одному столбцу. Есть вариант, что в случае сортировки по кортежу, что-то похожее тоже можно применить… например, сделать метод `updatePermutation`, принимающий аргументы offset и limit, и допереставляющий перестановку в диапазоне значений, в которых предыдущий столбец имел равные значения. From 2b4d9a6349a6c530d1574577254bde216ccced84 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2020 01:00:01 +0300 Subject: [PATCH 29/38] Add a test from @donmikel --- .../01285_engine_join_donmikel.reference | 1 + .../0_stateless/01285_engine_join_donmikel.sh | 65 ++++++++++++ .../01285_engine_join_donmikel.tsv | 100 ++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 tests/queries/0_stateless/01285_engine_join_donmikel.reference create mode 100755 tests/queries/0_stateless/01285_engine_join_donmikel.sh create mode 100644 tests/queries/0_stateless/01285_engine_join_donmikel.tsv diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.reference b/tests/queries/0_stateless/01285_engine_join_donmikel.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/01285_engine_join_donmikel.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.sh b/tests/queries/0_stateless/01285_engine_join_donmikel.sh new file mode 100755 index 00000000000..8a867921dbe --- /dev/null +++ b/tests/queries/0_stateless/01285_engine_join_donmikel.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. $CURDIR/../shell_config.sh + +$CLICKHOUSE_CLIENT --multiquery --query " +DROP TABLE IF EXISTS NmSubj; +DROP TABLE IF EXISTS events; + +create table NmSubj +( + NmId UInt32, + SubjectId UInt32 +) + engine = Join(All, inner, NmId); + +create table events +( + EventDate Date, + EventDateTime DateTime, + EventId String, + SessionId FixedString(36), + PageViewId FixedString(36), + UserId UInt64, + UniqUserId FixedString(36), + UrlReferrer String, + Param1 String, + Param2 String, + Param3 String, + Param4 String, + Param5 String, + Param6 String, + Param7 String, + Param8 String, + Param9 String, + Param10 String, + ApplicationType UInt8, + Locale String, + Lang String, + Version String, + Path String, + QueryString String, + UserHostAddress UInt32 +) + engine = MergeTree() + PARTITION BY (toYYYYMM(EventDate), EventId) + ORDER BY (EventId, EventDate, Locale, ApplicationType, intHash64(UserId)) + SAMPLE BY intHash64(UserId) + SETTINGS index_granularity = 8192; + +insert into NmSubj values (1, 1), (2, 2), (3, 3); +" + +$CLICKHOUSE_CLIENT --query "INSERT INTO events FORMAT TSV" < ${CURDIR}/01285_engine_join_donmikel.tsv + +$CLICKHOUSE_CLIENT --query " +SELECT toInt32(count() / 24) as Count +FROM events as e INNER JOIN NmSubj as ns +ON ns.NmId = toUInt32(e.Param1) +WHERE e.EventDate = today() - 7 AND e.EventId = 'GCO' AND ns.SubjectId = 2073" + +$CLICKHOUSE_CLIENT --multiquery --query " +DROP TABLE NmSubj; +DROP TABLE events; +" diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.tsv b/tests/queries/0_stateless/01285_engine_join_donmikel.tsv new file mode 100644 index 00000000000..a6f47144ff2 --- /dev/null +++ b/tests/queries/0_stateless/01285_engine_join_donmikel.tsv @@ -0,0 +1,100 @@ +2020-02-17 2020-02-17 19:40:09 EMC wFxiX8iZ3Vns8pO6bx0WvYftGEOM9WMff Ku5ofnKwBaQ0CtcdZa6sAnSQbyufu0zfd 0 X16qWFb_h8JONLGxs7oiXFZTdBFfTniRMG mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=33&userId=274574634&email=Z21haWwucnU%3D&source=mn 657583482 +2020-02-17 2020-02-17 20:23:53 EMC foco1fj5Li6ey>MSfn?jKHBBdiwKXiBmy7Ni ICZBjN7en5snnszHKbTVUU*xyOhuZEgI1EK 0 I7XAKnCMsD9tdFHmmcCrI~KqiKQdo7Gxd5 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=E17727463464LnJ1&source=mn 2992671877 +2020-02-17 2020-02-17 19:49:04 EMC q3AxEdMQhsr1NewVSKjHzYvh63XdxtGcBZX GVGMPZhenJd83JarI7Qicq5BKRfijT0ls 18536997 D_myvD3U4bmy5anwdNRFaxm KNoDfcCDL9U6 http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=AD0581918974621haWwuY29t&source=mn 1656424571 +2020-02-17 2020-02-17 21:55:32 EMC ef7tm7k7Yvpb4XIfvf7ON-s4Xv3ztD7SkB KIDkUzEgcAMu%ObAQAkI3jmm3638bSnX0 22341123 6NcB26yJrLhWgldC_HvV2fUOHNLRFvkLbA mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=E943634627468871D581940274695&userId=2746887723D&source=mn 1429586160 +2020-02-17 2020-02-17 21:37:47 EMC 3Y_KZLEJgn4bw?omv4EizIs1lUMz0Li0Ws0 I3QaVG9UvXWvWReLSHuDCs3DdgTbX8av6 28213800 KnoofunrqFKdIta8pMuR!FNX2yu1I6gNo https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail&v=2&s=3745772 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=5YWxpbC5jb20%3D&source=mn 872136829 +2020-02-17 2020-02-17 19:15:34 EMC JcHEtSzMb70K}04FW#ShpZw1ot_fb7wxj8 KG9bmRo6yweN2EX48CUBQcsOqzbHn6g3u 30047107 UKb5rJ5D\bFncBHxCioYswtJKAlv6Ua5MVSrj mn 48 Ru ru 7.0.32 /confirm/vipstatus ?key=75ydQ%3D%3D&source=mn 626057004 +2020-02-17 2020-02-17 18:15:04 EMC _sjptcVU5CF4sTnXh0C6DCYcOo0rNowf 0a0AcabOllf7\rN9X9aa7rVmJ0J1oZGJY2 30648090 Z2K_4kffe21iW1hl2jby1RXAAh5XtPLP3 mn 48 Ru ru 7.0.32 /confirm/vipstatus ?key=5&ts=15819420%3D&source=mn 3682305993 +2020-02-17 2020-02-17 16:10:20 EMC WM44OUvEZse_9hQ0NJsn0k47pX6_UnaS7 LgCWwTMZ+GwiQEbNT95zUnkMCXYqPs1Xt1 0 QUuSVpqq#SgMdveJDW0bQ6s8jn8MLXzL1w mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=CCD&source=mn 726391852 +2020-02-17 2020-02-17 17:30:44 EMC PGfQVZHuAY0A_kUA\04ceYRjWqgPJtfhCn 4TR9oRKCNrouy:zJBLBi8J4=u7Mkt9jgzgmB 0 xbCoBX9pTDaWiRxW@F3fOgKD9lX58oWsc mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=DB3D&source=pcap 2498835913 +2020-02-17 2020-02-17 19:10:51 EMC gnd66xWlKf2H2tiZdbivjoz4ILmJ4lp_ ysgw6WLdbNWSHQrybWjhH9uJqnlZMWAv 0 nDQDqsnm/dlpbFMPJE93b6EETHzFxUSzVv pcap 187 Ru ru 7.0.32 /confirm/vipstatus ?key=4&userId=274634688&userId=274688&userId=27465D85CD4F634627468877266792YXJ1&source=mn 816668334 +2020-02-17 2020-02-17 20:52:00 EMC pBzkcTpn50WpO]BZH7^js6TpPGp7Uid7mFg jQVdKDl4+Sp_j5CZc PtVoMGTIGOfYO6Q2 0 aQGyrF9s3qUMPq_dUnOiM6QVMX5PNQ_eiV pcap 187 Ru ru 7.0.32 /confirm/vipstatus ?key=50&email=YWlsLmNvbQ%3D%3D&source=pcap 1770779506 +2020-02-17 2020-02-17 21:31:14 EMC cyTu3wNRCI1TzkxCybFNz\rCkrdDZQxYtlW gmtJF_hWE8M79bWSaKkn8UT5Xdwfb0uK8T 0 RSJUU0SBDIXjpYORna59cWouc20W5VIP mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=4CD%3D&source=mn 3448881839 +2020-02-17 2020-02-17 17:15:46 EMC I3bvr8ce2SU_&UuDX4J6KRJl5uipnL2Or myxD8nQXFYnKqBoPGGo4Hy6hBTXRL04K 0 ntHjLMvhnb7b09Tu7jRkM8KgS5ni0w4oH mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=36274695&userId=2560&userId=27463465&userId=27468872EDB908&ts=158193819382 2447621965 +2020-02-17 2020-02-17 18:21:21 EMC _468tOVLPpIkgKvPGX8ypjfTZV09k8w5 0KakGIWk9uaL0Rv3]1T5wPeHPklVWzhytg 0 QAJxAuQADoEQqY0HUT56P0O4K5y55K_L https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail&v=2&s=92Ymail&v=2&s=3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D66email&v=2&s=93D158192YWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=5819392158193634&userId=274688&ts=15819408BD1haWwucnU%3D&source=mn 1547300784 +2020-02-17 2020-02-17 19:07:33 EMC Y8yBoZ7Cp2GUTQ8srLMZJFIZ85WgiBeau wmIgwW88aSGvvjKwV,8EooQonI_90taLrh 0 KXVqerLQyaMaMtuT1FzUjrLev8Fw11mkfO pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=72846239707&userId=274688&ts=1581921haWwucnU%3D&source=pcap 2777789095 +2020-02-17 2020-02-17 11:27:36 EMC n66D1ZN9C6y7LASQucHNdkw_QseVRnt32 yeqFeZFa,7cQ_JyTO3yJJKS QaCKEWERPx4Z 24388614 6h91HeiPyIKw8yV0p\tcaK99w_zmnmXU77RI https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail.ru/redir?u=https://click.mail&v=2&s=3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=http&o=mail.ru/redir?u=https://click.mail.ru/redir?u=https://mail%3DYWlsLnJ1%26source mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=6167462E4CD570A0312D6BF1BtYWlsLnJ1&source=mn 662531688 +2020-02-17 2020-02-17 14:11:06 EMC jamANU2zei_lBkfEk_Im5ow0gWB5G0gL1S So7Jc6gheYgclfoq8l0hqsPExrMNSXPLV 20685668 nwRXATVEY2PaYr34qwtZX0Q6ah_sceXjbL https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail&v=2&s=9444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69103Dmn&c=swm&r=http&o=mail&v=2&s=92YWlsLnJ1%26source%3Dpc&c=swm&r=https mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=420%3D&source=mn 3233174651 +2020-02-17 2020-02-17 14:32:04 EMC lsruHZ0Ywg1yDdrio.j5piNMTnqNxeaOQJ G6K62xDbm9aJ_kNlLqEo4AhVf46wDtWi 32046518 wgpf1yqVtVhLL7yT9sp1RbqaXGF81GmSiQ6 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=1581943D&source=pcap 3183159832 +2020-02-17 2020-02-17 15:11:37 EMC 5EW8o4vD850H>TQam3NA2@egQm32jpASIF KJKN5QAHdoqw_RGol5hLvSq6753cKLswM 32971244 8Vh3DVlCjfXs[5hqVMGDDP2iBjoiqhoACB https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail&v=2&s=3D27457AD3D274576userId%3D27461haWwucnU%253D%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6ts%3D158193Fkey%3D669%26email&v=2&s=3Fkey%3D69D01 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=5634&userId=274695&ts=15819440446274695&userId=27465&email=Z21haWwucnU%3D&source=pcap 574798405 +2020-02-17 2020-02-17 13:59:25 EMC i6LAlVL3U0GUFFtjiRgfPyjyattjzds6y2 IYv_jZ2k+bNO33nbLKRRGcJvTHVIueEDI 21779069 eUgOCUGbEZM15\'Of0Vz6zQmAT0T0zX3XSO http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=D1F9sail=YWxpbC5ydQ%3D%3D&source=mn 3475274476 +2020-02-17 2020-02-17 06:40:47 EMC A_3372QT5xhTmjNrokk3yZCtuOm52FfZT oOD7es9hGUs7jB3_)jVKh3Oo10IQvAu01 27168564 UYEHX0itbLKZjChWxLsP4a7LUXQLX4iB mn 48 Ru ru 7.0.32 /confirm/vipstatus ?key=467468874634&email=dG92&userId=274695&email=email=a2B7AD60&email=bWFpbC5ydQ%3D%3D&source=mn 3084609784 +2020-02-17 2020-02-17 14:10:13 EMC WSlfONgzmGj8kylbItxLW8MX4u60oapxy U8TMovlAcJTcA~u3KoOFG7pchP3CBPbcB4 28904171 nBfRyelBAkOfR70tIzBusuS0M8_KbRsOGI pcap 187 Ru ru 7.0.32 /confirm/vipstatus ?key=4462F5B37274695&email=Ymail=Z21haWwucnU%3D&source=mn 2020344465 +2020-02-17 2020-02-17 01:43:57 EMC khjsNWq7 Iq564D1u\'vWuKZHN4kfq0Kyfv hcAK5RGOjZHYbFJwufN7\tZeXCRWAEKoT2 0 G0Zi9TAk`t0aUcHRiOcb1|RgoT8xMOH4KV mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=1581940095&ts=158193819382&userId=27465D8B8F1936256C35B4B466&userId=2746887C1A97&userId=274695&userId=274634&userId 902363005 +2020-02-17 2020-02-17 02:17:35 EMC H56Q4qwa\'eYiFCmGR40Pp7,m6fDiE2nJSmB Efk7W3ZrgNsEi.anvkrteuN>lF2UlX0RZFzr 0 QhyRjVGEsD_4YaQiaLorQ6y0fbCxkEVzQ5 pcap 187 Ru ru 7.0.32 /confirm/vipstatus ?key=44695&email=YW5hcnU%3D&source=mn 2161496531 +2020-02-17 2020-02-17 03:20:48 EMC 1wBv9Q7rDQuImuif>0v1t8OXWSIXymKrI sNujW7iatZLPxEOpI1fZdT4p0QRaotsXPZ 0 LSKNxAcrTMsADK9VI7AMqm05blzLWXCJ05 pcap 187 Ru ru 7.0.32 /confirm/vipstatus ?key=AE1FAC2BFA1E75hce=mn 985607445 +2020-02-17 2020-02-17 06:18:16 EMC 6dqrbyyKHWPBEyPZGbJRKjxtmt0Aq8IsON k57VHYX59Ws9u[L3nNEruOGCVgGH_l27J1 0 _lwuZQycqGeyMxkRSM7pR1lHNU6_V0nVWG pcap 187 Ru ru 7.0.32 /confirm/vipstatus ?key=B27465D88&userId=274572BE3274577273AF0&email=a274578&userId=274692&userId=274688&userId=27465D8192 130983489 +2020-02-17 2020-02-17 09:20:13 EMC UCLTP56_YcQ9rWVJa8rif2Aq5NNqVThlwH 7Rx9OsBs7BemVxuArwZ7N5BqcFwK7CSI 0 JOIbkOPdPPbPCtqRV6wQHsdWYNnmarCxm mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=A6E819420%3D&source=pcap 2567294448 +2020-02-17 2020-02-17 13:05:00 EMC 28AZ4n0zpsVuwzenZ6*gd0VNMjnT8Kmfncv PFiGZRLTjK1CxfLJajtGVFOATfWnTE00 0 QYPUzvwV=w8ev0E77B|RK6Z<66ZX5y1YIK1J mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=C19213C0060312E9E28A82&email=bmFA2095&userId=2746346&userId=27457455266&ts=15819213D&source=pcap 2881908356 +2020-02-17 2020-02-17 13:40:53 EMC uDOkWKoTLCbB2SFYoepda0 itJJ9zJgyZVY d50Y87RQCO70ci0u{QDZcA8dU31qm0IS1 0 0MuqyXoTtKYpmheID1AiXWz9sZ0zroDj mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=A3274695&userId=2746346470&userId=274634&userId=256AEA56926&ts=158194407&userId=27463467720%3D&source=pcap 700256219 +2020-02-17 2020-02-17 13:56:10 EMC rOXnSk45QcoEkw03yMYbGUhD_YZ_yCx9OU DyGXuVg8;P6Tq8CDp2_0XI`zPz4wuG8iJJh 0 3IM74UyhDyv2Jamik02pDQJu_L10vfPCKTm mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=459&email=bmFA81942095&userId=27468872095&userId=27457F1944672Y29t&source=pcap 607085190 +2020-02-17 2020-02-17 13:57:55 EMC xo09ZqxjuPBo_2aYOgnG\\sH3HDRtPH7xW 1I3ceRxYeGTFgNy30ojqJ]qrOgqu1Ag8yO 0 PwQkOkG2N1_p*ZFjOwmM6FviVATMGY5_rbK mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=158193979967F628BBA48E274634&email=dGV4LnJ1&source=mn 3851759707 +2020-02-17 2020-02-17 14:09:55 EMC vmlDYHpCg270cyeLJZBJTwBBnnJHH99Av l9FPMTHyyKlHijgeO]2oJK90OO8hWvntXS8 0 0Xu7Ha8eqQdwcG3pm#GNMw6HzEsxoAszdK mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=BtYWlsLnJ1&source=mn 2783694582 +2020-02-17 2020-02-17 15:31:52 EMC WIzjCT0n1XqAlOuRi2YskEx619ZjBaFe4g z_NGqjbLB0IhQV1SxDRMNG0DlMUxPlqaQ6 0 bv75_7V5B4pBB2aNK\fRwnBDQm0ngrcFPLN mn 187 Ru ru 7.0.32 /confirm/vipstatus ?key=158193628FA603193692&email=bmFA934&userId=265&ts=158192YW5haWwucnU%3D&source=pcap 722494088 +2020-02-17 2020-02-17 06:28:37 EMC 434gE67EZuzQL3ssU31NIt0eoeEpm8YplE7RGkXuG https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dpc&c=swm&r=http&o=mail.ru/redir?u=https://click.mail.ru/redir?u=http&o=mail&v=2&s=974578Fvipstatus%3Fkey mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=1581938C8193634&email=a29t&source=mn 4267472950 +2020-02-17 2020-02-17 03:43:33 EMC TAl7tSOQaATLUUMZTx90iv0sqRoADhEn2r zvCR7ostHdMiQuVI8uxOyYGf0mm6k3ckHSL 32488481 9kFERwTF"AWKxn0YPOUv_tA8c5X2cdy4bA pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=1581936292&email=a256F095&ts=158192&userId=274695&userId=27465D84D65D8B8E7EA692YW5haWwucnU%3D&source 1722142578 +2020-02-17 2020-02-17 04:06:30 EMC kL0p32eE J8hj51lGJQdDpzRytjcc2_AnA eO7MVT5A47viQfb3A01JapoD001J0l_UAt 23078127 KDZSswsocsumoOtSx\\phTvuffYOTHGNoUa mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=7724D&source=mn 2669624134 +2020-02-17 2020-02-17 04:23:38 EMC q3PyeDUQPYw7QN9BwE1eP\nDzWsZDAv3GTN u3GnfRRy>bOqpqFvSDikeB8V0zHacc0ur 26965394 CxxiDtfVM9T6WuleL1P4mMbzF9k0TLWLF pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=58193695&ts=1581942095&email=bWFpbC5ydQ%3D%3D&source=mn 2552851257 +2020-02-17 2020-02-17 05:36:35 EMC ipti_lG94H8P5RgFwKzaoxUwWNEV9beK zolMT7vt_ttY|BmlD2PSejtg6O7WzJ2S_ 31107154 ha6rDRxktOKPjl2gzz7R2TpQu2KKta9Q mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=4942095&ts=15819369266&userId=274692158193634&userId=2745523552D695 1459132007 +2020-02-17 2020-02-17 00:45:31 EMC hXw0ETb7piXc>PKMzqxhV_F_UdWZnFB5O VXB5E4w28OrrZT0X1Sn9SrTWuJhjff7I 19656120 v06fJehG\'_m057sFduiQBx(mS5JnuSD6jph mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=B193695&ts=15819346165D85jb20%3D&source=pcap 1300966980 +2020-02-17 2020-02-17 03:44:40 EMC 0Q0EMlWcxIACIHUsZX4mEhJnxPpCJV6Qe ZqVPZ1Yb>rFSB\tzEMNdTEjoWIHLaQ2o5vI_E 17361913 AuRaBJOGvceRXWiMb9\'wolUnAGR0hxXQgOS pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=725634&email=bWFpbC5ydQ%3D%3D&source=mn 1552491257 +2020-02-17 2020-02-17 02:35:40 EMC mSk0tnq0soJQslU0KlkSmfnXFu6fqRw0A hjlyHsr0&gybjsAJ0WRqdLJz32l3lmiEgDN 32664890 YAoxYQxl"Ce8HvY2j90VWzKQxlGCTNRAyJ mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=6459&ts=158194464089&userId=27465D82&userId=27455229t&source=mn 3051884197 +2020-02-17 2020-02-17 13:44:36 EMC znOwy_e4iHG8(NdCh2qgz1tjBR0ZjsFus eNSFOPi9oax3Ru7x01JA4TWeX66Ev8T9U 18979240 LosZ3fZjVY8hXjH8AJgKx:O3FNAMVDRGlZ mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=71581944027465D819384019215819397327457AbWFpbC5ydQ%3D%3D&source=mn 1828619156 +2020-02-17 2020-02-17 02:07:55 EMC yvDmRiaYkm43iWRNnnW3YDPBqq55XWVXr 7gpmg2PirkSp]TYhcY7i6]DawnwV9tqC0b 22858229 yDybPZi3Oc7m,JVxfltuRtWm8TmiHQJOjY https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=9158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=3Dmn&c=swm&r=https://click.mail.ru/redir?u=http&o=mail%3DYm_ce%3Dmn mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=158189721581939265D89&email=dG1haWwuY29t&source=pcap 1735264694 +2020-02-17 2020-02-17 00:39:49 EMC BOA0kaQben9xa\fOjQ8YZt2L9UV_FYiMKYa3n 0BL0DaCp3weXj7aIq4ZBhaKzr0Uwpdpm 21820865 xATcz14a2hYJcGJb2gRsIIQ9K2A5fOGo https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail.ru/redir?u=https://mail&v=2&s=90BtYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6ts%3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27457%26ts%3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=http mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=77256160&userId=274634&email=bWFpbC5ydQ%3D%3D&source=mn 1855925025 +2020-02-17 2020-02-17 01:38:01 EMC uNpwiDVW2_p6HEMl4YIHw0z_vOpEqPyir napvpdbMVeSjy KJz5spT94Yu6wGCm_xgkX 27148457 R8wZY_PRklNkxX0Dn6uA{sGHEPaMT3kc8 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=3C3274695&userId=2745528194401908193467BD9189&ts=1581926&ts=1581939772B67B6AE4CD2DBD910B392YW5haWwucnU 2249892028 +2020-02-17 2020-02-17 23:32:22 EMC uKrr0IYx5Kt1\'QPtv@toS9nQ4ZcrvAD0l8 4dWFNMPK50HO0z6ZPSoX6g02Ceu35kLzjtB 25390020 CXq72Vsbolj2.WJUoa8UoTZlw2wHUHxWX https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email&v=2&s=3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D67972 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=BE304F9CDBC6&ts=1581934&email=YW5haWwuY29t&source=mn 3489237198 +2020-02-17 2020-02-17 22:36:42 EMC hQkEQSurISp_uy9ORd05 yjXxg7QByQH9 FURnnJ51xl2NiQHcN\r6hpIkfYa17e26Mt8 25551409 NBIMBrwi mVoYYuxJq_8Df0CM8CZkrZezFE mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=158194463465&userId=25616C3D&source=mn 2913713687 +2020-02-17 2020-02-17 20:19:16 EMC Xahxt0V3`JlMo+tyHm8MxJFT9NEywdtAZeD YV0_62kSEM9X 0GW7eHjXxF95KfKoarfR 27982291 iuEEgZrUYV04*QSEQg_uWbYo8cewVWPriGO mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=DFD92Y29t&source=mn 1136043092 +2020-02-17 2020-02-17 20:05:44 EMC gAraBOGNzDEdkz8qxSB68sQXrgyTFpX6 KC9GBYXCc2sL;fDL4#6DMT%2ELzdoqHtjMp 27849879 hmvHfcpQ7YcXyk6byq3rSqEUfDuei6M62h mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=71&source=mn 795793408 +2020-02-17 2020-02-17 01:03:41 EMC Sujwqk4khZF0dd3dRrn0zsOYRdkd9ynAR NaKyF0O3rR3r0wXNLf_3KSANWona7Ftry 30475228 calDaEsa910bkYOkR\bbKjQeUvAOoR7CQJH https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dmn&c=swm&r=http&o=mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=B1581918970&userId=274634&userId=27457&email=YW5hce=mn 3165615449 +2020-02-17 2020-02-17 18:40:14 EMC zBHwSrKIMfAFFJxuvWtIsZ2DA_nAX98Aa NQiRPz123luXCH0fvH?59tbjHdGAlVgiHhc 26251034 0KbAptV7u8SZ?V4N7WHvx[0vY5D_saLU2G https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=9158192Ymail&v=2&s=936source%3Dmn&c=swm&r=http&o=mail%3DbW9%26userId%3D2746158192YWlsLnJ1%26source mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=5&userId=274552746887BF4LnJ1&source=mn 115808044 +2020-02-17 2020-02-17 22:03:35 EMC dvrVwEyocDVpJ3YEozYAH1XuC51DQcby59 wcCGF71wVXRj0pz2h131p\\1c7jnQYzejlC 24153343 mHlXjDvel9Ab8kZxtR6ELtFOSq7X1Q8_7 https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D65 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=A63468874692&email=ce=pcap 1125992029 +2020-02-17 2020-02-17 13:11:31 EMC ELifUEthNP3xZTZ5RZd2tIgUWpctZJNKkdp 3qxE4qStGm1jIRPc9,JCHKxNAJqeRRgVaJ5 32523658 opEAQkCl*DGc0VozrkcYlaUmWvfN0IAYa https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=3%26userId%3D274570%26userId%3D274615819383%26userId%3D2745746193%26userId%3D2745745776source%3Dmn&c=swm&r=http&o=mail&v=2&s=34Fww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dmn&c=swm&r=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6126source%3Dmn&c=swm&r=https://click.mail%3DbWFpbC5ydQ%253D%253D mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=616&userId=27465&email=YWlsLmNvbQ%3D%3D&source=pcap 660510007 +2020-02-17 2020-02-17 23:08:20 EMC Ww8aun1EXuPT3stlfv12_A4nVx4RfDfGaK Gj6ixLDK{FkfiIQ0ZqHK9IsJMWys80zowLQy 28692974 nB20Riv63uMc]E9s0Bv7x1XcxILAKq2zwc pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=3D&source=pcap 458549627 +2020-02-17 2020-02-17 15:45:11 EMC _nkKyF0NV20B2AAxI@oOj8uSGus_lOrqBS 0vXB63kVoAv97D8cUCQSR\bmet_h2UhiQ4X 31753811 zA7KKsesYWPrI1KD"FFmdemMIJ3AcxwZr https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=9%26ts%3D1581944577457EC6source%3Dmn&c=swm&r=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D67D492YWlsLnJ1%26source%3Dmn&c=swm&r=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D685764%26email&v=2&s=9%26email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=DA8189&email=a2DC9serId=274634&userId=25695&userId=274688&userId=27465 4039013194 +2020-02-17 2020-02-17 14:54:17 EMC wHTfVQKqlnQp!PXzWG80qc0oWU1pvtKeL EaNVlRXuzZ33\fLB0Fmjet29f3xaJplvaiX 30992376 vJnOh3mpOb0h!xPw1slS0e[Ehj1AIx920Jo https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail.ru/redir?u=https://click.mail&v=2&s=3c pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=7462B60&email=cnU%3D&source=mn 2569826295 +2020-02-17 2020-02-17 22:51:48 EMC C1a6zgET p1v0mSMHJJgMxp8w7uJ7zN14J9Z 5cn50cUip60T6nRihVzsSRarErtllzgH6 26715992 0D4M2m041DlURNuFG9QJtIcjHVkp5_TEZ https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail&v=2&s=9446158194419447 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=DEBE34&email=Z21haWwucnU%3D&source=mn 1504060974 +2020-02-17 2020-02-17 13:53:54 EMC 5a2rniMpduiEUzYFsgZjKHqgoZWbDYts ziqQkEkD3tYgLGhe&QwROPlb3hkAIZW_UI 32696355 0rSWK9UmGDPuRvPwXqs3mK9cF0balOqbS http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=77274552E&ts=158192158192YXNvbQ%3D%3D&source=pcap 1675780127 +2020-02-17 2020-02-17 20:08:49 EMC 0IBozTjy/SMIkY5tY\\_GAd\'0VIzFgFK2VSY V0fFnEjkPzK3.y_ng3Nfs1oJtBLCWPq35z 33102601 NQhlj1k9bLy234WJi\rKtOd2239rJDYJSKh https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail.ru/redir?u=http&o=mail.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D61haWwucnU%253D%253D%26source%3Dmn&c=swm&r=https://click.mail%3DYWlsLnJ1%26source%3Dmn&c=swm&r=http&o=mail.ru/redir?u=https://click.mail.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69%26userId%3D2746193A%2F%2Fwwwwwww.wildberries.ru%2Fconfirm%2Fvipstatus mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=D8BD7B082&ts=15819392YW5haWwuY29t&source=mn 832387252 +2020-02-17 2020-02-17 21:39:56 EMC 3aiRtPZF[Sjelo0e2EVnauTwl0tsW9SRi7f i8OkH4au\rfMiBV0E4frxZu9WJpRG5bhp4 29554136 gzPJTobNMGgG1kUYVe9sOwfpHmKiF1p0 https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=http&o=mail%3DbW97B365 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=AE&ts=158194466&ts=15819213CD274688&ts=1581934688&email=Ymn 74272104 +2020-02-17 2020-02-17 17:21:43 EMC yqM2AqiUS2Lmo0S5KNYR-RKGxDrME0w5U T1Q2fyBqs8qhiWB_JBE3Ko_Ro0VD79pT0 18406884 mVoGr7MMzfVX0HtUKAYeCVX68_NOaiQY https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email%3DbWFpbC5ydQ%253D%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=9158192YWlsLnJ1%26source%3Dmn&c=swm&r=https mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=5819392&userId=27465D8B446887AF95&email=bWFpbC5ydQ%3D%3D&source=pcap 1422459332 +2020-02-17 2020-02-17 15:06:14 EMC NXf_EKmdwAkZZIKr0N0w4HuVTpNqZ30Ldk0 PpN_0d7ywY45Y_Wm#L2S01pN5klv0Wbsa 21385681 JH3MQSLtn6gUIe92ilJPT.EFDSucrCuGHZ mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=A4627465D819397D96AE65&email=dG1haWwuY29t&source=pcap 959722117 +2020-02-17 2020-02-17 19:52:47 EMC OY90Zcjc8ebLcK_uULQHDNvpwO1hN3MwIat ASdCffiid6DweXoVjmpL8~5DHOzhFnPRuW 33422761 4T1zxu2F4evId\rETTryZBYsCBkTxsXLeuC9 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=E97265D8A70F&ts=15819403193634&userId=2746887&email=ce=mn 997133189 +2020-02-17 2020-02-17 17:55:20 EMC ManEZ_qtzd0v\bdxdB07LpVWqoZUfpDzZW MnNRUoUE2ASWag8ClKK5l0yPRafHb0gsw 29428950 clyoRyW5\00F6wDd5P188B6k_OdKCbUFQqT mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=52666799692&email=dGF7470E634&email=ce=mn 3015790084 +2020-02-17 2020-02-17 20:07:34 EMC pr7D2hoMay0bwJjvH6UaPLZpeDXtq0Njs bDlkkbc3OnwzdcNZHDcBkR|V0ApitIVgPOv 22120090 TC2aVmxXNfOhSL0y2oqzb=drQElfeo0_OQ mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=581943D&source=mn 2666495492 +2020-02-17 2020-02-17 18:20:19 EMC a_EpwBGWRXNSTY_FyAhRB25SpX8Gw8JaW 0XDllB5u!HDENidqN/CRrkxwnjTXLeEKIyb 32607759 OKByJYVH3pLb:BWhWwvLnnpw0urPjP04i https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27461203Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email&v=2&s=3D158192YWlsLnJ1%26source%3Dpc&c=swm&r=https mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=BC66&ts=158193926&userId=256BFC35FD2AEAZ21haWwucnU%3D&source=mn 590291581 +2020-02-17 2020-02-17 10:27:11 EMC bn8x3TAHKlMOsdlmzybH0S*iDc0hYull7Zt 7PY8hsBErbSTnpoU7iPxD8CFjIMxK8eN8F 30559777 lgByHEG2ffmexKgJXv5rC6CW78KV4Kia mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=6667&email=bWFpbC5ydQ%3D%3D&source=mn 1898373016 +2020-02-17 2020-02-17 14:09:12 EMC jYFR0DCE2eTkEFCuAIyAHVYLUy9ENh9Cqc dhe0lvmTDchj8XhgtsZ3mCPzv5hTGPM8Q 24813187 Y7MmPfU2YhtaY85K"2xIfqNraWWqBp5QvK https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6source%3Dmn&c=swm&r=http&o=mail&v=2&s=96email%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=http&o=mail&v=2&s=944576userId%3D27461819443Fkey%3D646192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D274615819360 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=AD9397158192&userId=25616AEBD63465D88&email=bmRleC5ydQ%3D%3D&source=mn 1726149098 +2020-02-17 2020-02-17 18:27:00 EMC VnzEdMoo2lhW?Yr6AQyNVw yFnUj_l0pBiG EnQ5CpnJt3LM1hiuC7lsqJChM0i4JMZ0 28169388 zBJ8p4pSlPo_bkk4\t2ObECW2ReYnHoWrQH https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27461203Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email&v=2&s=3D158192YWlsLnJ1%26source%3Dmn&c=swm&r=https mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=397729t&source=mn 1102313974 +2020-02-17 2020-02-17 19:49:07 EMC fxpHcK20ouDlcjYFRhFT4DMtbMML0qb8ds hBF63Vf_qZrpsfD4EOfd6YGUBvflIJqF 22835697 TkBqOH2Jjp0wH35uDnjAxqgyb2P4HqtznK https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D2746158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=9158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=92YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail%3DbWFpbC5ydQ%253D%253D%26source%3Dpc&c=swm&r=http&o=mail%3DZG1haWwucnU%253D%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=9%26source%3Dpc&c pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=ADDABAA48E95&userId=2746346887D9420%3D&source=mn 3497823576 +2020-02-17 2020-02-17 13:28:56 EMC hZbZqj2vRV2FpCXlvxIEyqdeQrwceKg35 8fpYiVLE9gfSIzfP_$dzlj1H8tMzVbcg59 20129530 8bY0GHO5KMpHW9cIAke4yQQ94oQfvoHBxY mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=C5ydQ%3D%3D&source=mn 3102735519 +2020-02-17 2020-02-17 21:37:19 EMC TAlSFJzIsXzTWbuEymOTU320iUcNXmLT5j J2Vf06b6CMLFAvwH_wsUNOv4O9nnCy20 17587521 nEczrivzdrNaBYYvdnQRUT2pBBo8hu3kCvI mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=66&email=YWlsLnJ1&source=mn 1613554206 +2020-02-17 2020-02-17 21:58:32 EMC XPXxpWq0wHcK2FB4$JaOa9xIGhEaajLeS l59HKsiwgQuXgAweGZRZrEkN0Pqk_plZ4c 24296347 3s8mV_3W&hisrcwCm6QXCg_0zgLyr6nnqPB https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D274616 pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=467105695&email=a2YWlsLnJ1&source=pcap 2033387311 +2020-02-17 2020-02-17 16:54:16 EMC 2beVLC_ZYMmZuZbJZ1mjV LQeoXV0BsZKq J4Y22CvJx36nq6KNDMXBg!H5lUx1Z7BoFf 31434491 dabaBlCY351pXUW0HdErDTW_XGkqwmta https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6userId%3D27461203Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6Fconfirm%2Fvipstatus%3Fkey%3D661076userId%3D27457CD15819444613 mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=76031938874634&userId=2564BFA93626&userId=256&email=ce=mn 1613641575 +2020-02-17 2020-02-17 00:27:36 EMC Au3PRMFCW0Ij8iL0ZtsrryAvKJowqXJ8k ntqK0ayP8ltR0twuGraE`Vwgcmux6jFwg 21307924 ruixkHLKjmENcmpbvvbrrLU8FDLIpWOFtU https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://click.mail.ru/redir?u=https%3A%2F%2Fwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6email.ru/redir?u=https%3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D6CD2745785dQ mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=A41E772FFC5ydQ%3D%3D&source=pcap 2395792653 +2020-02-17 2020-02-17 22:03:46 EMC Tc9EmNpb5Xy0edNdTCLg7X2eLmGuTdEkIt YsCMp8oiUBPyhHBsDmEZs0xvixA2L6atY 23431515 Bh67PWI6jKZ6Q102nTeN(YP6e8kLoqiVA https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail&v=2&s=96userId%3D274616 pcap 1 Ru ru 7.0.32 /confirm/vipstatus ?key=5D819446&userId=27465D8191897C7462F400&userId=27463465D8192&email=Z21haWwuY29t&source=pcap 2211382541 +2020-02-17 2020-02-17 06:34:15 EMC S13SmJGaHVUBXqd3bs\tRpB3G_hXZBJjxOt6 FP_GWMlfw2REi3Up5Cqu0qWcA99YlCduy 27883082 xJ1eD6ixour7M0K_l6Ziv6awoxMpnzsaW https://click.mail%3DYmail%3Dberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D69444%26email.ru/redir?u=https://mail%3Dby%3D6%26ts%3D158193DBDB3A%2F%2Fwwww.wildberries.ru%2Fconfirm%2Fvipstatus%3Fkey%3D63158192YWlsLnJ1%26source%3Dmn&c=swm&r=https://click.mail&v=2&s=92YWlsLnJ1%26source mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=15819362hQG1haWwuY29t&source=pcap 2045856196 +2020-02-17 2020-02-17 00:36:06 EMC 2mjWiSh70SN_1xOBst7vbYq1z5EL26Z0i 7e5aIak4Pca2woYGZIJi!P_gRQdnPpHDP 28784173 b5hw0LlyfMCyYstJ9XbB6Pq7VN5hO9kIy mn 1 Ru ru 7.0.32 /confirm/vipstatus ?key=15819387AE3634&email=cnU%3D&source=mn 1255281910 From 5548c9d1ad8b752cac133193d774e6a84a03321c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2020 01:53:15 +0300 Subject: [PATCH 30/38] Repeat test on curl timeout --- tests/clickhouse-test | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 1a2735296e8..86a9ecebd96 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -30,6 +30,7 @@ from contextlib import closing MESSAGES_TO_RETRY = [ "DB::Exception: ZooKeeper session has been expired", "Coordination::Exception: Connection loss", + "Operation timed out", ] From 7df5280060cd05942b255710acb0815136d67887 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2020 02:09:11 +0300 Subject: [PATCH 31/38] Fix potentially flacky test --- .../0_stateless/00731_long_merge_tree_select_opened_files.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index 5177cb6087c..5ae9dcf67ec 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -14,9 +14,11 @@ $CLICKHOUSE_CLIENT $settings -n -q " DROP TABLE IF EXISTS merge_tree_table; CREATE TABLE merge_tree_table (id UInt64, date Date, uid UInt32) ENGINE = MergeTree(date, id, 8192);" + $CLICKHOUSE_CLIENT $settings -q "INSERT INTO merge_tree_table SELECT (intHash64(number)) % 10000, toDate('2018-08-01'), rand() FROM system.numbers LIMIT 10000000;" -$CLICKHOUSE_CLIENT $settings -q "OPTIMIZE TABLE merge_tree_table FINAL;" +# If merge is already happening, OPTIMIZE will be noop. But we have to ensure that the data is merged. +for i in {1..100}; do $CLICKHOUSE_CLIENT $settings --optimize_throw_if_noop=1 -q "OPTIMIZE TABLE merge_tree_table FINAL;" && break; sleep 1; done # The query may open more files if query log will be flushed during the query. # To lower this chance, we also flush logs before the query. From e8f52337aacb261f55d72716a09bc85490464286 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2020 03:09:12 +0300 Subject: [PATCH 32/38] Fix incorrect usage of fmtlib --- base/common/logger_useful.h | 18 +++++++++++++++--- src/Common/tests/gtest_log.cpp | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 src/Common/tests/gtest_log.cpp diff --git a/base/common/logger_useful.h b/base/common/logger_useful.h index c89ec63ef22..ad56ef7e857 100644 --- a/base/common/logger_useful.h +++ b/base/common/logger_useful.h @@ -15,15 +15,27 @@ using Poco::Message; using DB::LogsLevel; using DB::CurrentThread; -/// Logs a message to a specified logger with that level. -#define LOG_IMPL(logger, priority, PRIORITY, ...) do \ +namespace +{ + template constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); } + template constexpr auto firstArg(T && x, Ts &&...) { return std::forward(x); } +} + + +/// Logs a message to a specified logger with that level. +/// If more than one argument is provided, +/// the first argument is interpreted as template with {}-substitutions +/// and the latter arguments treat as values to substitute. +/// If only one argument is provided, it is threat as message without substitutions. + +#define LOG_IMPL(logger, priority, PRIORITY, ...) do \ { \ const bool is_clients_log = (CurrentThread::getGroup() != nullptr) && \ (CurrentThread::getGroup()->client_logs_level >= (priority)); \ if ((logger)->is((PRIORITY)) || is_clients_log) \ { \ - std::string formatted_message = fmt::format(__VA_ARGS__); \ + std::string formatted_message = numArgs(__VA_ARGS__) > 1 ? fmt::format(__VA_ARGS__) : firstArg(__VA_ARGS__); \ if (auto channel = (logger)->getChannel()) \ { \ std::string file_function; \ diff --git a/src/Common/tests/gtest_log.cpp b/src/Common/tests/gtest_log.cpp new file mode 100644 index 00000000000..a1e532f92e6 --- /dev/null +++ b/src/Common/tests/gtest_log.cpp @@ -0,0 +1,19 @@ +#include +#include +#include +#include + +#include +#include +#include + + +TEST(Logger, Log) +{ + Poco::Logger::root().setLevel("none"); + Poco::Logger::root().setChannel(Poco::AutoPtr(new Poco::NullChannel())); + Logger * log = &Logger::get("Log"); + + /// This test checks that we don't pass this string to fmtlib, because it is the only argument. + EXPECT_NO_THROW(LOG_INFO(log, "Hello {} World")); +} From 263bc6e097a271df52af743caa6339db2a95abfb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2020 03:26:56 +0300 Subject: [PATCH 33/38] Added missing file --- tests/queries/0_stateless/00342_escape_sequences.reference | 1 + 1 file changed, 1 insertion(+) create mode 100644 tests/queries/0_stateless/00342_escape_sequences.reference diff --git a/tests/queries/0_stateless/00342_escape_sequences.reference b/tests/queries/0_stateless/00342_escape_sequences.reference new file mode 100644 index 00000000000..50e0756d16f --- /dev/null +++ b/tests/queries/0_stateless/00342_escape_sequences.reference @@ -0,0 +1 @@ +07080C0A0D090B5C27225C3FAA From 13b0acdc9f44e474a0c02fa186dc2372d33130d6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2020 03:42:25 +0300 Subject: [PATCH 34/38] Fix strange code --- docker/test/test_runner.sh | 76 ++++++++++++++--------------- utils/ci/docker-multiarch/update.sh | 60 +++++++++++------------ 2 files changed, 68 insertions(+), 68 deletions(-) diff --git a/docker/test/test_runner.sh b/docker/test/test_runner.sh index 76e142e61f9..561117492b0 100755 --- a/docker/test/test_runner.sh +++ b/docker/test/test_runner.sh @@ -12,7 +12,7 @@ readonly CLICKHOUSE_PACKAGES_ARG="${2}" CLICKHOUSE_SERVER_IMAGE="${3}" if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then - readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild + readonly CLICKHOUSE_PACKAGES_DIR="$(realpath ${2})" # or --no-rebuild fi @@ -26,19 +26,19 @@ fi # TODO: optionally mount most recent clickhouse-test and queries directory from local machine if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then - docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \ - --target clickhouse-test-runner-base \ - -t clickhouse-test-runner-base:preinstall \ - "${CLICKHOUSE_DOCKER_DIR}/test/stateless" + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/test/stateless/clickhouse-statelest-test-runner.Dockerfile" \ + --target clickhouse-test-runner-base \ + -t clickhouse-test-runner-base:preinstall \ + "${CLICKHOUSE_DOCKER_DIR}/test/stateless" - docker rm -f clickhouse-test-runner-installing-packages || true - docker run \ - -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ - --name clickhouse-test-runner-installing-packages \ - clickhouse-test-runner-base:preinstall - docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local - docker rm -f clickhouse-test-runner-installing-packages || true + docker rm -f clickhouse-test-runner-installing-packages || true + docker run \ + -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ + --name clickhouse-test-runner-installing-packages \ + clickhouse-test-runner-base:preinstall + docker commit clickhouse-test-runner-installing-packages clickhouse-statelest-test-runner:local + docker rm -f clickhouse-test-runner-installing-packages || true fi # # Create a bind-volume to the clickhouse-test script file @@ -47,38 +47,38 @@ fi # Build server image (optional) from local packages if [ -z "${CLICKHOUSE_SERVER_IMAGE}" ]; then - CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local" + CLICKHOUSE_SERVER_IMAGE="yandex/clickhouse-server:local" - if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then - docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ - --target clickhouse-server-base \ - -t clickhouse-server-base:preinstall \ - "${CLICKHOUSE_DOCKER_DIR}/server" + if [ ${CLICKHOUSE_PACKAGES_ARG} != ${NO_REBUILD_FLAG} ]; then + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ + --target clickhouse-server-base \ + -t clickhouse-server-base:preinstall \ + "${CLICKHOUSE_DOCKER_DIR}/server" - docker rm -f clickhouse_server_base_installing_server || true - docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ - --name clickhouse_server_base_installing_server \ - clickhouse-server-base:preinstall - docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall + docker rm -f clickhouse_server_base_installing_server || true + docker run -v "${CLICKHOUSE_PACKAGES_DIR}:/packages" \ + --name clickhouse_server_base_installing_server \ + clickhouse-server-base:preinstall + docker commit clickhouse_server_base_installing_server clickhouse-server-base:postinstall - docker build \ - -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ - --target clickhouse-server \ - -t "${CLICKHOUSE_SERVER_IMAGE}" \ - "${CLICKHOUSE_DOCKER_DIR}/server" - fi + docker build \ + -f "${CLICKHOUSE_DOCKER_DIR}/server/local.Dockerfile" \ + --target clickhouse-server \ + -t "${CLICKHOUSE_SERVER_IMAGE}" \ + "${CLICKHOUSE_DOCKER_DIR}/server" + fi fi docker rm -f test-runner || true docker-compose down CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \ - docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ - create \ - --build --force-recreate + docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ + create \ + --build --force-recreate CLICKHOUSE_SERVER_IMAGE="${CLICKHOUSE_SERVER_IMAGE}" \ - docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ - run \ - --name test-runner \ - test-runner + docker-compose -f "${CLICKHOUSE_DOCKER_DIR}/test/test_runner_docker_compose.yaml" \ + run \ + --name test-runner \ + test-runner diff --git a/utils/ci/docker-multiarch/update.sh b/utils/ci/docker-multiarch/update.sh index 6e3e18047de..6abcf339607 100755 --- a/utils/ci/docker-multiarch/update.sh +++ b/utils/ci/docker-multiarch/update.sh @@ -37,27 +37,27 @@ fi # get the image if \ - wget -q --spider "$baseUrl/current" \ - && wget -q --spider "$baseUrl/current/$thisTar" \ - ; then - baseUrl+='/current' + wget -q --spider "$baseUrl/current" \ + && wget -q --spider "$baseUrl/current/$thisTar" \ + ; then + baseUrl+='/current' fi wget -qN "$baseUrl/"{{MD5,SHA{1,256}}SUMS{,.gpg},"$thisTarBase.manifest",'unpacked/build-info.txt'} || true wget -N "$baseUrl/$thisTar" # check checksum if [ -f SHA256SUMS ]; then - sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)" - if ! grep -q "$sha256sum" SHA256SUMS; then - echo >&2 "error: '$thisTar' has invalid SHA256" - exit 1 - fi + sha256sum="$(sha256sum "$thisTar" | cut -d' ' -f1)" + if ! grep -q "$sha256sum" SHA256SUMS; then + echo >&2 "error: '$thisTar' has invalid SHA256" + exit 1 + fi fi cat > Dockerfile <<-EOF - FROM scratch - ADD $thisTar / - ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO} + FROM scratch + ADD $thisTar / + ENV ARCH=${ARCH} UBUNTU_SUITE=${VERSION} DOCKER_REPO=${DOCKER_REPO} EOF # add qemu-user-static binary @@ -70,26 +70,26 @@ EOF fi cat >> Dockerfile <<-EOF - # a few minor docker-specific tweaks - # see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap - RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\ - && echo 'exit 101' >> /usr/sbin/policy-rc.d \\ - && chmod +x /usr/sbin/policy-rc.d \\ - && dpkg-divert --local --rename --add /sbin/initctl \\ - && cp -a /usr/sbin/policy-rc.d /sbin/initctl \\ - && sed -i 's/^exit.*/exit 0/' /sbin/initctl \\ - && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\ - && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\ - && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\ - && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\ - && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\ - && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes + # a few minor docker-specific tweaks + # see https://github.com/docker/docker/blob/master/contrib/mkimage/debootstrap + RUN echo '#!/bin/sh' > /usr/sbin/policy-rc.d \\ + && echo 'exit 101' >> /usr/sbin/policy-rc.d \\ + && chmod +x /usr/sbin/policy-rc.d \\ + && dpkg-divert --local --rename --add /sbin/initctl \\ + && cp -a /usr/sbin/policy-rc.d /sbin/initctl \\ + && sed -i 's/^exit.*/exit 0/' /sbin/initctl \\ + && echo 'force-unsafe-io' > /etc/dpkg/dpkg.cfg.d/docker-apt-speedup \\ + && echo 'DPkg::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' > /etc/apt/apt.conf.d/docker-clean \\ + && echo 'APT::Update::Post-Invoke { "rm -f /var/cache/apt/archives/*.deb /var/cache/apt/archives/partial/*.deb /var/cache/apt/*.bin || true"; };' >> /etc/apt/apt.conf.d/docker-clean \\ + && echo 'Dir::Cache::pkgcache ""; Dir::Cache::srcpkgcache "";' >> /etc/apt/apt.conf.d/docker-clean \\ + && echo 'Acquire::Languages "none";' > /etc/apt/apt.conf.d/docker-no-languages \\ + && echo 'Acquire::GzipIndexes "true"; Acquire::CompressionTypes::Order:: "gz";' > /etc/apt/apt.conf.d/docker-gzip-indexes - # enable the universe - RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list + # enable the universe + RUN sed -i 's/^#\s*\(deb.*universe\)$/\1/g' /etc/apt/sources.list - # overwrite this with 'CMD []' in a dependent Dockerfile - CMD ["/bin/bash"] + # overwrite this with 'CMD []' in a dependent Dockerfile + CMD ["/bin/bash"] EOF docker build -t "${DOCKER_REPO}:${TAG_ARCH}-${VERSION}" . From 025c1a8830ff3fcd7a2b42557ca5942fb38fdd49 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2020 03:46:42 +0300 Subject: [PATCH 35/38] Added another check #11170 --- tests/clickhouse-test | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 86a9ecebd96..2d155313b06 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -31,6 +31,7 @@ MESSAGES_TO_RETRY = [ "DB::Exception: ZooKeeper session has been expired", "Coordination::Exception: Connection loss", "Operation timed out", + "ConnectionPoolWithFailover: Connection failed at try", ] From 2d12b4d3ac606296675496c90fafcd62d7b17d30 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 29 May 2020 09:54:27 +0300 Subject: [PATCH 36/38] Fix crash when SET DEFAULT ROLE is called with wrong arguments. --- src/Access/ExtendedRoleSet.cpp | 24 ++++-- src/Interpreters/InterpreterSetRoleQuery.cpp | 2 +- src/Parsers/ASTExtendedRoleSet.h | 5 +- src/Parsers/ParserCreateUserQuery.cpp | 1 + src/Parsers/ParserSetRoleQuery.cpp | 2 + .../integration/test_default_role/__init__.py | 0 tests/integration/test_default_role/test.py | 77 +++++++++++++++++++ 7 files changed, 103 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_default_role/__init__.py create mode 100644 tests/integration/test_default_role/test.py diff --git a/src/Access/ExtendedRoleSet.cpp b/src/Access/ExtendedRoleSet.cpp index a29ee40380c..a8e674b3722 100644 --- a/src/Access/ExtendedRoleSet.cpp +++ b/src/Access/ExtendedRoleSet.cpp @@ -68,15 +68,27 @@ void ExtendedRoleSet::init(const ASTExtendedRoleSet & ast, const AccessControlMa { all = ast.all; - auto name_to_id = [id_mode{ast.id_mode}, manager](const String & name) -> UUID + auto name_to_id = [&ast, manager](const String & name) -> UUID { - if (id_mode) + if (ast.id_mode) return parse(name); assert(manager); - auto id = manager->find(name); - if (id) - return *id; - return manager->getID(name); + if (ast.can_contain_users && ast.can_contain_roles) + { + auto id = manager->find(name); + if (id) + return *id; + return manager->getID(name); + } + else if (ast.can_contain_users) + { + return manager->getID(name); + } + else + { + assert(ast.can_contain_roles); + return manager->getID(name); + } }; if (!ast.names.empty() && !all) diff --git a/src/Interpreters/InterpreterSetRoleQuery.cpp b/src/Interpreters/InterpreterSetRoleQuery.cpp index f8e0167d748..c627061dd51 100644 --- a/src/Interpreters/InterpreterSetRoleQuery.cpp +++ b/src/Interpreters/InterpreterSetRoleQuery.cpp @@ -62,7 +62,7 @@ void InterpreterSetRoleQuery::setRole(const ASTSetRoleQuery & query) void InterpreterSetRoleQuery::setDefaultRole(const ASTSetRoleQuery & query) { - context.checkAccess(AccessType::CREATE_USER | AccessType::DROP_USER); + context.checkAccess(AccessType::ALTER_USER); auto & access_control = context.getAccessControlManager(); std::vector to_users = ExtendedRoleSet{*query.to_users, access_control, context.getUserID()}.getMatchingIDs(access_control); diff --git a/src/Parsers/ASTExtendedRoleSet.h b/src/Parsers/ASTExtendedRoleSet.h index 8d619e5d6a0..656f563bd9a 100644 --- a/src/Parsers/ASTExtendedRoleSet.h +++ b/src/Parsers/ASTExtendedRoleSet.h @@ -15,7 +15,10 @@ public: bool all = false; Strings except_names; bool except_current_user = false; - bool id_mode = false; /// If true then `names` and `except_names` keeps UUIDs, not names. + + bool id_mode = false; /// true if `names` and `except_names` keep UUIDs, not names. + bool can_contain_roles = true; /// true if this set can contain names of roles. + bool can_contain_users = true; /// true if this set can contain names of users. bool empty() const { return names.empty() && !current_user && !all; } void replaceCurrentUserTagWithName(const String & current_user_name); diff --git a/src/Parsers/ParserCreateUserQuery.cpp b/src/Parsers/ParserCreateUserQuery.cpp index 76a06a0282f..a6a007855e2 100644 --- a/src/Parsers/ParserCreateUserQuery.cpp +++ b/src/Parsers/ParserCreateUserQuery.cpp @@ -227,6 +227,7 @@ namespace return false; default_roles = typeid_cast>(ast); + default_roles->can_contain_users = false; return true; }); } diff --git a/src/Parsers/ParserSetRoleQuery.cpp b/src/Parsers/ParserSetRoleQuery.cpp index e6ff7893891..a69480f89eb 100644 --- a/src/Parsers/ParserSetRoleQuery.cpp +++ b/src/Parsers/ParserSetRoleQuery.cpp @@ -18,6 +18,7 @@ namespace return false; roles = typeid_cast>(ast); + roles->can_contain_users = false; return true; }); } @@ -34,6 +35,7 @@ namespace return false; to_users = typeid_cast>(ast); + to_users->can_contain_roles = false; return true; }); } diff --git a/tests/integration/test_default_role/__init__.py b/tests/integration/test_default_role/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_default_role/test.py b/tests/integration/test_default_role/test.py new file mode 100644 index 00000000000..2b6b4698b20 --- /dev/null +++ b/tests/integration/test_default_role/test.py @@ -0,0 +1,77 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.test_tools import TSV +import re + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance('instance') + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + instance.query("CREATE USER john") + instance.query("CREATE ROLE rx") + instance.query("CREATE ROLE ry") + + yield cluster + + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def reset_users_and_roles(): + instance.query("CREATE USER OR REPLACE john") + yield + + +def test_set_default_roles(): + assert instance.query("SHOW CURRENT ROLES", user="john") == "" + + instance.query("GRANT rx, ry TO john") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + + instance.query("SET DEFAULT ROLE NONE TO john") + assert instance.query("SHOW CURRENT ROLES", user="john") == "" + + instance.query("SET DEFAULT ROLE rx TO john") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] ) + + instance.query("SET DEFAULT ROLE ry TO john") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) + + instance.query("SET DEFAULT ROLE ALL TO john") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + + instance.query("SET DEFAULT ROLE ALL EXCEPT rx TO john") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) + + +def test_alter_user(): + assert instance.query("SHOW CURRENT ROLES", user="john") == "" + + instance.query("GRANT rx, ry TO john") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + + instance.query("ALTER USER john DEFAULT ROLE NONE") + assert instance.query("SHOW CURRENT ROLES", user="john") == "" + + instance.query("ALTER USER john DEFAULT ROLE rx") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1]] ) + + instance.query("ALTER USER john DEFAULT ROLE ALL") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['rx', 0, 1], ['ry', 0, 1]] ) + + instance.query("ALTER USER john DEFAULT ROLE ALL EXCEPT rx") + assert instance.query("SHOW CURRENT ROLES", user="john") == TSV( [['ry', 0, 1]] ) + + +def test_wrong_set_default_role(): + assert "There is no user `rx`" in instance.query_and_get_error("SET DEFAULT ROLE NONE TO rx") + assert "There is no user `ry`" in instance.query_and_get_error("SET DEFAULT ROLE rx TO ry") + assert "There is no role `john`" in instance.query_and_get_error("SET DEFAULT ROLE john TO john") + assert "There is no role `john`" in instance.query_and_get_error("ALTER USER john DEFAULT ROLE john") + assert "There is no role `john`" in instance.query_and_get_error("ALTER USER john DEFAULT ROLE ALL EXCEPT john") From 09369c259c1cd343ef6a0324b0b93e110170cc10 Mon Sep 17 00:00:00 2001 From: BayoNet Date: Fri, 29 May 2020 12:43:49 +0300 Subject: [PATCH 37/38] DOCS-527: system.query_log (#11197) * DOCSUP-824: query log (#115) * Draft changes. * Next step. * Structure was changed. Missed information was added. * More edits. * Update docs/en/operations/system-tables.md Co-authored-by: BayoNet * Update docs/en/operations/system-tables.md Co-authored-by: BayoNet * Update docs/en/operations/system-tables.md Co-authored-by: BayoNet * Comments editing was done. * Translated to russian. Co-authored-by: Anastasia Uvarova Co-authored-by: AnaUvarova <64017504+AnaUvarova@users.noreply.github.com> * CLICKHOUSEDOCS-527: query_log description. * CLICKHOUSEDOCS-527: Updated structure. * CLICKHOUSEDOCS-527: Fixed links. * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Update docs/en/operations/system-tables.md Co-authored-by: Ivan Blinkov * Updated by comments. * CLICKHOUSEDOCS-527: Updated by comments . * CLICKHOUSEDOCS-527: Syncronized EN->RU Co-authored-by: Anastasia Uvarova Co-authored-by: AnaUvarova <64017504+AnaUvarova@users.noreply.github.com> Co-authored-by: Sergei Shtykov Co-authored-by: Ivan Blinkov --- .../settings.md | 4 +- docs/en/operations/settings/settings.md | 2 +- docs/en/operations/system-tables.md | 180 ++++++++---- .../settings.md | 4 +- docs/ru/operations/settings/settings.md | 2 +- docs/ru/operations/system-tables.md | 271 ++++++++---------- 6 files changed, 248 insertions(+), 215 deletions(-) diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index 93acf3cae7a..5961c701283 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -586,11 +586,11 @@ If the table doesn’t exist, ClickHouse will create it. If the structure of the ``` -## query\_thread\_log {#server_configuration_parameters-query-thread-log} +## query\_thread\_log {#server_configuration_parameters-query_thread_log} Setting for logging threads of queries received with the [log\_query\_threads=1](../settings/settings.md#settings-log-query-threads) setting. -Queries are logged in the [system.query\_thread\_log](../../operations/system-tables.md#system_tables-query-thread-log) table, not in a separate file. You can change the name of the table in the `table` parameter (see below). +Queries are logged in the [system.query\_thread\_log](../../operations/system-tables.md#system_tables-query_thread_log) table, not in a separate file. You can change the name of the table in the `table` parameter (see below). Use the following parameters to configure logging: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 7103819e499..880f0ffedb1 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -598,7 +598,7 @@ log_queries_min_type='EXCEPTION_WHILE_PROCESSING' Setting up query threads logging. -Queries’ threads runned by ClickHouse with this setup are logged according to the rules in the [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server configuration parameter. +Queries’ threads runned by ClickHouse with this setup are logged according to the rules in the [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server configuration parameter. Example: diff --git a/docs/en/operations/system-tables.md b/docs/en/operations/system-tables.md index cc5456f5324..f364d4e3068 100644 --- a/docs/en/operations/system-tables.md +++ b/docs/en/operations/system-tables.md @@ -5,7 +5,7 @@ toc_title: System Tables # System Tables {#system-tables} -## Introduction +## Introduction {#system-tables-introduction} System tables provide information about: @@ -18,9 +18,12 @@ System tables: - Available only for reading data. - Can't be dropped or altered, but can be detached. -The `metric_log`, `query_log`, `query_thread_log`, `trace_log` system tables store data in a storage filesystem. Other system tables store their data in RAM. ClickHouse server creates such system tables at the start. +Most of system tables store their data in RAM. ClickHouse server creates such system tables at the start. -### Sources of System Metrics +The [metric_log](#system_tables-metric_log), [query_log](#system_tables-query_log), [query_thread_log](#system_tables-query_thread_log), [trace_log](#system_tables-trace_log) system tables store data in a storage filesystem. You can alter them or remove from a disk manually. If you remove one of that tables from a disk, the ClickHouse server creates the table again at the time of the next recording. A storage period for these tables is not limited, and ClickHouse server doesn't delete their data automatically. You need to organize removing of outdated logs by yourself. For example, you can use [TTL](../sql-reference/statements/alter.md#manipulations-with-table-ttl) settings for removing outdated log records. + + +### Sources of System Metrics {#system-tables-sources-of-system-metrics} For collecting system metrics ClickHouse server uses: @@ -587,97 +590,150 @@ Columns: - `source_file` (LowCardinality(String)) — Source file from which the logging was done. - `source_line` (UInt64) — Source line from which the logging was done. -## system.query\_log {#system_tables-query_log} +## system.query_log {#system_tables-query_log} -Contains information about execution of queries. For each query, you can see processing start time, duration of processing, error messages and other information. +Contains information about executed queries, for example, start time, duration of processing, error messages. !!! note "Note" The table doesn’t contain input data for `INSERT` queries. -ClickHouse creates this table only if the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. +You can change settings of queries logging in the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) section of the server configuration. -To enable query logging, set the [log\_queries](settings/settings.md#settings-log-queries) parameter to 1. For details, see the [Settings](settings/settings.md) section. +You can disable queries logging by setting [log_queries = 0](settings/settings.md#settings-log-queries). We don't recommend to turn off logging because information in this table is important for solving issues. + +The flushing period of logs is set in `flush_interval_milliseconds` parameter of the [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server settings section. To force flushing logs, use the [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs) query. + +ClickHouse doesn't delete logs from the table automatically. See [Introduction](#system-tables-introduction) for more details. The `system.query_log` table registers two kinds of queries: 1. Initial queries that were run directly by the client. 2. Child queries that were initiated by other queries (for distributed query execution). For these types of queries, information about the parent queries is shown in the `initial_*` columns. +Each query creates one or two rows in the `query_log` table, depending on the status (see the `type` column) of the query: + +1. If the query execution was successful, two rows with the `QueryStart` and `QueryFinish` types are created . +2. If an error occurred during query processing, two events with the `QueryStart` and `ExceptionWhileProcessing` types are created . +3. If an error occurred before launching the query, a single event with the `ExceptionBeforeStart` type is created. + Columns: -- `type` (`Enum8`) — Type of event that occurred when executing the query. Values: +- `type` ([Enum8](../sql-reference/data-types/enum.md)) — Type of an event that occurred when executing the query. Values: - `'QueryStart' = 1` — Successful start of query execution. - `'QueryFinish' = 2` — Successful end of query execution. - `'ExceptionBeforeStart' = 3` — Exception before the start of query execution. - `'ExceptionWhileProcessing' = 4` — Exception during the query execution. -- `event_date` (Date) — Query starting date. -- `event_time` (DateTime) — Query starting time. -- `query_start_time` (DateTime) — Start time of query execution. -- `query_duration_ms` (UInt64) — Duration of query execution. -- `read_rows` (UInt64) — Number of read rows. -- `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. -- `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. -- `result_rows` (UInt64) — Number of rows in the result. -- `result_bytes` (UInt64) — Number of bytes in the result. -- `memory_usage` (UInt64) — Memory consumption by the query. -- `query` (String) — Query string. -- `exception` (String) — Exception message. -- `stack_trace` (String) — Stack trace (a list of methods called before the error occurred). An empty string, if the query is completed successfully. -- `is_initial_query` (UInt8) — Query type. Possible values: +- `event_date` ([Date](../sql-reference/data-types/date.md)) — Query starting date. +- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Query starting time. +- `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — Start time of query execution. +- `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution in milliseconds. +- `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or rows read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_rows` includes the total number of rows read at all replicas. Each replica sends it's `read_rows` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn't affect this value. +- `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Total number or bytes read from all tables and table functions participated in query. It includes usual subqueries, subqueries for `IN` and `JOIN`. For distributed queries `read_bytes` includes the total number of rows read at all replicas. Each replica sends it's `read_bytes` value, and the server-initiator of the query summarize all received and local values. The cache volumes doesn't affect this value. +- `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. +- `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. +- `result_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of rows in a result of the `SELECT` query, or a number of rows in the `INSERT` query. +- `result_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — RAM volume in bytes used to store a query result. +- `memory_usage` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Memory consumption by the query. +- `query` ([String](../sql-reference/data-types/string.md)) — Query string. +- `exception` ([String](../sql-reference/data-types/string.md)) — Exception message. +- `exception_code` ([Int32](../sql-reference/data-types/int-uint.md)) — Code of an exception. +- `stack_trace` ([String](../sql-reference/data-types/string.md)) — [Stack trace](https://en.wikipedia.org/wiki/Stack_trace). An empty string, if the query was completed successfully. +- `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md)) — Query type. Possible values: - 1 — Query was initiated by the client. - - 0 — Query was initiated by another query for distributed query execution. -- `user` (String) — Name of the user who initiated the current query. -- `query_id` (String) — ID of the query. -- `address` (IPv6) — IP address that was used to make the query. -- `port` (UInt16) — The client port that was used to make the query. -- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). -- `initial_query_id` (String) — ID of the initial query (for distributed query execution). -- `initial_address` (IPv6) — IP address that the parent query was launched from. -- `initial_port` (UInt16) — The client port that was used to make the parent query. -- `interface` (UInt8) — Interface that the query was initiated from. Possible values: + - 0 — Query was initiated by another query as part of distributed query execution. +- `user` ([String](../sql-reference/data-types/string.md)) — Name of the user who initiated the current query. +- `query_id` ([String](../sql-reference/data-types/string.md)) — ID of the query. +- `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query. +- `port` ([UInt16](../sql-reference/data-types/int-uint.md)) — The client port that was used to make the query. +- `initial_user` ([String](../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution). +- `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution). +- `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from. +- `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md)) — The client port that was used to make the parent query. +- `interface` ([UInt8](../sql-reference/data-types/int-uint.md)) — Interface that the query was initiated from. Possible values: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md). -- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run. -- `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name. -- `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version. +- `os_user` ([String](../sql-reference/data-types/string.md)) — Operating system username who runs [clickhouse-client](../interfaces/cli.md). +- `client_hostname` ([String](../sql-reference/data-types/string.md)) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run. +- `client_name` ([String](../sql-reference/data-types/string.md)) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name. +- `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. +- `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version. - `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - 0 — The query was launched from the TCP interface. - 1 — `GET` method was used. - 2 — `POST` method was used. -- `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request. -- `quota_key` (String) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`). -- `revision` (UInt32) — ClickHouse revision. -- `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution. -- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [system.events](#system_tables-events) -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` column. -- `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1. -- `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` column. +- `http_user_agent` ([String](../sql-reference/data-types/string.md)) — The `UserAgent` header passed in the HTTP request. +- `quota_key` ([String](../sql-reference/data-types/string.md)) — The “quota key” specified in the [quotas](quotas.md) setting (see `keyed`). +- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ClickHouse revision. +- `thread_numbers` ([Array(UInt32)](../sql-reference/data-types/array.md)) — Number of threads that are participating in query execution. +- `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Counters that measure different metrics. The description of them could be found in the table [system.events](#system_tables-events) +- `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — Values of metrics that are listed in the `ProfileEvents.Names` column. +- `Settings.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1. +- `Settings.Values` ([Array(String)](../sql-reference/data-types/array.md)) — Values of settings that are listed in the `Settings.Names` column. -Each query creates one or two rows in the `query_log` table, depending on the status of the query: +**Example** -1. If the query execution is successful, two events with types 1 and 2 are created (see the `type` column). -2. If an error occurred during query processing, two events with types 1 and 4 are created. -3. If an error occurred before launching the query, a single event with type 3 is created. +``` sql +SELECT * FROM system.query_log LIMIT 1 FORMAT Vertical; +``` -By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. +``` text +Row 1: +────── +type: QueryStart +event_date: 2020-05-13 +event_time: 2020-05-13 14:02:28 +query_start_time: 2020-05-13 14:02:28 +query_duration_ms: 0 +read_rows: 0 +read_bytes: 0 +written_rows: 0 +written_bytes: 0 +result_rows: 0 +result_bytes: 0 +memory_usage: 0 +query: SELECT 1 +exception_code: 0 +exception: +stack_trace: +is_initial_query: 1 +user: default +query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +address: ::ffff:127.0.0.1 +port: 57720 +initial_user: default +initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +initial_address: ::ffff:127.0.0.1 +initial_port: 57720 +interface: 1 +os_user: bayonet +client_hostname: clickhouse.ru-central1.internal +client_name: ClickHouse client +client_revision: 54434 +client_version_major: 20 +client_version_minor: 4 +client_version_patch: 1 +http_method: 0 +http_user_agent: +quota_key: +revision: 54434 +thread_ids: [] +ProfileEvents.Names: [] +ProfileEvents.Values: [] +Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage'] +Settings.Values: ['0','random','1','10000000000'] -When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted. +``` +**See Also** -!!! note "Note" - The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself. +- [system.query_thread_log](#system_tables-query_thread_log) — This table contains information about each query execution thread. -You can specify an arbitrary partitioning key for the `system.query_log` table in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `partition_by` parameter). - -## system.query\_thread\_log {#system_tables-query-thread-log} +## system.query_thread_log {#system_tables-query_thread_log} The table contains information about each query execution thread. -ClickHouse creates this table only if the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. +ClickHouse creates this table only if the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. To enable query logging, set the [log\_query\_threads](settings/settings.md#settings-log-query-threads) parameter to 1. For details, see the [Settings](settings/settings.md) section. @@ -729,14 +785,14 @@ Columns: - `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics for this thread. The description of them could be found in the table [system.events](#system_tables-events) - `ProfileEvents.Values` (Array(UInt64)) — Values of metrics for this thread that are listed in the `ProfileEvents.Names` column. -By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. +By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted. !!! note "Note" The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself. -You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) server setting (see the `partition_by` parameter). +You can specify an arbitrary partitioning key for the `system.query_thread_log` table in the [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) server setting (see the `partition_by` parameter). ## system.trace\_log {#system_tables-trace_log} diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index c664580c659..ccde7a945ac 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -574,11 +574,11 @@ ClickHouse проверит условия `min_part_size` и `min_part_size_rat ``` -## query\_thread\_log {#server_configuration_parameters-query-thread-log} +## query\_thread\_log {#server_configuration_parameters-query_thread_log} Настройка логирования потоков выполнения запросов, принятых с настройкой [log\_query\_threads=1](../settings/settings.md#settings-log-query-threads). -Запросы логируются не в отдельный файл, а в системную таблицу [system.query\_thread\_log](../../operations/server-configuration-parameters/settings.md#system_tables-query-thread-log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). +Запросы логируются не в отдельный файл, а в системную таблицу [system.query\_thread\_log](../../operations/server-configuration-parameters/settings.md#system_tables-query_thread_log). Вы можете изменить название этой таблицы в параметре `table` (см. ниже). При настройке логирования используются следующие параметры: diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 56c3042bfa3..4dd43e9607b 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -536,7 +536,7 @@ log_queries=1 Установка логирования информации о потоках выполнения запроса. -Лог информации о потоках выполнения запросов, переданных в ClickHouse с этой установкой, записывается согласно правилам конфигурационного параметра сервера [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log). +Лог информации о потоках выполнения запросов, переданных в ClickHouse с этой установкой, записывается согласно правилам конфигурационного параметра сервера [query\_thread\_log](../server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Пример: diff --git a/docs/ru/operations/system-tables.md b/docs/ru/operations/system-tables.md index ae0e67a4515..8954a484560 100644 --- a/docs/ru/operations/system-tables.md +++ b/docs/ru/operations/system-tables.md @@ -1,4 +1,7 @@ -# Системные таблицы {#sistemnye-tablitsy} +# Системные таблицы {#system-tables} + + +## Введение {#system-tables-introduction} Системные таблицы используются для реализации части функциональности системы, а также предоставляют доступ к информации о работе системы. Вы не можете удалить системную таблицу (хотя можете сделать DETACH). @@ -544,182 +547,156 @@ CurrentMetric_ReplicatedChecks: 0 - `source_file` (LowCardinality(String)) — Исходный файл, из которого была сделана запись. - `source_line` (UInt64) — Исходная строка, из которой была сделана запись. -## system.query\_log {#system_tables-query_log} +## system.query_log {#system_tables-query_log} -Содержит информацию о выполнении запросов. Для каждого запроса вы можете увидеть время начала обработки, продолжительность обработки, сообщения об ошибках и другую информацию. +Содержит информацию о выполняемых запросах, например, время начала обработки, продолжительность обработки, сообщения об ошибках. !!! note "Внимание" Таблица не содержит входных данных для запросов `INSERT`. -ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. +Настойки логгирования можно изменить в секции серверной конфигурации [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). -Чтобы включить логирование, задайте значение параметра [log\_queries](settings/settings.md#settings-log-queries) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md#settings). +Можно отключить логгирование настройкой [log_queries = 0](settings/settings.md#settings-log-queries). По-возможности, не отключайте логгирование, поскольку информация из таблицы важна при решении проблем. + +Период сброса логов в таблицу задаётся параметром `flush_interval_milliseconds` в конфигурационной секции [query_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос [SYSTEM FLUSH LOGS](../sql-reference/statements/system.md#query_language-system-flush_logs). + +ClickHouse не удаляет логи из таблица автоматически. Смотрите [Введение](#system-tables-introduction). + +Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`). + + + +Если таблицу удалить вручную, она создается заново автоматически «на лету». При этом все логи на момент удаления таблицы будут убраны. Таблица `system.query_log` содержит информацию о двух видах запросов: 1. Первоначальные запросы, которые были выполнены непосредственно клиентом. 2. Дочерние запросы, инициированные другими запросами (для выполнения распределенных запросов). Для дочерних запросов информация о первоначальном запросе содержится в столбцах `initial_*`. +В зависимости от статуса (столбец `type`) каждый запрос создаёт одну или две строки в таблице `query_log`: + +1. Если запрос выполнен успешно, создаются два события типа `QueryStart` и `QueryFinish`. +2. Если во время обработки запроса возникла ошибка, создаются два события с типами `QueryStart` и `ExceptionWhileProcessing`. +3. Если ошибка произошла ещё до запуска запроса, создается одно событие с типом `ExceptionBeforeStart`. + Столбцы: -- `type` (`Enum8`) — тип события, произошедшего при выполнении запроса. Значения: +- `type` ([Enum8](../sql-reference/data-types/enum.md)) — тип события, произошедшего при выполнении запроса. Значения: - `'QueryStart' = 1` — успешное начало выполнения запроса. - `'QueryFinish' = 2` — успешное завершение выполнения запроса. - `'ExceptionBeforeStart' = 3` — исключение перед началом обработки запроса. - `'ExceptionWhileProcessing' = 4` — исключение во время обработки запроса. -- `event_date` (Date) — дата начала запроса. -- `event_time` (DateTime) — время начала запроса. -- `query_start_time` (DateTime) — время начала обработки запроса. -- `query_duration_ms` (UInt64) — длительность обработки запроса. -- `read_rows` (UInt64) — количество прочитанных строк. -- `read_bytes` (UInt64) — количество прочитанных байтов. -- `written_rows` (UInt64) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0. -- `written_bytes` (UInt64) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. -- `result_rows` (UInt64) — количество строк в результате. -- `result_bytes` (UInt64) — объём результата в байтах. -- `memory_usage` (UInt64) — потребление RAM запросом. -- `query` (String) — текст запроса. -- `exception` (String) — сообщение исключения, если запрос завершился по исключению. -- `stack_trace` (String) — трассировка (список функций, последовательно вызванных перед ошибкой). Пустая строка, если запрос успешно завершен. -- `is_initial_query` (UInt8) — вид запроса. Возможные значения: +- `event_date` ([Date](../sql-reference/data-types/date.md)) — дата начала запроса. +- `event_time` ([DateTime](../sql-reference/data-types/datetime.md)) — время начала запроса. +- `query_start_time` ([DateTime](../sql-reference/data-types/datetime.md)) — время начала обработки запроса. +- `query_duration_ms` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — длительность выполнения запроса в миллисекундах. +- `read_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество строк, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_rows` включает в себя общее количество строк, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_rows`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. +- `read_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — Общее количество байтов, считанных из всех таблиц и табличных функций, участвующих в запросе. Включает в себя обычные подзапросы, подзапросы для `IN` и `JOIN`. Для распределенных запросов `read_bytes` включает в себя общее количество байтов, прочитанных на всех репликах. Каждая реплика передает собственное значение `read_bytes`, а сервер-инициатор запроса суммирует все полученные и локальные значения. Объемы кэша не учитываюся. +- `written_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество записанных строк для запросов `INSERT`. Для других запросов, значение столбца 0. +- `written_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — объём записанных данных в байтах для запросов `INSERT`. Для других запросов, значение столбца 0. +- `result_rows` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — количество строк в результате запроса `SELECT` или количество строк в запросе `INSERT`. +- `result_bytes` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — объём RAM в байтах, использованный для хранения результата запроса. +- `memory_usage` ([UInt64](../sql-reference/data-types/int-uint.md#uint-ranges)) — потребление RAM запросом. +- `query` ([String](../sql-reference/data-types/string.md)) — текст запроса. +- `exception` ([String](../sql-reference/data-types/string.md)) — сообщение исключения, если запрос завершился по исключению. +- `exception_code` ([Int32](../sql-reference/data-types/int-uint.md)) — код исключения. +- `stack_trace` ([String](../sql-reference/data-types/string.md)) — [stack trace](https://en.wikipedia.org/wiki/Stack_trace). Пустая строка, если запрос успешно завершен. +- `is_initial_query` ([UInt8](../sql-reference/data-types/int-uint.md)) — вид запроса. Возможные значения: - 1 — запрос был инициирован клиентом. - - 0 — запрос был инициирован другим запросом при распределенном запросе. -- `user` (String) — пользователь, запустивший текущий запрос. -- `query_id` (String) — ID запроса. -- `address` (IPv6) — IP адрес, с которого пришел запрос. -- `port` (UInt16) — порт, с которого клиент сделал запрос -- `initial_user` (String) — пользователь, запустивший первоначальный запрос (для распределенных запросов). -- `initial_query_id` (String) — ID родительского запроса. -- `initial_address` (IPv6) — IP адрес, с которого пришел родительский запрос. -- `initial_port` (UInt16) — порт, с которого клиент сделал родительский запрос. -- `interface` (UInt8) — интерфейс, с которого ушёл запрос. Возможные значения: + - 0 — запрос был инициирован другим запросом при выполнении распределенного запроса. +- `user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший текущий запрос. +- `query_id` ([String](../sql-reference/data-types/string.md)) — ID запроса. +- `address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел запрос. +- `port` ([UInt16](../sql-reference/data-types/int-uint.md)) — порт, с которого клиент сделал запрос +- `initial_user` ([String](../sql-reference/data-types/string.md)) — пользователь, запустивший первоначальный запрос (для распределенных запросов). +- `initial_query_id` ([String](../sql-reference/data-types/string.md)) — ID родительского запроса. +- `initial_address` ([IPv6](../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого пришел родительский запрос. +- `initial_port` ([UInt16](../sql-reference/data-types/int-uint.md)) — порт, с которого клиент сделал родительский запрос. +- `interface` ([UInt8](../sql-reference/data-types/int-uint.md)) — интерфейс, с которого ушёл запрос. Возможные значения: - 1 — TCP. - 2 — HTTP. -- `os_user` (String) — имя пользователя в OS, который запустил [clickhouse-client](../interfaces/cli.md). -- `client_hostname` (String) — имя сервера, с которого присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. -- `client_name` (String) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. -- `client_revision` (UInt32) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `client_version_major` (UInt32) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `client_version_minor` (UInt32) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `client_version_patch` (UInt32) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. -- `http_method` (UInt8) — HTTP метод, инициировавший запрос. Возможные значения: +- `os_user` ([String](../sql-reference/data-types/string.md)) — имя пользователя операционной системы, который запустил [clickhouse-client](../interfaces/cli.md). +- `client_hostname` ([String](../sql-reference/data-types/string.md)) — имя сервера, с которого присоединился [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_name` ([String](../sql-reference/data-types/string.md)) — [clickhouse-client](../interfaces/cli.md) или другой TCP клиент. +- `client_revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_major` ([UInt32](../sql-reference/data-types/int-uint.md)) — старшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_minor` ([UInt32](../sql-reference/data-types/int-uint.md)) — младшая версия [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `client_version_patch` ([UInt32](../sql-reference/data-types/int-uint.md)) — патч [clickhouse-client](../interfaces/cli.md) или другого TCP клиента. +- `http_method` ([UInt8](../sql-reference/data-types/int-uint.md)) — HTTP метод, инициировавший запрос. Возможные значения: - 0 — запрос запущен с интерфейса TCP. - 1 — `GET`. - 2 — `POST`. -- `http_user_agent` (String) — HTTP заголовок `UserAgent`. -- `quota_key` (String) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). -- `revision` (UInt32) — ревизия ClickHouse. -- `thread_numbers` (Array(UInt32)) — количество потоков, участвующих в обработке запросов. -- `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events -- `ProfileEvents.Values` (Array(UInt64)) — метрики, перечисленные в столбце `ProfileEvents.Names`. -- `Settings.Names` (Array(String)) — имена настроек, которые меняются, когда клиент выполняет запрос. Чтобы разрешить логирование изменений настроек, установите параметр `log_query_settings` равным 1. -- `Settings.Values` (Array(String)) — Значения настроек, которые перечислены в столбце `Settings.Names`. +- `http_user_agent` ([String](../sql-reference/data-types/string.md)) — HTTP заголовок `UserAgent`. +- `quota_key` ([String](../sql-reference/data-types/string.md)) — «ключ квоты» из настроек [квот](quotas.md) (см. `keyed`). +- `revision` ([UInt32](../sql-reference/data-types/int-uint.md)) — ревизия ClickHouse. +- `thread_numbers` ([Array(UInt32)](../sql-reference/data-types/array.md)) — количество потоков, участвующих в обработке запросов. +- `ProfileEvents.Names` ([Array(String)](../sql-reference/data-types/array.md)) — Счетчики для изменения различных метрик. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events +- `ProfileEvents.Values` ([Array(UInt64)](../sql-reference/data-types/array.md)) — метрики, перечисленные в столбце `ProfileEvents.Names`. +- `Settings.Names` ([Array(String)](../sql-reference/data-types/array.md)) — имена настроек, которые меняются, когда клиент выполняет запрос. Чтобы разрешить логирование изменений настроек, установите параметр `log_query_settings` равным 1. +- `Settings.Values` ([Array(String)](../sql-reference/data-types/array.md)) — Значения настроек, которые перечислены в столбце `Settings.Names`. -Каждый запрос создаёт одну или две строки в таблице `query_log`, в зависимости от статуса запроса: +**Пример** -1. Если запрос выполнен успешно, создаются два события типа 1 и 2 (смотрите столбец `type`). -2. Если во время обработки запроса произошла ошибка, создаются два события с типами 1 и 4. -3. Если ошибка произошла до запуска запроса, создается одно событие с типом 3. +``` sql +SELECT * FROM system.query_log LIMIT 1 FORMAT Vertical; +``` -По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. +``` text +Row 1: +────── +type: QueryStart +event_date: 2020-05-13 +event_time: 2020-05-13 14:02:28 +query_start_time: 2020-05-13 14:02:28 +query_duration_ms: 0 +read_rows: 0 +read_bytes: 0 +written_rows: 0 +written_bytes: 0 +result_rows: 0 +result_bytes: 0 +memory_usage: 0 +query: SELECT 1 +exception_code: 0 +exception: +stack_trace: +is_initial_query: 1 +user: default +query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +address: ::ffff:127.0.0.1 +port: 57720 +initial_user: default +initial_query_id: 5e834082-6f6d-4e34-b47b-cd1934f4002a +initial_address: ::ffff:127.0.0.1 +initial_port: 57720 +interface: 1 +os_user: bayonet +client_hostname: clickhouse.ru-central1.internal +client_name: ClickHouse client +client_revision: 54434 +client_version_major: 20 +client_version_minor: 4 +client_version_patch: 1 +http_method: 0 +http_user_agent: +quota_key: +revision: 54434 +thread_ids: [] +ProfileEvents.Names: [] +ProfileEvents.Values: [] +Settings.Names: ['use_uncompressed_cache','load_balancing','log_queries','max_memory_usage'] +Settings.Values: ['0','random','1','10000000000'] -Если таблицу удалить вручную, она пересоздастся автоматически «на лету». При этом все логи на момент удаления таблицы будут удалены. +``` +**Смотрите также** -!!! note "Примечание" - Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов. +- [system.query_thread_log](#system_tables-query_thread_log) — в этой таблице содержится информация о цепочке каждого выполненного запроса. -Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) (параметр `partition_by`). - -## system.query\_log {#system_tables-query_log} - -Contains information about execution of queries. For each query, you can see processing start time, duration of processing, error messages and other information. - -!!! note "Note" - The table doesn’t contain input data for `INSERT` queries. - -ClickHouse creates this table only if the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server parameter is specified. This parameter sets the logging rules, such as the logging interval or the name of the table the queries will be logged in. - -To enable query logging, set the [log\_queries](settings/settings.md#settings-log-queries) parameter to 1. For details, see the [Settings](settings/settings.md) section. - -The `system.query_log` table registers two kinds of queries: - -1. Initial queries that were run directly by the client. -2. Child queries that were initiated by other queries (for distributed query execution). For these types of queries, information about the parent queries is shown in the `initial_*` columns. - -Columns: - -- `type` (`Enum8`) — Type of event that occurred when executing the query. Values: - - `'QueryStart' = 1` — Successful start of query execution. - - `'QueryFinish' = 2` — Successful end of query execution. - - `'ExceptionBeforeStart' = 3` — Exception before the start of query execution. - - `'ExceptionWhileProcessing' = 4` — Exception during the query execution. -- `event_date` (Date) — Query starting date. -- `event_time` (DateTime) — Query starting time. -- `query_start_time` (DateTime) — Start time of query execution. -- `query_duration_ms` (UInt64) — Duration of query execution. -- `read_rows` (UInt64) — Number of read rows. -- `read_bytes` (UInt64) — Number of read bytes. -- `written_rows` (UInt64) — For `INSERT` queries, the number of written rows. For other queries, the column value is 0. -- `written_bytes` (UInt64) — For `INSERT` queries, the number of written bytes. For other queries, the column value is 0. -- `result_rows` (UInt64) — Number of rows in the result. -- `result_bytes` (UInt64) — Number of bytes in the result. -- `memory_usage` (UInt64) — Memory consumption by the query. -- `query` (String) — Query string. -- `exception` (String) — Exception message. -- `stack_trace` (String) — Stack trace (a list of methods called before the error occurred). An empty string, if the query is completed successfully. -- `is_initial_query` (UInt8) — Query type. Possible values: - - 1 — Query was initiated by the client. - - 0 — Query was initiated by another query for distributed query execution. -- `user` (String) — Name of the user who initiated the current query. -- `query_id` (String) — ID of the query. -- `address` (IPv6) — IP address that was used to make the query. -- `port` (UInt16) — The client port that was used to make the query. -- `initial_user` (String) — Name of the user who ran the initial query (for distributed query execution). -- `initial_query_id` (String) — ID of the initial query (for distributed query execution). -- `initial_address` (IPv6) — IP address that the parent query was launched from. -- `initial_port` (UInt16) — The client port that was used to make the parent query. -- `interface` (UInt8) — Interface that the query was initiated from. Possible values: - - 1 — TCP. - - 2 — HTTP. -- `os_user` (String) — OS’s username who runs [clickhouse-client](../interfaces/cli.md). -- `client_hostname` (String) — Hostname of the client machine where the [clickhouse-client](../interfaces/cli.md) or another TCP client is run. -- `client_name` (String) — The [clickhouse-client](../interfaces/cli.md) or another TCP client name. -- `client_revision` (UInt32) — Revision of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_major` (UInt32) — Major version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_minor` (UInt32) — Minor version of the [clickhouse-client](../interfaces/cli.md) or another TCP client. -- `client_version_patch` (UInt32) — Patch component of the [clickhouse-client](../interfaces/cli.md) or another TCP client version. -- `http_method` (UInt8) — HTTP method that initiated the query. Possible values: - - 0 — The query was launched from the TCP interface. - - 1 — `GET` method was used. - - 2 — `POST` method was used. -- `http_user_agent` (String) — The `UserAgent` header passed in the HTTP request. -- `quota_key` (String) — The «quota key» specified in the [quotas](quotas.md) setting (see `keyed`). -- `revision` (UInt32) — ClickHouse revision. -- `thread_numbers` (Array(UInt32)) — Number of threads that are participating in query execution. -- `ProfileEvents.Names` (Array(String)) — Counters that measure different metrics. The description of them could be found in the table [system.events](#system_tables-events) -- `ProfileEvents.Values` (Array(UInt64)) — Values of metrics that are listed in the `ProfileEvents.Names` column. -- `Settings.Names` (Array(String)) — Names of settings that were changed when the client ran the query. To enable logging changes to settings, set the `log_query_settings` parameter to 1. -- `Settings.Values` (Array(String)) — Values of settings that are listed in the `Settings.Names` column. - -Each query creates one or two rows in the `query_log` table, depending on the status of the query: - -1. If the query execution is successful, two events with types 1 and 2 are created (see the `type` column). -2. If an error occurred during query processing, two events with types 1 and 4 are created. -3. If an error occurred before launching the query, a single event with type 3 is created. - -By default, logs are added to the table at intervals of 7.5 seconds. You can set this interval in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `flush_interval_milliseconds` parameter). To flush the logs forcibly from the memory buffer into the table, use the `SYSTEM FLUSH LOGS` query. - -When the table is deleted manually, it will be automatically created on the fly. Note that all the previous logs will be deleted. - -!!! note "Note" - The storage period for logs is unlimited. Logs aren’t automatically deleted from the table. You need to organize the removal of outdated logs yourself. - -You can specify an arbitrary partitioning key for the `system.query_log` table in the [query\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-log) server setting (see the `partition_by` parameter). -\#\# system.query\_thread\_log {\#system\_tables-query-thread-log} +## system.query_thread_log {#system_tables-query_thread_log} Содержит информацию о каждом потоке выполняемых запросов. -ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. +ClickHouse создаёт таблицу только в том случае, когда установлен конфигурационный параметр сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log). Параметр задаёт правила ведения лога, такие как интервал логирования или имя таблицы, в которую будут логгироваться запросы. Чтобы включить логирование, задайте значение параметра [log\_query\_threads](settings/settings.md#settings-log-query-threads) равным 1. Подробности смотрите в разделе [Настройки](settings/settings.md#settings). @@ -770,16 +747,16 @@ ClickHouse создаёт таблицу только в том случае, к - `ProfileEvents.Names` (Array(String)) — Счетчики для изменения различных метрик для данного потока. Описание метрик можно получить из таблицы [system.events](#system_tables-events)(\#system\_tables-events - `ProfileEvents.Values` (Array(UInt64)) — метрики для данного потока, перечисленные в столбце `ProfileEvents.Names`. -По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. +По умолчанию, строки добавляются в таблицу логирования с интервалом в 7,5 секунд. Можно задать интервал в конфигурационном параметре сервера [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (смотрите параметр `flush_interval_milliseconds`). Чтобы принудительно записать логи из буффера памяти в таблицу, используйте запрос `SYSTEM FLUSH LOGS`. Если таблицу удалить вручную, она пересоздастся автоматически «на лету». При этом все логи на момент удаления таблицы будут удалены. !!! note "Примечание" Срок хранения логов не ограничен. Логи не удаляются из таблицы автоматически. Вам необходимо самостоятельно организовать удаление устаревших логов. -Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query-thread-log) (параметр `partition_by`). +Можно указать произвольный ключ партиционирования для таблицы `system.query_log` в конфигурации [query\_thread\_log](server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) (параметр `partition_by`). -## system.query_thread_log {#system_tables-query-thread-log} +## system.query_thread_log {#system_tables-query_thread_log} Содержит информацию о каждом потоке исполнения запроса. From 3f53b5b335c9993c092809253f736af406d8ade1 Mon Sep 17 00:00:00 2001 From: Alexander Kazakov Date: Fri, 29 May 2020 12:45:55 +0300 Subject: [PATCH 38/38] Fixed improper logging of failures in destructors of certain metrics (#11264) --- src/Common/MemoryStatisticsOS.cpp | 15 ++++++++++++++- src/Common/ProcfsMetricsProvider.cpp | 22 +++++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/Common/MemoryStatisticsOS.cpp b/src/Common/MemoryStatisticsOS.cpp index adc5bf5d904..6082d23cbd0 100644 --- a/src/Common/MemoryStatisticsOS.cpp +++ b/src/Common/MemoryStatisticsOS.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB @@ -19,6 +20,7 @@ namespace ErrorCodes extern const int FILE_DOESNT_EXIST; extern const int CANNOT_OPEN_FILE; extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; + extern const int CANNOT_CLOSE_FILE; } static constexpr auto filename = "/proc/self/statm"; @@ -35,7 +37,18 @@ MemoryStatisticsOS::MemoryStatisticsOS() MemoryStatisticsOS::~MemoryStatisticsOS() { if (0 != ::close(fd)) - tryLogCurrentException(__PRETTY_FUNCTION__); + { + try + { + throwFromErrno( + "File descriptor for \"" + std::string(filename) + "\" could not be closed. " + "Something seems to have gone wrong. Inspect errno.", ErrorCodes::CANNOT_CLOSE_FILE); + } + catch (const ErrnoException &) + { + DB::tryLogCurrentException(__PRETTY_FUNCTION__); + } + } } MemoryStatisticsOS::Data MemoryStatisticsOS::get() const diff --git a/src/Common/ProcfsMetricsProvider.cpp b/src/Common/ProcfsMetricsProvider.cpp index 633558a7b67..fcc4124aa85 100644 --- a/src/Common/ProcfsMetricsProvider.cpp +++ b/src/Common/ProcfsMetricsProvider.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -22,6 +23,7 @@ namespace ErrorCodes { extern const int FILE_DOESNT_EXIST; extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_CLOSE_FILE; extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR; } @@ -39,6 +41,20 @@ namespace errno == ENOENT ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE); } +inline void emitErrorMsgWithFailedToCloseFile(const std::string & filename) +{ + try + { + throwFromErrno( + "File descriptor for \"" + filename + "\" could not be closed. " + "Something seems to have gone wrong. Inspect errno.", ErrorCodes::CANNOT_CLOSE_FILE); + } + catch (const ErrnoException &) + { + DB::tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + ssize_t readFromFD(const int fd, const char * filename, char * buf, size_t buf_size) { ssize_t res = 0; @@ -100,11 +116,11 @@ ProcfsMetricsProvider::ProcfsMetricsProvider(const pid_t /*tid*/) ProcfsMetricsProvider::~ProcfsMetricsProvider() { if (stats_version >= 3 && 0 != ::close(thread_io_fd)) - tryLogCurrentException(__PRETTY_FUNCTION__); + emitErrorMsgWithFailedToCloseFile(thread_io); if (0 != ::close(thread_stat_fd)) - tryLogCurrentException(__PRETTY_FUNCTION__); + emitErrorMsgWithFailedToCloseFile(thread_stat); if (0 != ::close(thread_schedstat_fd)) - tryLogCurrentException(__PRETTY_FUNCTION__); + emitErrorMsgWithFailedToCloseFile(thread_schedstat); }