From f0125262b743b0e8b8ad82b08abd1a74bfe8eb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 24 Jun 2021 18:54:04 +0200 Subject: [PATCH] Add views and it's tables to system.query_log --- docs/en/operations/system-tables/query_log.md | 1 + .../PushingToViewsBlockOutputStream.cpp | 11 ++++++++-- src/Interpreters/Context.cpp | 8 ++++++- src/Interpreters/Context.h | 8 +++++-- src/Interpreters/InterpreterSelectQuery.cpp | 4 +++- src/Interpreters/QueryLog.cpp | 4 ++++ src/Interpreters/QueryLog.h | 1 + src/Interpreters/executeQuery.cpp | 1 + ...query_views_log_current_database.reference | 8 +++---- ...01925_query_views_log_current_database.sql | 22 ++++++++++++++----- 10 files changed, 53 insertions(+), 15 deletions(-) diff --git a/docs/en/operations/system-tables/query_log.md b/docs/en/operations/system-tables/query_log.md index 7c76d2f0a4f..fe102aeeb3e 100644 --- a/docs/en/operations/system-tables/query_log.md +++ b/docs/en/operations/system-tables/query_log.md @@ -50,6 +50,7 @@ Columns: - `query_kind` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Type of the query. - `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the databases present in the query. - `tables` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the tables present in the query. +- `views` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the (materialized or live) views present in the query. - `columns` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the columns present in the query. - `exception_code` ([Int32](../../sql-reference/data-types/int-uint.md)) — Code of an exception. - `exception` ([String](../../sql-reference/data-types/string.md)) — Exception message. diff --git a/src/DataStreams/PushingToViewsBlockOutputStream.cpp b/src/DataStreams/PushingToViewsBlockOutputStream.cpp index 4e583011ad3..c88e5a73fd0 100644 --- a/src/DataStreams/PushingToViewsBlockOutputStream.cpp +++ b/src/DataStreams/PushingToViewsBlockOutputStream.cpp @@ -86,7 +86,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( ASTPtr query; BlockOutputStreamPtr out; QueryViewsLogElement::ViewType type = QueryViewsLogElement::ViewType::DEFAULT; - String target_name = database_table.getNameForLogs(); + String target_name = database_table.getFullTableName(); if (auto * materialized_view = dynamic_cast(dependent_table.get())) { @@ -98,7 +98,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( auto inner_table_id = inner_table->getStorageID(); auto inner_metadata_snapshot = inner_table->getInMemoryMetadataPtr(); query = dependent_metadata_snapshot->getSelectQuery().inner_query; - target_name = inner_table_id.getNameForLogs(); + target_name = inner_table_id.getFullTableName(); std::unique_ptr insert = std::make_unique(); insert->table_id = inner_table_id; @@ -147,6 +147,13 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream( target_name, type, thread_status, 0, std::chrono::system_clock::now(), QueryViewsLogElement::ViewStatus::INIT}; views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out), nullptr, std::move(runtime_stats)}); current_thread = running_thread; + + /// Add the view to the query_log + if (!no_destination) + { + getContext()->getQueryContext()->addQueryAccessInfo( + backQuoteIfNeed(database_table.getDatabaseName()), target_name, {}, "", database_table.getFullTableName()); + } } /// Do not push to destination table if the flag is set diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 798efeed150..ddc63a9a267 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1064,7 +1064,11 @@ bool Context::hasScalar(const String & name) const void Context::addQueryAccessInfo( - const String & quoted_database_name, const String & full_quoted_table_name, const Names & column_names, const String & projection_name) + const String & quoted_database_name, + const String & full_quoted_table_name, + const Names & column_names, + const String & projection_name, + const String & view_name) { assert(!isGlobalContext() || getApplicationType() == ApplicationType::LOCAL); std::lock_guard lock(query_access_info.mutex); @@ -1074,6 +1078,8 @@ void Context::addQueryAccessInfo( query_access_info.columns.emplace(full_quoted_table_name + "." + backQuoteIfNeed(column_name)); if (!projection_name.empty()) query_access_info.projections.emplace(full_quoted_table_name + "." + backQuoteIfNeed(projection_name)); + if (!view_name.empty()) + query_access_info.views.emplace(view_name); } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index b8b4ef37399..759f00b4265 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -212,6 +212,7 @@ private: tables = rhs.tables; columns = rhs.columns; projections = rhs.projections; + views = rhs.views; } QueryAccessInfo(QueryAccessInfo && rhs) = delete; @@ -228,6 +229,7 @@ private: std::swap(tables, rhs.tables); std::swap(columns, rhs.columns); std::swap(projections, rhs.projections); + std::swap(views, rhs.views); } /// To prevent a race between copy-constructor and other uses of this structure. @@ -235,7 +237,8 @@ private: std::set databases{}; std::set tables{}; std::set columns{}; - std::set projections; + std::set projections{}; + std::set views{}; }; QueryAccessInfo query_access_info; @@ -459,7 +462,8 @@ public: const String & quoted_database_name, const String & full_quoted_table_name, const Names & column_names, - const String & projection_name = {}); + const String & projection_name = {}, + const String & view_name = {}); /// Supported factories for records in query_log enum class QueryLogFactories diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 79e60a9a02c..1fd63e31884 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -1952,12 +1952,14 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (context->hasQueryContext() && !options.is_internal) { + const String view_name{}; auto local_storage_id = storage->getStorageID(); context->getQueryContext()->addQueryAccessInfo( backQuoteIfNeed(local_storage_id.getDatabaseName()), local_storage_id.getFullTableName(), required_columns, - query_info.projection ? query_info.projection->desc->name : ""); + query_info.projection ? query_info.projection->desc->name : "", + view_name); } /// Create step which reads from empty source if storage has no data. diff --git a/src/Interpreters/QueryLog.cpp b/src/Interpreters/QueryLog.cpp index 3f668e5e0ab..c3bd1aec49d 100644 --- a/src/Interpreters/QueryLog.cpp +++ b/src/Interpreters/QueryLog.cpp @@ -67,6 +67,8 @@ NamesAndTypesList QueryLogElement::getNamesAndTypes() std::make_shared(std::make_shared()))}, {"projections", std::make_shared( std::make_shared(std::make_shared()))}, + {"views", std::make_shared( + std::make_shared(std::make_shared()))}, {"exception_code", std::make_shared()}, {"exception", std::make_shared()}, {"stack_trace", std::make_shared()}, @@ -159,6 +161,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const auto & column_tables = typeid_cast(*columns[i++]); auto & column_columns = typeid_cast(*columns[i++]); auto & column_projections = typeid_cast(*columns[i++]); + auto & column_views = typeid_cast(*columns[i++]); auto fill_column = [](const std::set & data, ColumnArray & column) { @@ -176,6 +179,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const fill_column(query_tables, column_tables); fill_column(query_columns, column_columns); fill_column(query_projections, column_projections); + fill_column(query_views, column_views); } columns[i++]->insert(exception_code); diff --git a/src/Interpreters/QueryLog.h b/src/Interpreters/QueryLog.h index 0aa02104306..c7421063fb5 100644 --- a/src/Interpreters/QueryLog.h +++ b/src/Interpreters/QueryLog.h @@ -58,6 +58,7 @@ struct QueryLogElement std::set query_tables; std::set query_columns; std::set query_projections; + std::set query_views; std::unordered_set used_aggregate_functions; std::unordered_set used_aggregate_function_combinators; diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 75b89554820..206a22f9cd3 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -652,6 +652,7 @@ static std::tuple executeQueryImpl( elem.query_tables = info.tables; elem.query_columns = info.columns; elem.query_projections = info.projections; + elem.query_views = info.views; } interpreter->extendQueryLogElem(elem, ast, context, query_database, query_table); diff --git a/tests/queries/0_stateless/01925_query_views_log_current_database.reference b/tests/queries/0_stateless/01925_query_views_log_current_database.reference index 3512f030329..1dc181eb789 100644 --- a/tests/queries/0_stateless/01925_query_views_log_current_database.reference +++ b/tests/queries/0_stateless/01925_query_views_log_current_database.reference @@ -1,4 +1,4 @@ -{"stage":"Query log rows","read_rows":"100","written_rows":"201"} -{"stage":"Depending views","view_name":"default.matview_a_to_b","view_query":"SELECT toFloat64(a) AS a, b AS count FROM default.table_a"} -{"stage":"Depending views","view_name":"default.matview_b_to_c","view_query":"SELECT sum(a) AS a FROM default.table_b"} -{"stage":"Depending views","view_name":"default.table_b_live_view","view_query":"SELECT sum(a + b) FROM default.table_b"} +{"stage":"Query log rows","read_rows":"100","written_rows":"201","databases":["_table_function","default"],"tables":["_table_function.numbers","default.table_a","default.table_b","default.table_b_live_view","default.table_c"],"views":["default.matview_a_to_b","default.matview_b_to_c","default.table_b_live_view"]} +{"stage":"Depending views","view_name":"default.matview_a_to_b","view_target":"default.table_b","view_query":"SELECT toFloat64(a) AS a, b AS count FROM default.table_a"} +{"stage":"Depending views","view_name":"default.matview_b_to_c","view_target":"default.table_c","view_query":"SELECT sum(a) AS a FROM default.table_b"} +{"stage":"Depending views","view_name":"default.table_b_live_view","view_target":"default.table_b_live_view","view_query":"SELECT sum(a + b) FROM default.table_b"} diff --git a/tests/queries/0_stateless/01925_query_views_log_current_database.sql b/tests/queries/0_stateless/01925_query_views_log_current_database.sql index 5fa7570d036..d074d1f3852 100644 --- a/tests/queries/0_stateless/01925_query_views_log_current_database.sql +++ b/tests/queries/0_stateless/01925_query_views_log_current_database.sql @@ -10,6 +10,8 @@ CREATE TABLE table_c (a Float64) ENGINE = MergeTree ORDER BY a; -- SETUP MATERIALIZED VIEWS CREATE MATERIALIZED VIEW matview_a_to_b TO table_b AS SELECT toFloat64(a) AS a, b AS count FROM table_a; CREATE MATERIALIZED VIEW matview_b_to_c TO table_c AS SELECT SUM(a) as a FROM table_b; +-- We don't include test materialized views here since the name is based on the uuid on atomic databases +-- and that makes the test harder to read -- SETUP LIVE VIEW ---- table_b_live_view (Int64) @@ -25,15 +27,25 @@ SET log_queries=1; INSERT INTO table_a SELECT '111', * FROM numbers(100); SYSTEM FLUSH LOGS; -SELECT 'Query log rows' as stage, read_rows, written_rows +SELECT + 'Query log rows' as stage, + read_rows, + written_rows, + arraySort(databases) as databases, + arraySort(tables) as tables, + arraySort(views) as views FROM system.query_log WHERE - query like '-- INSERT 1%INSERT INTO table_a%' - AND current_database = currentDatabase() - AND event_date >= yesterday() + query like '-- INSERT 1%INSERT INTO table_a%' + AND current_database = currentDatabase() + AND event_date >= yesterday() FORMAT JSONEachRow; -SELECT 'Depending views' as stage, view_name, view_query +SELECT + 'Depending views' as stage, + view_name, + view_target, + view_query FROM system.query_views_log WHERE initial_query_id = (