thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 09:32:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add views and it's tables to system.query_log

Browse Source
This commit is contained in:
Raúl Marín 2021-06-24 18:54:04 +02:00
parent 14e8029e01
commit f0125262b7
10 changed files with 53 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/operations/system-tables/query_log.md
View File

@ -50,6 +50,7 @@ Columns:
- `query_kind` ([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md)) — Type of the query.
- `databases` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the databases present in the query.
- `tables` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the tables present in the query.
- `views` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the (materialized or live) views present in the query.
- `columns` ([Array](../../sql-reference/data-types/array.md)([LowCardinality(String)](../../sql-reference/data-types/lowcardinality.md))) — Names of the columns present in the query.
- `exception_code` ([Int32](../../sql-reference/data-types/int-uint.md)) — Code of an exception.
- `exception` ([String](../../sql-reference/data-types/string.md)) — Exception message.

11
src/DataStreams/PushingToViewsBlockOutputStream.cpp
View File

@ -86,7 +86,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
ASTPtr query;
BlockOutputStreamPtr out;
QueryViewsLogElement::ViewType type = QueryViewsLogElement::ViewType::DEFAULT;
String target_name = database_table.getNameForLogs();
String target_name = database_table.getFullTableName();
if (auto * materialized_view = dynamic_cast<StorageMaterializedView *>(dependent_table.get()))
{
@ -98,7 +98,7 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
auto inner_table_id = inner_table->getStorageID();
auto inner_metadata_snapshot = inner_table->getInMemoryMetadataPtr();
query = dependent_metadata_snapshot->getSelectQuery().inner_query;
target_name = inner_table_id.getNameForLogs();
target_name = inner_table_id.getFullTableName();
std::unique_ptr<ASTInsertQuery> insert = std::make_unique<ASTInsertQuery>();
insert->table_id = inner_table_id;
@ -147,6 +147,13 @@ PushingToViewsBlockOutputStream::PushingToViewsBlockOutputStream(
target_name, type, thread_status, 0, std::chrono::system_clock::now(), QueryViewsLogElement::ViewStatus::INIT};
views.emplace_back(ViewInfo{std::move(query), database_table, std::move(out), nullptr, std::move(runtime_stats)});
current_thread = running_thread;
/// Add the view to the query_log
if (!no_destination)
{
getContext()->getQueryContext()->addQueryAccessInfo(
backQuoteIfNeed(database_table.getDatabaseName()), target_name, {}, "", database_table.getFullTableName());
}
}
/// Do not push to destination table if the flag is set

8
src/Interpreters/Context.cpp
View File

@ -1064,7 +1064,11 @@ bool Context::hasScalar(const String & name) const
void Context::addQueryAccessInfo(
const String & quoted_database_name, const String & full_quoted_table_name, const Names & column_names, const String & projection_name)
const String & quoted_database_name,
const String & full_quoted_table_name,
const Names & column_names,
const String & projection_name,
const String & view_name)
{
assert(!isGlobalContext() || getApplicationType() == ApplicationType::LOCAL);
std::lock_guard<std::mutex> lock(query_access_info.mutex);
@ -1074,6 +1078,8 @@ void Context::addQueryAccessInfo(
query_access_info.columns.emplace(full_quoted_table_name + "." + backQuoteIfNeed(column_name));
if (!projection_name.empty())
query_access_info.projections.emplace(full_quoted_table_name + "." + backQuoteIfNeed(projection_name));
if (!view_name.empty())
query_access_info.views.emplace(view_name);
}

8
src/Interpreters/Context.h
View File

@ -212,6 +212,7 @@ private:
tables = rhs.tables;
columns = rhs.columns;
projections = rhs.projections;
views = rhs.views;
}
QueryAccessInfo(QueryAccessInfo && rhs) = delete;
@ -228,6 +229,7 @@ private:
std::swap(tables, rhs.tables);
std::swap(columns, rhs.columns);
std::swap(projections, rhs.projections);
std::swap(views, rhs.views);
}
/// To prevent a race between copy-constructor and other uses of this structure.
@ -235,7 +237,8 @@ private:
std::set<std::string> databases{};
std::set<std::string> tables{};
std::set<std::string> columns{};
std::set<std::string> projections;
std::set<std::string> projections{};
std::set<std::string> views{};
};
QueryAccessInfo query_access_info;
@ -459,7 +462,8 @@ public:
const String & quoted_database_name,
const String & full_quoted_table_name,
const Names & column_names,
const String & projection_name = {});
const String & projection_name = {},
const String & view_name = {});
/// Supported factories for records in query_log
enum class QueryLogFactories

4
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -1952,12 +1952,14 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
if (context->hasQueryContext() && !options.is_internal)
{
const String view_name{};
auto local_storage_id = storage->getStorageID();
context->getQueryContext()->addQueryAccessInfo(
backQuoteIfNeed(local_storage_id.getDatabaseName()),
local_storage_id.getFullTableName(),
required_columns,
query_info.projection ? query_info.projection->desc->name : "");
query_info.projection ? query_info.projection->desc->name : "",
view_name);
}
/// Create step which reads from empty source if storage has no data.

4
src/Interpreters/QueryLog.cpp
View File

@ -67,6 +67,8 @@ NamesAndTypesList QueryLogElement::getNamesAndTypes()
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
{"projections", std::make_shared<DataTypeArray>(
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
{"views", std::make_shared<DataTypeArray>(
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()))},
{"exception_code", std::make_shared<DataTypeInt32>()},
{"exception", std::make_shared<DataTypeString>()},
{"stack_trace", std::make_shared<DataTypeString>()},
@ -159,6 +161,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
auto & column_tables = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_columns = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_projections = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_views = typeid_cast<ColumnArray &>(*columns[i++]);
auto fill_column = [](const std::set<String> & data, ColumnArray & column)
{
@ -176,6 +179,7 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
fill_column(query_tables, column_tables);
fill_column(query_columns, column_columns);
fill_column(query_projections, column_projections);
fill_column(query_views, column_views);
}
columns[i++]->insert(exception_code);

1
src/Interpreters/QueryLog.h
View File

@ -58,6 +58,7 @@ struct QueryLogElement
std::set<String> query_tables;
std::set<String> query_columns;
std::set<String> query_projections;
std::set<String> query_views;
std::unordered_set<String> used_aggregate_functions;
std::unordered_set<String> used_aggregate_function_combinators;

1
src/Interpreters/executeQuery.cpp
View File

@ -652,6 +652,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
elem.query_tables = info.tables;
elem.query_columns = info.columns;
elem.query_projections = info.projections;
elem.query_views = info.views;
}
interpreter->extendQueryLogElem(elem, ast, context, query_database, query_table);

8
tests/queries/0_stateless/01925_query_views_log_current_database.reference
View File

@ -1,4 +1,4 @@
{"stage":"Query log rows","read_rows":"100","written_rows":"201"}
{"stage":"Depending views","view_name":"default.matview_a_to_b","view_query":"SELECT toFloat64(a) AS a, b AS count FROM default.table_a"}
{"stage":"Depending views","view_name":"default.matview_b_to_c","view_query":"SELECT sum(a) AS a FROM default.table_b"}
{"stage":"Depending views","view_name":"default.table_b_live_view","view_query":"SELECT sum(a + b) FROM default.table_b"}
{"stage":"Query log rows","read_rows":"100","written_rows":"201","databases":["_table_function","default"],"tables":["_table_function.numbers","default.table_a","default.table_b","default.table_b_live_view","default.table_c"],"views":["default.matview_a_to_b","default.matview_b_to_c","default.table_b_live_view"]}
{"stage":"Depending views","view_name":"default.matview_a_to_b","view_target":"default.table_b","view_query":"SELECT toFloat64(a) AS a, b AS count FROM default.table_a"}
{"stage":"Depending views","view_name":"default.matview_b_to_c","view_target":"default.table_c","view_query":"SELECT sum(a) AS a FROM default.table_b"}
{"stage":"Depending views","view_name":"default.table_b_live_view","view_target":"default.table_b_live_view","view_query":"SELECT sum(a + b) FROM default.table_b"}

22
tests/queries/0_stateless/01925_query_views_log_current_database.sql
View File

@ -10,6 +10,8 @@ CREATE TABLE table_c (a Float64) ENGINE = MergeTree ORDER BY a;
-- SETUP MATERIALIZED VIEWS
CREATE MATERIALIZED VIEW matview_a_to_b TO table_b AS SELECT toFloat64(a) AS a, b AS count FROM table_a;
CREATE MATERIALIZED VIEW matview_b_to_c TO table_c AS SELECT SUM(a) as a FROM table_b;
-- We don't include test materialized views here since the name is based on the uuid on atomic databases
-- and that makes the test harder to read
-- SETUP LIVE VIEW
---- table_b_live_view (Int64)
@ -25,15 +27,25 @@ SET log_queries=1;
INSERT INTO table_a SELECT '111', * FROM numbers(100);
SYSTEM FLUSH LOGS;
SELECT 'Query log rows' as stage, read_rows, written_rows
SELECT
'Query log rows' as stage,
read_rows,
written_rows,
arraySort(databases) as databases,
arraySort(tables) as tables,
arraySort(views) as views
FROM system.query_log
WHERE
query like '-- INSERT 1%INSERT INTO table_a%'
AND current_database = currentDatabase()
AND event_date >= yesterday()
query like '-- INSERT 1%INSERT INTO table_a%'
AND current_database = currentDatabase()
AND event_date >= yesterday()
FORMAT JSONEachRow;
SELECT 'Depending views' as stage, view_name, view_query
SELECT
'Depending views' as stage,
view_name,
view_target,
view_query
FROM system.query_views_log
WHERE initial_query_id =
(