thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Better

Browse Source
This commit is contained in:
kssenii 2021-01-21 21:15:11 +00:00
parent 1ec463b61f
commit daab2c91bb
15 changed files with 141 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/AggregateFunctions/AggregateFunctionFactory.cpp
View File

@ -64,7 +64,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get(
{
auto query_context = CurrentThread::get().getQueryContext();
if (query_context && query_context->getSettingsRef().log_queries)
query_context->addQueryFactoriesInfo("AggregateFunction", name);
query_context->addQueryFactoriesInfo(Context::QueryLogFactories::AggregateFunction, name);
}
auto type_without_low_cardinality = convertLowCardinalityTypesToNested(argument_types);

2
src/DataTypes/DataTypeFactory.cpp
View File

@ -82,7 +82,7 @@ DataTypePtr DataTypeFactory::get(const String & family_name_param, const ASTPtr
{
auto query_context = CurrentThread::get().getQueryContext();
if (query_context && query_context->getSettingsRef().log_queries)
query_context->addQueryFactoriesInfo("DataType", family_name);
query_context->addQueryFactoriesInfo(Context::QueryLogFactories::DataType, family_name);
}
return findCreatorByName(family_name)(parameters);

2
src/Databases/DatabaseFactory.cpp
View File

@ -98,7 +98,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String
ErrorCodes::UNKNOWN_ELEMENT_IN_AST);
if (context.hasQueryContext() && context.getSettingsRef().log_queries)
context.getQueryContext().addQueryFactoriesInfo("Database", engine_name);
context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::Database, engine_name);
if (engine_name == "Ordinary")
return std::make_shared<DatabaseOrdinary>(database_name, metadata_path, context);

2
src/Dictionaries/DictionaryFactory.cpp
View File

@ -48,7 +48,7 @@ DictionaryPtr DictionaryFactory::create(
LOG_TRACE(&Poco::Logger::get("DictionaryFactory"), "Created dictionary source '{}' for dictionary '{}'", source_ptr->toString(), name);
if (context.hasQueryContext() && context.getSettingsRef().log_queries)
context.getQueryContext().addQueryFactoriesInfo("Dictionary", name);
context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, name);
const auto & layout_type = keys.front();

6
src/Formats/FormatFactory.cpp
View File

@ -279,7 +279,7 @@ InputFormatPtr FormatFactory::getInputFormat(
const Settings & settings = context.getSettingsRef();
if (context.hasQueryContext() && settings.log_queries)
context.getQueryContext().addQueryFactoriesInfo("Format", name);
context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::Format, name);
auto format_settings = _format_settings
? *_format_settings : getFormatSettings(context);
@ -324,7 +324,7 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible(
ParallelFormattingOutputFormat::Params builder{buf, sample, formatter_creator, settings.max_threads};
if (context.hasQueryContext() && settings.log_queries)
context.getQueryContext().addQueryFactoriesInfo("Format", name);
context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::Format, name);
return std::make_shared<ParallelFormattingOutputFormat>(builder);
}
@ -343,7 +343,7 @@ OutputFormatPtr FormatFactory::getOutputFormat(
throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
if (context.hasQueryContext() && context.getSettingsRef().log_queries)
context.getQueryContext().addQueryFactoriesInfo("Format", name);
context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::Format, name);
RowOutputFormatParams params;
params.callback = std::move(callback);

2
src/Functions/FunctionFactory.cpp
View File

@ -76,7 +76,7 @@ FunctionOverloadResolverPtr FunctionFactory::get(
const Context & context) const
{
if (context.hasQueryContext() && context.getSettingsRef().log_queries)
context.getQueryContext().addQueryFactoriesInfo("Function", name);
context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::Function, name);
return std::make_shared<FunctionOverloadResolverAdaptor>(getImpl(name, context));
}

44
src/Interpreters/Context.cpp
View File

@ -954,27 +954,37 @@ void Context::addQueryAccessInfo(const String & quoted_database_name, const Stri
}
void Context::addQueryFactoriesInfo(String factory_type, const String & created_object) const
void Context::addQueryFactoriesInfo(QueryLogFactories factory_type, const String & created_object) const
{
assert(global_context != this || getApplicationType() == ApplicationType::LOCAL);
auto lock = getLock();
if (factory_type == "AggregateFunction")
query_factories_info.aggregate_functions.emplace(created_object);
else if (factory_type == "Database")
query_factories_info.databases.emplace(created_object);
else if (factory_type == "DataType")
query_factories_info.data_types.emplace(created_object);
else if (factory_type == "Dictionary")
query_factories_info.dictionaries.emplace(created_object);
else if (factory_type == "Format")
query_factories_info.formats.emplace(created_object);
else if (factory_type == "Function")
query_factories_info.functions.emplace(created_object);
else if (factory_type == "Storage")
query_factories_info.storages.emplace(created_object);
else if (factory_type == "TableFunction")
query_factories_info.table_functions.emplace(created_object);
switch (factory_type)
{
case QueryLogFactories::AggregateFunction:
query_factories_info.aggregate_functions.emplace(created_object);
break;
case QueryLogFactories::Database:
query_factories_info.databases.emplace(created_object);
break;
case QueryLogFactories::DataType:
query_factories_info.data_types.emplace(created_object);
break;
case QueryLogFactories::Dictionary:
query_factories_info.dictionaries.emplace(created_object);
break;
case QueryLogFactories::Format:
query_factories_info.formats.emplace(created_object);
break;
case QueryLogFactories::Function:
query_factories_info.functions.emplace(created_object);
break;
case QueryLogFactories::Storage:
query_factories_info.storages.emplace(created_object);
break;
case QueryLogFactories::TableFunction:
query_factories_info.table_functions.emplace(created_object);
}
}

33
src/Interpreters/Context.h
View File

@ -201,18 +201,20 @@ private:
QueryAccessInfo query_access_info;
/// Record names of created objects of factories (for testing, etc)
struct QueryFactoriesInfo
{
std::set<std::string> aggregate_functions;
std::set<std::string> databases;
std::set<std::string> data_types;
std::set<std::string> dictionaries;
std::set<std::string> formats;
std::set<std::string> functions;
std::set<std::string> storages;
std::set<std::string> table_functions;
std::unordered_set<std::string> aggregate_functions;
std::unordered_set<std::string> databases;
std::unordered_set<std::string> data_types;
std::unordered_set<std::string> dictionaries;
std::unordered_set<std::string> formats;
std::unordered_set<std::string> functions;
std::unordered_set<std::string> storages;
std::unordered_set<std::string> table_functions;
};
/// Needs to be chandged while having const context in factories methods
mutable QueryFactoriesInfo query_factories_info;
//TODO maybe replace with temporary tables?
@ -383,8 +385,21 @@ public:
const QueryAccessInfo & getQueryAccessInfo() const { return query_access_info; }
void addQueryAccessInfo(const String & quoted_database_name, const String & full_quoted_table_name, const Names & column_names);
/// Supported factories for records in query_log
enum class QueryLogFactories
{
AggregateFunction,
Database,
DataType,
Dictionary,
Format,
Function,
Storage,
TableFunction
};
const QueryFactoriesInfo & getQueryFactoriesInfo() const { return query_factories_info; }
void addQueryFactoriesInfo(String factory_type, const String & created_object) const;
void addQueryFactoriesInfo(QueryLogFactories factory_type, const String & created_object) const;
StoragePtr executeTableFunction(const ASTPtr & table_expression);

88
src/Interpreters/QueryLog.cpp
View File

@ -97,14 +97,14 @@ Block QueryLogElement::createBlock()
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Settings.Names"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Settings.Values"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_aggregate_functions"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_databases"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_data_types"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_dictionaries"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_formats"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_functions"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_storages"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "factory_table_functions"}
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_aggregate_functions"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_databases"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_data_types"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_dictionaries"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_formats"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_functions"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_storages"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "used_table_functions"}
};
}
@ -140,21 +140,23 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
auto & column_tables = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_columns = typeid_cast<ColumnArray &>(*columns[i++]);
auto insert_array_from_set = [](const std::set<String> & data, ColumnArray & column)
{
size_t size = 0;
for (const auto & name : data)
auto fill_column = [](const std::set<String> & data, ColumnArray & column)
{
column.getData().insertData(name.data(), name.size());
++size;
}
auto & offsets = column.getOffsets();
offsets.push_back(offsets.back() + size);
};
size_t size = 0;
for (const auto & name : data)
{
column.getData().insertData(name.data(), name.size());
++size;
}
auto & offsets = column.getOffsets();
offsets.push_back(offsets.back() + size);
};
insert_array_from_set(query_databases, column_databases);
insert_array_from_set(query_tables, column_tables);
insert_array_from_set(query_columns, column_columns);
fill_column(query_databases, column_databases);
fill_column(query_tables, column_tables);
fill_column(query_columns, column_columns);
}
columns[i++]->insert(exception_code);
columns[i++]->insertData(exception.data(), exception.size());
@ -196,23 +198,37 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
columns[i++]->insertDefault();
}
auto & column_aggregate_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_database_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_data_type_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_dictionary_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_format_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_storage_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_table_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
{
auto fill_column = [](const std::unordered_set<String> & data, ColumnArray & column)
{
size_t size = 0;
for (const auto & name : data)
{
column.getData().insertData(name.data(), name.size());
++size;
}
auto & offsets = column.getOffsets();
offsets.push_back(offsets.back() + size);
};
insert_array_from_set(factory_aggregate_functions, column_aggregate_function_factory_objects);
insert_array_from_set(factory_databases, column_database_factory_objects);
insert_array_from_set(factory_data_types, column_data_type_factory_objects);
insert_array_from_set(factory_dictionaries, column_dictionary_factory_objects);
insert_array_from_set(factory_formats, column_format_factory_objects);
insert_array_from_set(factory_functions, column_function_factory_objects);
insert_array_from_set(factory_storages, column_storage_factory_objects);
insert_array_from_set(factory_table_functions, column_table_function_factory_objects);
auto & column_aggregate_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_database_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_data_type_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_dictionary_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_format_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_storage_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
auto & column_table_function_factory_objects = typeid_cast<ColumnArray &>(*columns[i++]);
fill_column(used_aggregate_functions, column_aggregate_function_factory_objects);
fill_column(used_databases, column_database_factory_objects);
fill_column(used_data_types, column_data_type_factory_objects);
fill_column(used_dictionaries, column_dictionary_factory_objects);
fill_column(used_formats, column_format_factory_objects);
fill_column(used_functions, column_function_factory_objects);
fill_column(used_storages, column_storage_factory_objects);
fill_column(used_table_functions, column_table_function_factory_objects);
}
}
void QueryLogElement::appendClientInfo(const ClientInfo & client_info, MutableColumns & columns, size_t & i)

16
src/Interpreters/QueryLog.h
View File

@ -58,14 +58,14 @@ struct QueryLogElement
std::set<String> query_tables;
std::set<String> query_columns;
std::set<String> factory_aggregate_functions;
std::set<String> factory_databases;
std::set<String> factory_data_types;
std::set<String> factory_dictionaries;
std::set<String> factory_formats;
std::set<String> factory_functions;
std::set<String> factory_storages;
std::set<String> factory_table_functions;
std::unordered_set<String> used_aggregate_functions;
std::unordered_set<String> used_databases;
std::unordered_set<String> used_data_types;
std::unordered_set<String> used_dictionaries;
std::unordered_set<String> used_formats;
std::unordered_set<String> used_functions;
std::unordered_set<String> used_storages;
std::unordered_set<String> used_table_functions;
Int32 exception_code{}; // because ErrorCodes are int
String exception;

16
src/Interpreters/executeQuery.cpp
View File

@ -735,14 +735,14 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
elem.profile_counters = std::move(info.profile_counters);
const auto & factories_info = context.getQueryFactoriesInfo();
elem.factory_aggregate_functions = factories_info.aggregate_functions;
elem.factory_databases = factories_info.databases;
elem.factory_data_types = factories_info.data_types;
elem.factory_dictionaries = factories_info.dictionaries;
elem.factory_formats = factories_info.formats;
elem.factory_functions = factories_info.functions;
elem.factory_storages = factories_info.storages;
elem.factory_table_functions = factories_info.table_functions;
elem.used_aggregate_functions = factories_info.aggregate_functions;
elem.used_databases = factories_info.databases;
elem.used_data_types = factories_info.data_types;
elem.used_dictionaries = factories_info.dictionaries;
elem.used_formats = factories_info.formats;
elem.used_functions = factories_info.functions;
elem.used_storages = factories_info.storages;
elem.used_table_functions = factories_info.table_functions;
if (log_queries && elem.type >= log_queries_min_type && Int64(elem.query_duration_ms) >= log_queries_min_query_duration_ms)
{

2
src/Storages/StorageFactory.cpp
View File

@ -191,7 +191,7 @@ StoragePtr StorageFactory::get(
}
if (local_context.hasQueryContext() && context.getSettingsRef().log_queries)
local_context.getQueryContext().addQueryFactoriesInfo("Storage", name);
local_context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::Storage, name);
return res;
}

2
src/TableFunctions/TableFunctionFactory.cpp
View File

@ -46,7 +46,7 @@ TableFunctionPtr TableFunctionFactory::get(
res->parseArguments(ast_function, context);
if (context.hasQueryContext() && context.getSettingsRef().log_queries)
context.getQueryContext().addQueryFactoriesInfo("TableFunction", table_function->name);
context.getQueryContext().addQueryFactoriesInfo(Context::QueryLogFactories::TableFunction, table_function->name);
return res;
}

14
tests/queries/0_stateless/01656_test_query_log_factories_info.reference
View File

@ -1,13 +1,13 @@
1 100
aggreagte_functions table_functions functions
['count','uniq'] ['numbers'] ['addDays','plus','toDate']
used_aggregate_functions used_table_functions used_functions
['uniq','count'] ['numbers'] ['plus','toDate','addDays']
\N
\N
functions data_types
used_functions used_data_types
['cast'] ['Nullable','UInt8']
data_types databases storages
[] ['Atomic'] []
data_types databases storages
['DateTime','String'] [] ['Memory']
used_databases
['Atomic']
used_data_types used_storages
['DateTime','String'] ['Memory']

13
tests/queries/0_stateless/01656_test_query_log_factories_info.sql
View File

@ -1,16 +1,13 @@
SELECT uniq(1, 1, 2, 2), count(toDate('2000-12-05') + number as d) FROM numbers(100);
SYSTEM FLUSH LOGS;
SELECT factory_aggregate_functions AS aggreagte_functions,
factory_table_functions AS table_functions,
factory_functions AS functions
FROM system.query_log WHERE type = 'QueryFinish' AND
(query LIKE '%toDate(\'2000-12-05\')%')
SELECT used_aggregate_functions, used_table_functions, used_functions
FROM system.query_log WHERE type = 'QueryFinish' AND (query LIKE '%toDate(\'2000-12-05\')%')
ORDER BY query_start_time DESC LIMIT 1 FORMAT TabSeparatedWithNames;
SELECT '';
SELECT CAST(arrayJoin([NULL, NULL]) AS Nullable(UInt8)) AS x;
SYSTEM FLUSH LOGS;
SELECT factory_functions AS functions, factory_data_types as data_types
SELECT used_functions, used_data_types
FROM system.query_log WHERE type = 'QueryFinish' AND query LIKE 'SELECT CAST(arrayJ%'
ORDER BY query_start_time DESC LIMIT 1 FORMAT TabSeparatedWithNames;
SELECT '';
@ -19,11 +16,11 @@ DROP database IF EXISTS test_query_log_factories_info1;
CREATE database test_query_log_factories_info1 ENGINE=Atomic;
CREATE OR REPLACE TABLE test_query_log_factories_info1.memory_table (event String, date DateTime) ENGINE=Memory();
SYSTEM FLUSH LOGS;
SELECT factory_data_types AS data_types, factory_databases AS databases, factory_storages AS storages
SELECT used_databases
FROM system.query_log
WHERE type == 'QueryFinish' AND (query LIKE '%database test_query_log_factories_info%')
ORDER BY query_start_time DESC LIMIT 1 FORMAT TabSeparatedWithNames;
SELECT factory_data_types AS data_types, factory_databases AS databases, factory_storages AS storages
SELECT used_data_types, used_storages
FROM system.query_log
WHERE type == 'QueryFinish' AND (query LIKE '%TABLE test%')
ORDER BY query_start_time DESC LIMIT 1 FORMAT TabSeparatedWithNames;