thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-17 13:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

(feature) Use Map data type for system logs tables

Browse Source
This commit is contained in:
sundy-li 2021-01-05 10:10:19 +00:00
parent aff724ea7d
commit cef8bcc011
8 changed files with 100 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
src/Core/Settings.cpp
View File

@ -2,6 +2,7 @@
#include <Poco/Util/AbstractConfiguration.h> #include <Poco/Util/AbstractConfiguration.h>
#include <Columns/ColumnArray.h> #include <Columns/ColumnArray.h>
#include <Columns/ColumnMap.h>
#include <Common/typeid_cast.h> #include <Common/typeid_cast.h>
#include <string.h> #include <string.h>
#include <boost/program_options/options_description.hpp> #include <boost/program_options/options_description.hpp>
@ -56,40 +57,27 @@ void Settings::loadSettingsFromConfig(const String & path, const Poco::Util::Abs
} }
} }
void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_values_, bool changed_only) void Settings::dumpToMapColumn(IColumn * column, bool changed_only)
{ {
/// Convert ptr and make simple check /// Convert ptr and make simple check
auto * column_names = (column_names_) ? &typeid_cast<ColumnArray &>(*column_names_) : nullptr; auto * column_map = column ? &typeid_cast<ColumnMap &>(*column) : nullptr;
auto * column_values = (column_values_) ? &typeid_cast<ColumnArray &>(*column_values_) : nullptr; if (!column_map)
return;
size_t count = 0; auto & offsets = column_map->getNestedColumn().getOffsets();
auto & tuple_column = column_map->getNestedData();
auto & key_column = tuple_column.getColumn(0);
auto & value_column = tuple_column.getColumn(1);
size_t size = 0;
for (const auto & setting : all(changed_only ? SKIP_UNCHANGED : SKIP_NONE)) for (const auto & setting : all(changed_only ? SKIP_UNCHANGED : SKIP_NONE))
{
if (column_names)
{ {
auto name = setting.getName(); auto name = setting.getName();
column_names->getData().insertData(name.data(), name.size()); key_column.insertData(name.data(), name.size());
} value_column.insert(setting.getValueString());
if (column_values) size++;
column_values->getData().insert(setting.getValueString());
++count;
}
if (column_names)
{
auto & offsets = column_names->getOffsets();
offsets.push_back(offsets.back() + count);
}
/// Nested columns case
bool the_same_offsets = column_names && column_values && column_names->getOffsetsPtr() == column_values->getOffsetsPtr();
if (column_values && !the_same_offsets)
{
auto & offsets = column_values->getOffsets();
offsets.push_back(offsets.back() + count);
} }
offsets.push_back(offsets.back() + size);
} }
void Settings::addProgramOptions(boost::program_options::options_description & options) void Settings::addProgramOptions(boost::program_options::options_description & options)

2
src/Core/Settings.h
View File

@ -523,7 +523,7 @@ struct Settings : public BaseSettings<SettingsTraits>
void loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config); void loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config);
/// Dumps profile events to two columns of type Array(String) /// Dumps profile events to two columns of type Array(String)
void dumpToArrayColumns(IColumn * column_names, IColumn * column_values, bool changed_only = true); void dumpToMapColumn(IColumn * column, bool changed_only = true);
/// Adds program options to set the settings from a command line. /// Adds program options to set the settings from a command line.
/// (Don't forget to call notify() on the `variables_map` after parsing it!) /// (Don't forget to call notify() on the `variables_map` after parsing it!)

3
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -618,7 +618,8 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
} }
} }
if (!create.attach && !settings.allow_experimental_map_type) // enable allow_experimental_map_type for system tables
if (create.database != "system" && !create.attach && !settings.allow_experimental_map_type)
{ {
for (const auto & name_and_type_pair : properties.columns.getAllPhysical()) for (const auto & name_and_type_pair : properties.columns.getAllPhysical())
{ {

43
src/Interpreters/ProfileEventsExt.cpp
View File

@ -3,6 +3,7 @@
#include <Columns/ColumnsNumber.h> #include <Columns/ColumnsNumber.h>
#include <Columns/ColumnString.h> #include <Columns/ColumnString.h>
#include <Columns/ColumnArray.h> #include <Columns/ColumnArray.h>
#include <Columns/ColumnMap.h>
#include <DataTypes/DataTypesNumber.h> #include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeString.h> #include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeArray.h> #include <DataTypes/DataTypeArray.h>
@ -11,14 +12,18 @@ namespace ProfileEvents
{ {
/// Put implementation here to avoid extra linking dependencies for clickhouse_common_io /// Put implementation here to avoid extra linking dependencies for clickhouse_common_io
void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_, DB::IColumn * column_values_, bool nonzero_only) void dumpToMapColumn(const Counters & counters, DB::IColumn * column, bool nonzero_only)
{ {
/// Convert ptr and make simple check auto * column_map = column ? &typeid_cast<DB::ColumnMap &>(*column) : nullptr;
auto * column_names = (column_names_) ? &typeid_cast<DB::ColumnArray &>(*column_names_) : nullptr; if (!column_map)
auto * column_values = (column_values_) ? &typeid_cast<DB::ColumnArray &>(*column_values_) : nullptr; return;
auto & offsets = column_map->getNestedColumn().getOffsets();
auto & tuple_column = column_map->getNestedData();
auto & key_column = tuple_column.getColumn(0);
auto & value_column = tuple_column.getColumn(1);
size_t size = 0; size_t size = 0;
for (Event event = 0; event < Counters::num_counters; ++event) for (Event event = 0; event < Counters::num_counters; ++event)
{ {
UInt64 value = counters[event].load(std::memory_order_relaxed); UInt64 value = counters[event].load(std::memory_order_relaxed);
@ -26,34 +31,12 @@ void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_,
if (nonzero_only && 0 == value) if (nonzero_only && 0 == value)
continue; continue;
++size;
if (column_names)
{
const char * desc = ProfileEvents::getName(event); const char * desc = ProfileEvents::getName(event);
column_names->getData().insertData(desc, strlen(desc)); key_column.insertData(desc, strlen(desc));
value_column.insert(value);
size++;
} }
if (column_values)
column_values->getData().insert(value);
}
if (column_names)
{
auto & offsets = column_names->getOffsets();
offsets.push_back(offsets.back() + size); offsets.push_back(offsets.back() + size);
} }
if (column_values)
{
/// Nested columns case
bool the_same_offsets = column_names && column_names->getOffsetsPtr().get() == column_values->getOffsetsPtr().get();
if (!the_same_offsets)
{
auto & offsets = column_values->getOffsets();
offsets.push_back(offsets.back() + size);
}
}
}
} }

4
src/Interpreters/ProfileEventsExt.h
View File

@ -6,7 +6,7 @@
namespace ProfileEvents namespace ProfileEvents
{ {
/// Dumps profile events to two columns Array(String) and Array(UInt64) /// Dumps profile events to columns Map(String, UInt64)
void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names, DB::IColumn * column_value, bool nonzero_only = true); void dumpToMapColumn(const Counters & counters, DB::IColumn * column, bool nonzero_only = true);
} }

33
src/Interpreters/QueryLog.cpp
View File

@ -3,6 +3,7 @@
#include <Columns/ColumnFixedString.h> #include <Columns/ColumnFixedString.h>
#include <Columns/ColumnString.h> #include <Columns/ColumnString.h>
#include <Columns/ColumnsNumber.h> #include <Columns/ColumnsNumber.h>
#include <Columns/ColumnMap.h>
#include <DataTypes/DataTypeArray.h> #include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDateTime64.h> #include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypeDate.h> #include <DataTypes/DataTypeDate.h>
@ -10,6 +11,7 @@
#include <DataTypes/DataTypeDateTime.h> #include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeEnum.h> #include <DataTypes/DataTypeEnum.h>
#include <DataTypes/DataTypeFactory.h> #include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeMap.h>
#include <DataTypes/DataTypeLowCardinality.h> #include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypeString.h> #include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h> #include <DataTypes/DataTypesNumber.h>
@ -19,6 +21,7 @@
#include <Common/ClickHouseRevision.h> #include <Common/ClickHouseRevision.h>
#include <Common/IPv6ToBinary.h> #include <Common/IPv6ToBinary.h>
#include <Common/ProfileEvents.h> #include <Common/ProfileEvents.h>
#include <Common/typeid_cast.h>
namespace DB namespace DB
@ -35,8 +38,7 @@ Block QueryLogElement::createBlock()
{"ExceptionWhileProcessing", static_cast<Int8>(EXCEPTION_WHILE_PROCESSING)} {"ExceptionWhileProcessing", static_cast<Int8>(EXCEPTION_WHILE_PROCESSING)}
}); });
return return {
{
{std::move(query_status_datatype), "type"}, {std::move(query_status_datatype), "type"},
{std::make_shared<DataTypeDate>(), "event_date"}, {std::make_shared<DataTypeDate>(), "event_date"},
{std::make_shared<DataTypeDateTime>(), "event_time"}, {std::make_shared<DataTypeDateTime>(), "event_time"},
@ -57,12 +59,9 @@ Block QueryLogElement::createBlock()
{std::make_shared<DataTypeString>(), "query"}, {std::make_shared<DataTypeString>(), "query"},
{std::make_shared<DataTypeUInt64>(), "normalized_query_hash"}, {std::make_shared<DataTypeUInt64>(), "normalized_query_hash"},
{std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()), "query_kind"}, {std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()), "query_kind"},
{std::make_shared<DataTypeArray>( {std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "databases"},
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "databases"}, {std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "tables"},
{std::make_shared<DataTypeArray>( {std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "columns"},
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "tables"},
{std::make_shared<DataTypeArray>(
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "columns"},
{std::make_shared<DataTypeInt32>(), "exception_code"}, {std::make_shared<DataTypeInt32>(), "exception_code"},
{std::make_shared<DataTypeString>(), "exception"}, {std::make_shared<DataTypeString>(), "exception"},
{std::make_shared<DataTypeString>(), "stack_trace"}, {std::make_shared<DataTypeString>(), "stack_trace"},
@ -92,10 +91,8 @@ Block QueryLogElement::createBlock()
{std::make_shared<DataTypeUInt32>(), "revision"}, {std::make_shared<DataTypeUInt32>(), "revision"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "thread_ids"}, {std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "thread_ids"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "ProfileEvents.Names"}, {std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeUInt64>()), "ProfileEvents"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "ProfileEvents.Values"}, {std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()), "Settings"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Settings.Names"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Settings.Values"}
}; };
} }
@ -165,26 +162,22 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
if (profile_counters) if (profile_counters)
{ {
auto * column_names = columns[i++].get(); auto * column = columns[i++].get();
auto * column_values = columns[i++].get(); ProfileEvents::dumpToMapColumn(*profile_counters, column, true);
ProfileEvents::dumpToArrayColumns(*profile_counters, column_names, column_values, true);
} }
else else
{ {
columns[i++]->insertDefault(); columns[i++]->insertDefault();
columns[i++]->insertDefault();
} }
if (query_settings) if (query_settings)
{ {
auto * column_names = columns[i++].get(); auto * column = columns[i++].get();
auto * column_values = columns[i++].get(); query_settings->dumpToMapColumn(column, true);
query_settings->dumpToArrayColumns(column_names, column_values, true);
} }
else else
{ {
columns[i++]->insertDefault(); columns[i++]->insertDefault();
columns[i++]->insertDefault();
} }
} }

10
src/Interpreters/QueryThreadLog.cpp
View File

@ -5,6 +5,7 @@
#include <Columns/ColumnsNumber.h> #include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeArray.h> #include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDate.h> #include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeMap.h>
#include <DataTypes/DataTypeDateTime.h> #include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDateTime64.h> #include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypeFactory.h> #include <DataTypes/DataTypeFactory.h>
@ -66,8 +67,7 @@ Block QueryThreadLogElement::createBlock()
{std::make_shared<DataTypeUInt32>(), "revision"}, {std::make_shared<DataTypeUInt32>(), "revision"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "ProfileEvents.Names"}, {std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeUInt64>()), "ProfileEvents"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "ProfileEvents.Values"}
}; };
} }
@ -104,14 +104,12 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const
if (profile_counters) if (profile_counters)
{ {
auto * column_names = columns[i++].get(); auto * column = columns[i++].get();
auto * column_values = columns[i++].get(); ProfileEvents::dumpToMapColumn(*profile_counters, column, true);
dumpToArrayColumns(*profile_counters, column_names, column_values, true);
} }
else else
{ {
columns[i++]->insertDefault(); columns[i++]->insertDefault();
columns[i++]->insertDefault();
} }
} }

16
src/Storages/System/StorageSystemProcesses.cpp
View File

@ -123,28 +123,24 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Contex
} }
{ {
IColumn * column_profile_events_names = res_columns[i++].get(); IColumn * column = res_columns[i++].get();
IColumn * column_profile_events_values = res_columns[i++].get();
if (process.profile_counters) if (process.profile_counters)
ProfileEvents::dumpToArrayColumns(*process.profile_counters, column_profile_events_names, column_profile_events_values, true); ProfileEvents::dumpToMapColumn(*process.profile_counters, column, true);
else else
{ {
column_profile_events_names->insertDefault(); column->insertDefault();
column_profile_events_values->insertDefault();
} }
} }
{ {
IColumn * column_settings_names = res_columns[i++].get(); IColumn * column = res_columns[i++].get();
IColumn * column_settings_values = res_columns[i++].get();
if (process.query_settings) if (process.query_settings)
process.query_settings->dumpToArrayColumns(column_settings_names, column_settings_values, true); process.query_settings->dumpToMapColumn(column, true);
else else
{ {
column_settings_names->insertDefault(); column->insertDefault();
column_settings_values->insertDefault();
} }
} }
} }