thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-17 13:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

(feature) Use Map data type for system logs tables

Browse Source
This commit is contained in:
sundy-li 2021-01-05 10:10:19 +00:00
parent aff724ea7d
commit cef8bcc011
8 changed files with 100 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/Core/Settings.cpp
View File

@ -2,6 +2,7 @@
#include <Poco/Util/AbstractConfiguration.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnMap.h>
#include <Common/typeid_cast.h>
#include <string.h>
#include <boost/program_options/options_description.hpp>
@ -56,40 +57,27 @@ void Settings::loadSettingsFromConfig(const String & path, const Poco::Util::Abs
}
}
void Settings::dumpToArrayColumns(IColumn * column_names_, IColumn * column_values_, bool changed_only)
void Settings::dumpToMapColumn(IColumn * column, bool changed_only)
{
/// Convert ptr and make simple check
auto * column_names = (column_names_) ? &typeid_cast<ColumnArray &>(*column_names_) : nullptr;
auto * column_values = (column_values_) ? &typeid_cast<ColumnArray &>(*column_values_) : nullptr;
auto * column_map = column ? &typeid_cast<ColumnMap &>(*column) : nullptr;
if (!column_map)
return;
size_t count = 0;
auto & offsets = column_map->getNestedColumn().getOffsets();
auto & tuple_column = column_map->getNestedData();
auto & key_column = tuple_column.getColumn(0);
auto & value_column = tuple_column.getColumn(1);
size_t size = 0;
for (const auto & setting : all(changed_only ? SKIP_UNCHANGED : SKIP_NONE))
{
if (column_names)
{
auto name = setting.getName();
column_names->getData().insertData(name.data(), name.size());
}
if (column_values)
column_values->getData().insert(setting.getValueString());
++count;
}
if (column_names)
{
auto & offsets = column_names->getOffsets();
offsets.push_back(offsets.back() + count);
}
/// Nested columns case
bool the_same_offsets = column_names && column_values && column_names->getOffsetsPtr() == column_values->getOffsetsPtr();
if (column_values && !the_same_offsets)
{
auto & offsets = column_values->getOffsets();
offsets.push_back(offsets.back() + count);
auto name = setting.getName();
key_column.insertData(name.data(), name.size());
value_column.insert(setting.getValueString());
size++;
}
offsets.push_back(offsets.back() + size);
}
void Settings::addProgramOptions(boost::program_options::options_description & options)

2
src/Core/Settings.h
View File

@ -523,7 +523,7 @@ struct Settings : public BaseSettings<SettingsTraits>
void loadSettingsFromConfig(const String & path, const Poco::Util::AbstractConfiguration & config);
/// Dumps profile events to two columns of type Array(String)
void dumpToArrayColumns(IColumn * column_names, IColumn * column_values, bool changed_only = true);
void dumpToMapColumn(IColumn * column, bool changed_only = true);
/// Adds program options to set the settings from a command line.
/// (Don't forget to call notify() on the `variables_map` after parsing it!)

3
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -618,7 +618,8 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat
}
}
if (!create.attach && !settings.allow_experimental_map_type)
// enable allow_experimental_map_type for system tables
if (create.database != "system" && !create.attach && !settings.allow_experimental_map_type)
{
for (const auto & name_and_type_pair : properties.columns.getAllPhysical())
{

47
src/Interpreters/ProfileEventsExt.cpp
View File

@ -3,6 +3,7 @@
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnMap.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeArray.h>
@ -11,14 +12,18 @@ namespace ProfileEvents
{
/// Put implementation here to avoid extra linking dependencies for clickhouse_common_io
void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_, DB::IColumn * column_values_, bool nonzero_only)
void dumpToMapColumn(const Counters & counters, DB::IColumn * column, bool nonzero_only)
{
/// Convert ptr and make simple check
auto * column_names = (column_names_) ? &typeid_cast<DB::ColumnArray &>(*column_names_) : nullptr;
auto * column_values = (column_values_) ? &typeid_cast<DB::ColumnArray &>(*column_values_) : nullptr;
auto * column_map = column ? &typeid_cast<DB::ColumnMap &>(*column) : nullptr;
if (!column_map)
return;
auto & offsets = column_map->getNestedColumn().getOffsets();
auto & tuple_column = column_map->getNestedData();
auto & key_column = tuple_column.getColumn(0);
auto & value_column = tuple_column.getColumn(1);
size_t size = 0;
for (Event event = 0; event < Counters::num_counters; ++event)
{
UInt64 value = counters[event].load(std::memory_order_relaxed);
@ -26,34 +31,12 @@ void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names_,
if (nonzero_only && 0 == value)
continue;
++size;
if (column_names)
{
const char * desc = ProfileEvents::getName(event);
column_names->getData().insertData(desc, strlen(desc));
}
if (column_values)
column_values->getData().insert(value);
}
if (column_names)
{
auto & offsets = column_names->getOffsets();
offsets.push_back(offsets.back() + size);
}
if (column_values)
{
/// Nested columns case
bool the_same_offsets = column_names && column_names->getOffsetsPtr().get() == column_values->getOffsetsPtr().get();
if (!the_same_offsets)
{
auto & offsets = column_values->getOffsets();
offsets.push_back(offsets.back() + size);
}
const char * desc = ProfileEvents::getName(event);
key_column.insertData(desc, strlen(desc));
value_column.insert(value);
size++;
}
offsets.push_back(offsets.back() + size);
}
}

4
src/Interpreters/ProfileEventsExt.h
View File

@ -6,7 +6,7 @@
namespace ProfileEvents
{
/// Dumps profile events to two columns Array(String) and Array(UInt64)
void dumpToArrayColumns(const Counters & counters, DB::IColumn * column_names, DB::IColumn * column_value, bool nonzero_only = true);
/// Dumps profile events to columns Map(String, UInt64)
void dumpToMapColumn(const Counters & counters, DB::IColumn * column, bool nonzero_only = true);
}

117
src/Interpreters/QueryLog.cpp
View File

@ -3,6 +3,7 @@
#include <Columns/ColumnFixedString.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnMap.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypeDate.h>
@ -10,6 +11,7 @@
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeEnum.h>
#include <DataTypes/DataTypeFactory.h>
#include <DataTypes/DataTypeMap.h>
#include <DataTypes/DataTypeLowCardinality.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
@ -19,6 +21,7 @@
#include <Common/ClickHouseRevision.h>
#include <Common/IPv6ToBinary.h>
#include <Common/ProfileEvents.h>
#include <Common/typeid_cast.h>
namespace DB
@ -35,67 +38,61 @@ Block QueryLogElement::createBlock()
{"ExceptionWhileProcessing", static_cast<Int8>(EXCEPTION_WHILE_PROCESSING)}
});
return
{
{std::move(query_status_datatype), "type"},
{std::make_shared<DataTypeDate>(), "event_date"},
{std::make_shared<DataTypeDateTime>(), "event_time"},
{std::make_shared<DataTypeDateTime64>(6), "event_time_microseconds"},
{std::make_shared<DataTypeDateTime>(), "query_start_time"},
{std::make_shared<DataTypeDateTime64>(6), "query_start_time_microseconds"},
{std::make_shared<DataTypeUInt64>(), "query_duration_ms"},
return {
{std::move(query_status_datatype), "type"},
{std::make_shared<DataTypeDate>(), "event_date"},
{std::make_shared<DataTypeDateTime>(), "event_time"},
{std::make_shared<DataTypeDateTime64>(6), "event_time_microseconds"},
{std::make_shared<DataTypeDateTime>(), "query_start_time"},
{std::make_shared<DataTypeDateTime64>(6), "query_start_time_microseconds"},
{std::make_shared<DataTypeUInt64>(), "query_duration_ms"},
{std::make_shared<DataTypeUInt64>(), "read_rows"},
{std::make_shared<DataTypeUInt64>(), "read_bytes"},
{std::make_shared<DataTypeUInt64>(), "written_rows"},
{std::make_shared<DataTypeUInt64>(), "written_bytes"},
{std::make_shared<DataTypeUInt64>(), "result_rows"},
{std::make_shared<DataTypeUInt64>(), "result_bytes"},
{std::make_shared<DataTypeUInt64>(), "memory_usage"},
{std::make_shared<DataTypeUInt64>(), "read_rows"},
{std::make_shared<DataTypeUInt64>(), "read_bytes"},
{std::make_shared<DataTypeUInt64>(), "written_rows"},
{std::make_shared<DataTypeUInt64>(), "written_bytes"},
{std::make_shared<DataTypeUInt64>(), "result_rows"},
{std::make_shared<DataTypeUInt64>(), "result_bytes"},
{std::make_shared<DataTypeUInt64>(), "memory_usage"},
{std::make_shared<DataTypeString>(), "current_database"},
{std::make_shared<DataTypeString>(), "query"},
{std::make_shared<DataTypeUInt64>(), "normalized_query_hash"},
{std::make_shared<DataTypeString>(), "current_database"},
{std::make_shared<DataTypeString>(), "query"},
{std::make_shared<DataTypeUInt64>(), "normalized_query_hash"},
{std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>()), "query_kind"},
{std::make_shared<DataTypeArray>(
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "databases"},
{std::make_shared<DataTypeArray>(
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "tables"},
{std::make_shared<DataTypeArray>(
std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "columns"},
{std::make_shared<DataTypeInt32>(), "exception_code"},
{std::make_shared<DataTypeString>(), "exception"},
{std::make_shared<DataTypeString>(), "stack_trace"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "databases"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "tables"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeLowCardinality>(std::make_shared<DataTypeString>())), "columns"},
{std::make_shared<DataTypeInt32>(), "exception_code"},
{std::make_shared<DataTypeString>(), "exception"},
{std::make_shared<DataTypeString>(), "stack_trace"},
{std::make_shared<DataTypeUInt8>(), "is_initial_query"},
{std::make_shared<DataTypeString>(), "user"},
{std::make_shared<DataTypeString>(), "query_id"},
{DataTypeFactory::instance().get("IPv6"), "address"},
{std::make_shared<DataTypeUInt16>(), "port"},
{std::make_shared<DataTypeString>(), "initial_user"},
{std::make_shared<DataTypeString>(), "initial_query_id"},
{DataTypeFactory::instance().get("IPv6"), "initial_address"},
{std::make_shared<DataTypeUInt16>(), "initial_port"},
{std::make_shared<DataTypeUInt8>(), "interface"},
{std::make_shared<DataTypeString>(), "os_user"},
{std::make_shared<DataTypeString>(), "client_hostname"},
{std::make_shared<DataTypeString>(), "client_name"},
{std::make_shared<DataTypeUInt32>(), "client_revision"},
{std::make_shared<DataTypeUInt32>(), "client_version_major"},
{std::make_shared<DataTypeUInt32>(), "client_version_minor"},
{std::make_shared<DataTypeUInt32>(), "client_version_patch"},
{std::make_shared<DataTypeUInt8>(), "http_method"},
{std::make_shared<DataTypeString>(), "http_user_agent"},
{std::make_shared<DataTypeString>(), "forwarded_for"},
{std::make_shared<DataTypeString>(), "quota_key"},
{std::make_shared<DataTypeUInt8>(), "is_initial_query"},
{std::make_shared<DataTypeString>(), "user"},
{std::make_shared<DataTypeString>(), "query_id"},
{DataTypeFactory::instance().get("IPv6"), "address"},
{std::make_shared<DataTypeUInt16>(), "port"},
{std::make_shared<DataTypeString>(), "initial_user"},
{std::make_shared<DataTypeString>(), "initial_query_id"},
{DataTypeFactory::instance().get("IPv6"), "initial_address"},
{std::make_shared<DataTypeUInt16>(), "initial_port"},
{std::make_shared<DataTypeUInt8>(), "interface"},
{std::make_shared<DataTypeString>(), "os_user"},
{std::make_shared<DataTypeString>(), "client_hostname"},
{std::make_shared<DataTypeString>(), "client_name"},
{std::make_shared<DataTypeUInt32>(), "client_revision"},
{std::make_shared<DataTypeUInt32>(), "client_version_major"},
{std::make_shared<DataTypeUInt32>(), "client_version_minor"},
{std::make_shared<DataTypeUInt32>(), "client_version_patch"},
{std::make_shared<DataTypeUInt8>(), "http_method"},
{std::make_shared<DataTypeString>(), "http_user_agent"},
{std::make_shared<DataTypeString>(), "forwarded_for"},
{std::make_shared<DataTypeString>(), "quota_key"},
{std::make_shared<DataTypeUInt32>(), "revision"},
{std::make_shared<DataTypeUInt32>(), "revision"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "thread_ids"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "ProfileEvents.Names"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "ProfileEvents.Values"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Settings.Names"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Settings.Values"}
{std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeUInt64>()), "ProfileEvents"},
{std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()), "Settings"},
};
}
@ -165,26 +162,22 @@ void QueryLogElement::appendToBlock(MutableColumns & columns) const
if (profile_counters)
{
auto * column_names = columns[i++].get();
auto * column_values = columns[i++].get();
ProfileEvents::dumpToArrayColumns(*profile_counters, column_names, column_values, true);
auto * column = columns[i++].get();
ProfileEvents::dumpToMapColumn(*profile_counters, column, true);
}
else
{
columns[i++]->insertDefault();
columns[i++]->insertDefault();
}
if (query_settings)
{
auto * column_names = columns[i++].get();
auto * column_values = columns[i++].get();
query_settings->dumpToArrayColumns(column_names, column_values, true);
auto * column = columns[i++].get();
query_settings->dumpToMapColumn(column, true);
}
else
{
columns[i++]->insertDefault();
columns[i++]->insertDefault();
}
}

10
src/Interpreters/QueryThreadLog.cpp
View File

@ -5,6 +5,7 @@
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeMap.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypeFactory.h>
@ -66,8 +67,7 @@ Block QueryThreadLogElement::createBlock()
{std::make_shared<DataTypeUInt32>(), "revision"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "ProfileEvents.Names"},
{std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>()), "ProfileEvents.Values"}
{std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeUInt64>()), "ProfileEvents"},
};
}
@ -104,14 +104,12 @@ void QueryThreadLogElement::appendToBlock(MutableColumns & columns) const
if (profile_counters)
{
auto * column_names = columns[i++].get();
auto * column_values = columns[i++].get();
dumpToArrayColumns(*profile_counters, column_names, column_values, true);
auto * column = columns[i++].get();
ProfileEvents::dumpToMapColumn(*profile_counters, column, true);
}
else
{
columns[i++]->insertDefault();
columns[i++]->insertDefault();
}
}

16
src/Storages/System/StorageSystemProcesses.cpp
View File

@ -123,28 +123,24 @@ void StorageSystemProcesses::fillData(MutableColumns & res_columns, const Contex
}
{
IColumn * column_profile_events_names = res_columns[i++].get();
IColumn * column_profile_events_values = res_columns[i++].get();
IColumn * column = res_columns[i++].get();
if (process.profile_counters)
ProfileEvents::dumpToArrayColumns(*process.profile_counters, column_profile_events_names, column_profile_events_values, true);
ProfileEvents::dumpToMapColumn(*process.profile_counters, column, true);
else
{
column_profile_events_names->insertDefault();
column_profile_events_values->insertDefault();
column->insertDefault();
}
}
{
IColumn * column_settings_names = res_columns[i++].get();
IColumn * column_settings_values = res_columns[i++].get();
IColumn * column = res_columns[i++].get();
if (process.query_settings)
process.query_settings->dumpToArrayColumns(column_settings_names, column_settings_values, true);
process.query_settings->dumpToMapColumn(column, true);
else
{
column_settings_names->insertDefault();
column_settings_values->insertDefault();
column->insertDefault();
}
}
}