ClickHouse/src/Storages/System/StorageSystemReplicas.cpp

209 lines
9.6 KiB
C++
Raw Normal View History

#include <Columns/ColumnString.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeDateTime.h>
#include <Storages/System/StorageSystemReplicas.h>
#include <Storages/StorageReplicatedMergeTree.h>
2017-11-20 05:22:54 +00:00
#include <Storages/VirtualColumnUtils.h>
#include <Access/ContextAccess.h>
2017-07-13 20:58:19 +00:00
#include <Common/typeid_cast.h>
#include <Databases/IDatabase.h>
2020-01-24 18:13:17 +00:00
#include <Processors/Sources/SourceFromSingleChunk.h>
namespace DB
{
2020-08-12 20:40:13 +00:00
StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_)
: IStorage(table_id_)
{
2020-06-19 15:39:41 +00:00
StorageInMemoryMetadata storage_metadata;
storage_metadata.setColumns(ColumnsDescription({
{ "database", std::make_shared<DataTypeString>() },
{ "table", std::make_shared<DataTypeString>() },
{ "engine", std::make_shared<DataTypeString>() },
{ "is_leader", std::make_shared<DataTypeUInt8>() },
{ "can_become_leader", std::make_shared<DataTypeUInt8>() },
{ "is_readonly", std::make_shared<DataTypeUInt8>() },
{ "is_session_expired", std::make_shared<DataTypeUInt8>() },
{ "future_parts", std::make_shared<DataTypeUInt32>() },
{ "parts_to_check", std::make_shared<DataTypeUInt32>() },
{ "zookeeper_path", std::make_shared<DataTypeString>() },
{ "replica_name", std::make_shared<DataTypeString>() },
{ "replica_path", std::make_shared<DataTypeString>() },
{ "columns_version", std::make_shared<DataTypeInt32>() },
{ "queue_size", std::make_shared<DataTypeUInt32>() },
{ "inserts_in_queue", std::make_shared<DataTypeUInt32>() },
{ "merges_in_queue", std::make_shared<DataTypeUInt32>() },
{ "part_mutations_in_queue", std::make_shared<DataTypeUInt32>() },
{ "queue_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "inserts_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "merges_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "part_mutations_oldest_time", std::make_shared<DataTypeDateTime>() },
{ "oldest_part_to_get", std::make_shared<DataTypeString>() },
{ "oldest_part_to_merge_to", std::make_shared<DataTypeString>() },
{ "oldest_part_to_mutate_to", std::make_shared<DataTypeString>() },
{ "log_max_index", std::make_shared<DataTypeUInt64>() },
{ "log_pointer", std::make_shared<DataTypeUInt64>() },
{ "last_queue_update", std::make_shared<DataTypeDateTime>() },
{ "absolute_delay", std::make_shared<DataTypeUInt64>() },
{ "total_replicas", std::make_shared<DataTypeUInt8>() },
{ "active_replicas", std::make_shared<DataTypeUInt8>() },
2020-02-15 00:11:09 +00:00
{ "zookeeper_exception", std::make_shared<DataTypeString>() },
}));
2020-06-19 15:39:41 +00:00
setInMemoryMetadata(storage_metadata);
}
2020-08-06 12:24:05 +00:00
Pipe StorageSystemReplicas::read(
const Names & column_names,
const StorageMetadataPtr & metadata_snapshot,
SelectQueryInfo & query_info,
ContextPtr context,
QueryProcessingStage::Enum /*processed_stage*/,
2017-12-02 02:47:12 +00:00
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
2020-06-19 17:17:13 +00:00
metadata_snapshot->check(column_names, getVirtuals(), getStorageID());
const auto access = context->getAccess();
const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
2020-01-24 16:20:36 +00:00
/// We collect a set of replicated tables.
std::map<String, std::map<String, StoragePtr>> replicated_tables;
for (const auto & db : DatabaseCatalog::instance().getDatabases())
{
/// Check if database can contain replicated tables
2020-10-27 11:29:10 +00:00
if (!db.second->canContainMergeTreeTables())
2019-10-01 12:44:17 +00:00
continue;
const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first);
2020-05-28 20:10:45 +00:00
for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
{
2020-06-02 02:06:16 +00:00
const auto & table = iterator->table();
if (!table)
continue;
if (!dynamic_cast<const StorageReplicatedMergeTree *>(table.get()))
2020-01-24 16:20:36 +00:00
continue;
if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name()))
2020-01-24 16:20:36 +00:00
continue;
2020-06-02 02:06:16 +00:00
replicated_tables[db.first][iterator->name()] = table;
}
}
2018-08-27 16:31:45 +00:00
/// Do you need columns that require a ZooKeeper request to compute.
bool with_zk_fields = false;
2018-08-27 16:31:45 +00:00
for (const auto & column_name : column_names)
{
2018-08-27 16:31:45 +00:00
if ( column_name == "log_max_index"
|| column_name == "log_pointer"
|| column_name == "total_replicas"
2020-02-15 00:11:09 +00:00
|| column_name == "active_replicas"
|| column_name == "zookeeper_exception")
{
with_zk_fields = true;
break;
}
}
MutableColumnPtr col_database_mut = ColumnString::create();
MutableColumnPtr col_table_mut = ColumnString::create();
MutableColumnPtr col_engine_mut = ColumnString::create();
for (auto & db : replicated_tables)
{
for (auto & table : db.second)
{
col_database_mut->insert(db.first);
col_table_mut->insert(table.first);
col_engine_mut->insert(table.second->getName());
}
}
ColumnPtr col_database = std::move(col_database_mut);
ColumnPtr col_table = std::move(col_table_mut);
ColumnPtr col_engine = std::move(col_engine_mut);
/// Determine what tables are needed by the conditions in the query.
{
Block filtered_block
{
{ col_database, std::make_shared<DataTypeString>(), "database" },
{ col_table, std::make_shared<DataTypeString>(), "table" },
{ col_engine, std::make_shared<DataTypeString>(), "engine" },
};
VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context);
if (!filtered_block.rows())
2020-08-06 12:24:05 +00:00
return {};
col_database = filtered_block.getByName("database").column;
col_table = filtered_block.getByName("table").column;
col_engine = filtered_block.getByName("engine").column;
}
MutableColumns res_columns = metadata_snapshot->getSampleBlock().cloneEmptyColumns();
for (size_t i = 0, size = col_database->size(); i < size; ++i)
{
StorageReplicatedMergeTree::Status status;
2017-11-04 16:46:14 +00:00
dynamic_cast<StorageReplicatedMergeTree &>(
*replicated_tables
[(*col_database)[i].safeGet<const String &>()]
[(*col_table)[i].safeGet<const String &>()]).getStatus(status, with_zk_fields);
size_t col_num = 3;
res_columns[col_num++]->insert(status.is_leader);
res_columns[col_num++]->insert(status.can_become_leader);
res_columns[col_num++]->insert(status.is_readonly);
res_columns[col_num++]->insert(status.is_session_expired);
res_columns[col_num++]->insert(status.queue.future_parts);
res_columns[col_num++]->insert(status.parts_to_check);
res_columns[col_num++]->insert(status.zookeeper_path);
res_columns[col_num++]->insert(status.replica_name);
res_columns[col_num++]->insert(status.replica_path);
res_columns[col_num++]->insert(status.columns_version);
res_columns[col_num++]->insert(status.queue.queue_size);
res_columns[col_num++]->insert(status.queue.inserts_in_queue);
res_columns[col_num++]->insert(status.queue.merges_in_queue);
res_columns[col_num++]->insert(status.queue.part_mutations_in_queue);
res_columns[col_num++]->insert(status.queue.queue_oldest_time);
res_columns[col_num++]->insert(status.queue.inserts_oldest_time);
res_columns[col_num++]->insert(status.queue.merges_oldest_time);
res_columns[col_num++]->insert(status.queue.part_mutations_oldest_time);
res_columns[col_num++]->insert(status.queue.oldest_part_to_get);
res_columns[col_num++]->insert(status.queue.oldest_part_to_merge_to);
res_columns[col_num++]->insert(status.queue.oldest_part_to_mutate_to);
res_columns[col_num++]->insert(status.log_max_index);
res_columns[col_num++]->insert(status.log_pointer);
res_columns[col_num++]->insert(status.queue.last_queue_update);
res_columns[col_num++]->insert(status.absolute_delay);
res_columns[col_num++]->insert(status.total_replicas);
res_columns[col_num++]->insert(status.active_replicas);
2020-02-15 00:11:09 +00:00
res_columns[col_num++]->insert(status.zookeeper_exception);
}
Block header = metadata_snapshot->getSampleBlock();
2020-01-24 18:13:17 +00:00
2020-01-24 18:33:47 +00:00
Columns fin_columns;
fin_columns.reserve(res_columns.size());
2020-01-24 18:13:17 +00:00
for (auto & col : res_columns)
2020-01-24 18:33:47 +00:00
fin_columns.emplace_back(std::move(col));
2020-01-24 18:13:17 +00:00
2020-01-24 18:33:47 +00:00
fin_columns[0] = std::move(col_database);
fin_columns[1] = std::move(col_table);
fin_columns[2] = std::move(col_engine);
2020-01-24 18:13:17 +00:00
2020-01-24 18:33:47 +00:00
UInt64 num_rows = fin_columns.at(0)->size();
Chunk chunk(std::move(fin_columns), num_rows);
2020-08-06 12:24:05 +00:00
return Pipe(std::make_shared<SourceFromSingleChunk>(metadata_snapshot->getSampleBlock(), std::move(chunk)));
}
}