2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnString.h>
|
|
|
|
#include <DataTypes/DataTypeString.h>
|
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
|
|
|
#include <DataTypes/DataTypeDateTime.h>
|
|
|
|
#include <Storages/System/StorageSystemReplicas.h>
|
|
|
|
#include <Storages/StorageReplicatedMergeTree.h>
|
2017-11-20 05:22:54 +00:00
|
|
|
#include <Storages/VirtualColumnUtils.h>
|
2020-03-07 17:37:38 +00:00
|
|
|
#include <Access/ContextAccess.h>
|
2017-07-13 20:58:19 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Databases/IDatabase.h>
|
2020-01-24 18:13:17 +00:00
|
|
|
#include <Processors/Sources/SourceFromSingleChunk.h>
|
2014-10-06 05:18:17 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
|
2020-08-12 20:40:13 +00:00
|
|
|
StorageSystemReplicas::StorageSystemReplicas(const StorageID & table_id_)
|
|
|
|
: IStorage(table_id_)
|
2018-01-25 14:42:39 +00:00
|
|
|
{
|
2020-06-19 15:39:41 +00:00
|
|
|
StorageInMemoryMetadata storage_metadata;
|
|
|
|
storage_metadata.setColumns(ColumnsDescription({
|
2017-04-17 15:06:12 +00:00
|
|
|
{ "database", std::make_shared<DataTypeString>() },
|
|
|
|
{ "table", std::make_shared<DataTypeString>() },
|
|
|
|
{ "engine", std::make_shared<DataTypeString>() },
|
|
|
|
{ "is_leader", std::make_shared<DataTypeUInt8>() },
|
2019-02-13 13:05:58 +00:00
|
|
|
{ "can_become_leader", std::make_shared<DataTypeUInt8>() },
|
2017-04-17 15:06:12 +00:00
|
|
|
{ "is_readonly", std::make_shared<DataTypeUInt8>() },
|
|
|
|
{ "is_session_expired", std::make_shared<DataTypeUInt8>() },
|
|
|
|
{ "future_parts", std::make_shared<DataTypeUInt32>() },
|
|
|
|
{ "parts_to_check", std::make_shared<DataTypeUInt32>() },
|
|
|
|
{ "zookeeper_path", std::make_shared<DataTypeString>() },
|
|
|
|
{ "replica_name", std::make_shared<DataTypeString>() },
|
|
|
|
{ "replica_path", std::make_shared<DataTypeString>() },
|
|
|
|
{ "columns_version", std::make_shared<DataTypeInt32>() },
|
|
|
|
{ "queue_size", std::make_shared<DataTypeUInt32>() },
|
|
|
|
{ "inserts_in_queue", std::make_shared<DataTypeUInt32>() },
|
|
|
|
{ "merges_in_queue", std::make_shared<DataTypeUInt32>() },
|
2018-06-06 13:22:30 +00:00
|
|
|
{ "part_mutations_in_queue", std::make_shared<DataTypeUInt32>() },
|
2017-04-17 15:06:12 +00:00
|
|
|
{ "queue_oldest_time", std::make_shared<DataTypeDateTime>() },
|
|
|
|
{ "inserts_oldest_time", std::make_shared<DataTypeDateTime>() },
|
|
|
|
{ "merges_oldest_time", std::make_shared<DataTypeDateTime>() },
|
2018-06-06 13:22:30 +00:00
|
|
|
{ "part_mutations_oldest_time", std::make_shared<DataTypeDateTime>() },
|
2017-04-17 15:06:12 +00:00
|
|
|
{ "oldest_part_to_get", std::make_shared<DataTypeString>() },
|
|
|
|
{ "oldest_part_to_merge_to", std::make_shared<DataTypeString>() },
|
2018-06-06 13:22:30 +00:00
|
|
|
{ "oldest_part_to_mutate_to", std::make_shared<DataTypeString>() },
|
2017-04-17 15:06:12 +00:00
|
|
|
{ "log_max_index", std::make_shared<DataTypeUInt64>() },
|
|
|
|
{ "log_pointer", std::make_shared<DataTypeUInt64>() },
|
|
|
|
{ "last_queue_update", std::make_shared<DataTypeDateTime>() },
|
|
|
|
{ "absolute_delay", std::make_shared<DataTypeUInt64>() },
|
|
|
|
{ "total_replicas", std::make_shared<DataTypeUInt8>() },
|
|
|
|
{ "active_replicas", std::make_shared<DataTypeUInt8>() },
|
2020-02-15 00:11:09 +00:00
|
|
|
{ "zookeeper_exception", std::make_shared<DataTypeString>() },
|
2018-03-13 14:18:11 +00:00
|
|
|
}));
|
2020-06-19 15:39:41 +00:00
|
|
|
setInMemoryMetadata(storage_metadata);
|
2014-10-06 05:18:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-08-06 12:24:05 +00:00
|
|
|
Pipe StorageSystemReplicas::read(
|
2017-04-01 07:20:54 +00:00
|
|
|
const Names & column_names,
|
2020-06-16 15:51:29 +00:00
|
|
|
const StorageMetadataPtr & metadata_snapshot,
|
2020-09-20 17:52:17 +00:00
|
|
|
SelectQueryInfo & query_info,
|
2021-04-10 23:33:54 +00:00
|
|
|
ContextPtr context,
|
2018-09-08 11:29:23 +00:00
|
|
|
QueryProcessingStage::Enum /*processed_stage*/,
|
2017-12-02 02:47:12 +00:00
|
|
|
const size_t /*max_block_size*/,
|
|
|
|
const unsigned /*num_streams*/)
|
2014-10-06 05:18:17 +00:00
|
|
|
{
|
2020-06-19 17:17:13 +00:00
|
|
|
metadata_snapshot->check(column_names, getVirtuals(), getStorageID());
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
const auto access = context->getAccess();
|
2020-03-07 17:37:38 +00:00
|
|
|
const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
|
2020-01-24 16:20:36 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// We collect a set of replicated tables.
|
|
|
|
std::map<String, std::map<String, StoragePtr>> replicated_tables;
|
2020-02-10 13:10:17 +00:00
|
|
|
for (const auto & db : DatabaseCatalog::instance().getDatabases())
|
2018-08-13 09:11:58 +00:00
|
|
|
{
|
2020-10-15 15:57:17 +00:00
|
|
|
/// Check if database can contain replicated tables
|
2020-10-27 11:29:10 +00:00
|
|
|
if (!db.second->canContainMergeTreeTables())
|
2019-10-01 12:44:17 +00:00
|
|
|
continue;
|
2020-03-07 17:37:38 +00:00
|
|
|
const bool check_access_for_tables = check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, db.first);
|
2020-05-28 20:10:45 +00:00
|
|
|
for (auto iterator = db.second->getTablesIterator(context); iterator->isValid(); iterator->next())
|
2018-08-13 09:11:58 +00:00
|
|
|
{
|
2020-06-02 02:06:16 +00:00
|
|
|
const auto & table = iterator->table();
|
|
|
|
if (!table)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
if (!dynamic_cast<const StorageReplicatedMergeTree *>(table.get()))
|
2020-01-24 16:20:36 +00:00
|
|
|
continue;
|
2020-03-07 17:37:38 +00:00
|
|
|
if (check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, db.first, iterator->name()))
|
2020-01-24 16:20:36 +00:00
|
|
|
continue;
|
2020-06-02 02:06:16 +00:00
|
|
|
replicated_tables[db.first][iterator->name()] = table;
|
2018-08-13 09:11:58 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-27 16:31:45 +00:00
|
|
|
/// Do you need columns that require a ZooKeeper request to compute.
|
2017-04-01 07:20:54 +00:00
|
|
|
bool with_zk_fields = false;
|
2018-08-27 16:31:45 +00:00
|
|
|
for (const auto & column_name : column_names)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2018-08-27 16:31:45 +00:00
|
|
|
if ( column_name == "log_max_index"
|
|
|
|
|| column_name == "log_pointer"
|
|
|
|
|| column_name == "total_replicas"
|
2020-02-15 00:11:09 +00:00
|
|
|
|| column_name == "active_replicas"
|
|
|
|
|| column_name == "zookeeper_exception")
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
with_zk_fields = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-16 00:49:03 +00:00
|
|
|
MutableColumnPtr col_database_mut = ColumnString::create();
|
|
|
|
MutableColumnPtr col_table_mut = ColumnString::create();
|
|
|
|
MutableColumnPtr col_engine_mut = ColumnString::create();
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
for (auto & db : replicated_tables)
|
|
|
|
{
|
|
|
|
for (auto & table : db.second)
|
|
|
|
{
|
2017-12-16 00:49:03 +00:00
|
|
|
col_database_mut->insert(db.first);
|
|
|
|
col_table_mut->insert(table.first);
|
|
|
|
col_engine_mut->insert(table.second->getName());
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-12-16 00:49:03 +00:00
|
|
|
ColumnPtr col_database = std::move(col_database_mut);
|
|
|
|
ColumnPtr col_table = std::move(col_table_mut);
|
|
|
|
ColumnPtr col_engine = std::move(col_engine_mut);
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Determine what tables are needed by the conditions in the query.
|
|
|
|
{
|
2017-12-16 00:49:03 +00:00
|
|
|
Block filtered_block
|
|
|
|
{
|
|
|
|
{ col_database, std::make_shared<DataTypeString>(), "database" },
|
|
|
|
{ col_table, std::make_shared<DataTypeString>(), "table" },
|
|
|
|
{ col_engine, std::make_shared<DataTypeString>(), "engine" },
|
|
|
|
};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-07-15 03:48:36 +00:00
|
|
|
VirtualColumnUtils::filterBlockWithQuery(query_info.query, filtered_block, context);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
if (!filtered_block.rows())
|
2020-08-06 12:24:05 +00:00
|
|
|
return {};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-12-16 00:49:03 +00:00
|
|
|
col_database = filtered_block.getByName("database").column;
|
|
|
|
col_table = filtered_block.getByName("table").column;
|
|
|
|
col_engine = filtered_block.getByName("engine").column;
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-06-16 15:51:29 +00:00
|
|
|
MutableColumns res_columns = metadata_snapshot->getSampleBlock().cloneEmptyColumns();
|
2017-12-16 00:49:03 +00:00
|
|
|
|
|
|
|
for (size_t i = 0, size = col_database->size(); i < size; ++i)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
StorageReplicatedMergeTree::Status status;
|
2017-11-04 16:46:14 +00:00
|
|
|
dynamic_cast<StorageReplicatedMergeTree &>(
|
2017-04-01 07:20:54 +00:00
|
|
|
*replicated_tables
|
2017-12-16 00:49:03 +00:00
|
|
|
[(*col_database)[i].safeGet<const String &>()]
|
|
|
|
[(*col_table)[i].safeGet<const String &>()]).getStatus(status, with_zk_fields);
|
|
|
|
|
|
|
|
size_t col_num = 3;
|
2018-10-22 08:54:54 +00:00
|
|
|
res_columns[col_num++]->insert(status.is_leader);
|
2019-02-13 13:05:58 +00:00
|
|
|
res_columns[col_num++]->insert(status.can_become_leader);
|
2018-10-22 08:54:54 +00:00
|
|
|
res_columns[col_num++]->insert(status.is_readonly);
|
|
|
|
res_columns[col_num++]->insert(status.is_session_expired);
|
|
|
|
res_columns[col_num++]->insert(status.queue.future_parts);
|
|
|
|
res_columns[col_num++]->insert(status.parts_to_check);
|
2017-12-16 00:49:03 +00:00
|
|
|
res_columns[col_num++]->insert(status.zookeeper_path);
|
|
|
|
res_columns[col_num++]->insert(status.replica_name);
|
|
|
|
res_columns[col_num++]->insert(status.replica_path);
|
2018-10-22 08:54:54 +00:00
|
|
|
res_columns[col_num++]->insert(status.columns_version);
|
|
|
|
res_columns[col_num++]->insert(status.queue.queue_size);
|
|
|
|
res_columns[col_num++]->insert(status.queue.inserts_in_queue);
|
|
|
|
res_columns[col_num++]->insert(status.queue.merges_in_queue);
|
|
|
|
res_columns[col_num++]->insert(status.queue.part_mutations_in_queue);
|
|
|
|
res_columns[col_num++]->insert(status.queue.queue_oldest_time);
|
|
|
|
res_columns[col_num++]->insert(status.queue.inserts_oldest_time);
|
|
|
|
res_columns[col_num++]->insert(status.queue.merges_oldest_time);
|
|
|
|
res_columns[col_num++]->insert(status.queue.part_mutations_oldest_time);
|
2017-12-16 00:49:03 +00:00
|
|
|
res_columns[col_num++]->insert(status.queue.oldest_part_to_get);
|
|
|
|
res_columns[col_num++]->insert(status.queue.oldest_part_to_merge_to);
|
2018-06-06 13:22:30 +00:00
|
|
|
res_columns[col_num++]->insert(status.queue.oldest_part_to_mutate_to);
|
2017-12-16 00:49:03 +00:00
|
|
|
res_columns[col_num++]->insert(status.log_max_index);
|
|
|
|
res_columns[col_num++]->insert(status.log_pointer);
|
2018-10-22 08:54:54 +00:00
|
|
|
res_columns[col_num++]->insert(status.queue.last_queue_update);
|
|
|
|
res_columns[col_num++]->insert(status.absolute_delay);
|
|
|
|
res_columns[col_num++]->insert(status.total_replicas);
|
|
|
|
res_columns[col_num++]->insert(status.active_replicas);
|
2020-02-15 00:11:09 +00:00
|
|
|
res_columns[col_num++]->insert(status.zookeeper_exception);
|
2017-12-16 00:49:03 +00:00
|
|
|
}
|
|
|
|
|
2020-06-16 15:51:29 +00:00
|
|
|
Block header = metadata_snapshot->getSampleBlock();
|
2020-01-24 18:13:17 +00:00
|
|
|
|
2020-01-24 18:33:47 +00:00
|
|
|
Columns fin_columns;
|
|
|
|
fin_columns.reserve(res_columns.size());
|
2020-01-24 18:13:17 +00:00
|
|
|
|
|
|
|
for (auto & col : res_columns)
|
2020-01-24 18:33:47 +00:00
|
|
|
fin_columns.emplace_back(std::move(col));
|
2020-01-24 18:13:17 +00:00
|
|
|
|
2020-01-24 18:33:47 +00:00
|
|
|
fin_columns[0] = std::move(col_database);
|
|
|
|
fin_columns[1] = std::move(col_table);
|
|
|
|
fin_columns[2] = std::move(col_engine);
|
2020-01-24 18:13:17 +00:00
|
|
|
|
2020-01-24 18:33:47 +00:00
|
|
|
UInt64 num_rows = fin_columns.at(0)->size();
|
|
|
|
Chunk chunk(std::move(fin_columns), num_rows);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-08-06 12:24:05 +00:00
|
|
|
return Pipe(std::make_shared<SourceFromSingleChunk>(metadata_snapshot->getSampleBlock(), std::move(chunk)));
|
2014-10-06 05:18:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
}
|