2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnString.h>
|
|
|
|
#include <Columns/ColumnsNumber.h>
|
|
|
|
#include <DataTypes/DataTypeString.h>
|
|
|
|
#include <DataTypes/DataTypeDateTime.h>
|
|
|
|
#include <DataStreams/OneBlockInputStream.h>
|
|
|
|
#include <Storages/System/StorageSystemTables.h>
|
2017-11-20 05:22:54 +00:00
|
|
|
#include <Storages/VirtualColumnUtils.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Databases/IDatabase.h>
|
|
|
|
#include <Interpreters/Context.h>
|
2018-01-30 17:47:04 +00:00
|
|
|
#include <Parsers/ASTCreateQuery.h>
|
|
|
|
#include <Parsers/queryToString.h>
|
|
|
|
#include <Common/typeid_cast.h>
|
2018-06-05 19:46:49 +00:00
|
|
|
#include <Common/StringUtils/StringUtils.h>
|
2018-01-25 17:50:57 +00:00
|
|
|
#include <DataTypes/DataTypesNumber.h>
|
2018-08-10 10:15:59 +00:00
|
|
|
#include <DataTypes/DataTypeArray.h>
|
2012-05-08 11:19:00 +00:00
|
|
|
|
|
|
|
|
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2018-01-30 17:47:04 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int CANNOT_GET_CREATE_TABLE_QUERY;
|
2019-03-28 23:17:22 +00:00
|
|
|
extern const int TABLE_IS_DROPPED;
|
2018-01-30 17:47:04 +00:00
|
|
|
}
|
|
|
|
|
2018-02-08 14:14:08 +00:00
|
|
|
|
2015-01-21 04:30:33 +00:00
|
|
|
StorageSystemTables::StorageSystemTables(const std::string & name_)
|
2018-01-25 14:42:39 +00:00
|
|
|
: name(name_)
|
|
|
|
{
|
2018-03-13 14:18:11 +00:00
|
|
|
setColumns(ColumnsDescription(
|
2018-02-08 21:15:46 +00:00
|
|
|
{
|
2017-10-11 14:41:49 +00:00
|
|
|
{"database", std::make_shared<DataTypeString>()},
|
|
|
|
{"name", std::make_shared<DataTypeString>()},
|
2018-02-08 21:15:46 +00:00
|
|
|
{"engine", std::make_shared<DataTypeString>()},
|
|
|
|
{"is_temporary", std::make_shared<DataTypeUInt8>()},
|
2018-02-21 19:26:59 +00:00
|
|
|
{"data_path", std::make_shared<DataTypeString>()},
|
|
|
|
{"metadata_path", std::make_shared<DataTypeString>()},
|
2018-07-07 18:58:48 +00:00
|
|
|
{"metadata_modification_time", std::make_shared<DataTypeDateTime>()},
|
2018-08-12 02:55:01 +00:00
|
|
|
{"dependencies_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
|
|
|
|
{"dependencies_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>())},
|
2018-07-07 18:58:48 +00:00
|
|
|
{"create_table_query", std::make_shared<DataTypeString>()},
|
2018-11-08 17:45:41 +00:00
|
|
|
{"engine_full", std::make_shared<DataTypeString>()},
|
|
|
|
{"partition_key", std::make_shared<DataTypeString>()},
|
2018-11-27 17:07:10 +00:00
|
|
|
{"sorting_key", std::make_shared<DataTypeString>()},
|
|
|
|
{"primary_key", std::make_shared<DataTypeString>()},
|
|
|
|
{"sampling_key", std::make_shared<DataTypeString>()},
|
2018-03-13 14:18:11 +00:00
|
|
|
}));
|
2012-05-08 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
2015-01-21 04:30:33 +00:00
|
|
|
|
2017-12-16 00:49:03 +00:00
|
|
|
static ColumnPtr getFilteredDatabases(const ASTPtr & query, const Context & context)
|
2015-01-21 04:30:33 +00:00
|
|
|
{
|
2017-12-16 00:49:03 +00:00
|
|
|
MutableColumnPtr column = ColumnString::create();
|
2017-04-01 07:20:54 +00:00
|
|
|
for (const auto & db : context.getDatabases())
|
2017-12-16 00:49:03 +00:00
|
|
|
column->insert(db.first);
|
2016-08-04 23:35:07 +00:00
|
|
|
|
2018-11-24 01:48:06 +00:00
|
|
|
Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
|
2017-04-01 07:20:54 +00:00
|
|
|
VirtualColumnUtils::filterBlockWithQuery(query, block, context);
|
2017-12-16 00:49:03 +00:00
|
|
|
return block.getByPosition(0).column;
|
2013-02-06 11:26:35 +00:00
|
|
|
}
|
|
|
|
|
2012-05-08 11:19:00 +00:00
|
|
|
|
2019-01-23 14:48:50 +00:00
|
|
|
class TablesBlockInputStream : public IBlockInputStream
|
2012-05-08 11:19:00 +00:00
|
|
|
{
|
2018-08-25 16:07:48 +00:00
|
|
|
public:
|
|
|
|
TablesBlockInputStream(
|
|
|
|
std::vector<UInt8> columns_mask,
|
|
|
|
Block header,
|
2019-02-10 16:55:12 +00:00
|
|
|
UInt64 max_block_size,
|
2018-08-25 16:07:48 +00:00
|
|
|
ColumnPtr databases,
|
|
|
|
const Context & context)
|
2018-11-08 17:45:41 +00:00
|
|
|
: columns_mask(std::move(columns_mask)), header(std::move(header)), max_block_size(max_block_size), databases(std::move(databases)), context(context) {}
|
2018-08-25 16:07:48 +00:00
|
|
|
|
|
|
|
String getName() const override { return "Tables"; }
|
|
|
|
Block getHeader() const override { return header; }
|
|
|
|
|
|
|
|
protected:
|
|
|
|
Block readImpl() override
|
|
|
|
{
|
|
|
|
if (done)
|
|
|
|
return {};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
Block res = header;
|
|
|
|
MutableColumns res_columns = header.cloneEmptyColumns();
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
size_t rows_count = 0;
|
|
|
|
while (rows_count < max_block_size)
|
|
|
|
{
|
|
|
|
if (tables_it && !tables_it->isValid())
|
|
|
|
++database_idx;
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
|
|
|
|
{
|
|
|
|
database_name = databases->getDataAt(database_idx).toString();
|
|
|
|
database = context.tryGetDatabase(database_name);
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (!database || !context.hasDatabaseAccessRights(database_name))
|
|
|
|
{
|
|
|
|
/// Database was deleted just now or the user has no access.
|
|
|
|
++database_idx;
|
|
|
|
continue;
|
|
|
|
}
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
break;
|
|
|
|
}
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-11-26 00:56:50 +00:00
|
|
|
/// This is for temporary tables. They are output in single block regardless to max_block_size.
|
2018-08-25 16:07:48 +00:00
|
|
|
if (database_idx >= databases->size())
|
|
|
|
{
|
|
|
|
if (context.hasSessionContext())
|
|
|
|
{
|
|
|
|
Tables external_tables = context.getSessionContext().getExternalTables();
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
for (auto table : external_tables)
|
|
|
|
{
|
|
|
|
size_t src_index = 0;
|
|
|
|
size_t res_index = 0;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(table.first);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(table.second->getName());
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
2018-10-22 08:54:54 +00:00
|
|
|
res_columns[res_index++]->insert(1u);
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(table.second->getName());
|
2018-11-08 17:45:41 +00:00
|
|
|
|
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
|
|
|
|
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
|
|
|
|
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-11-11 15:44:21 +00:00
|
|
|
|
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insertDefault();
|
2018-08-25 16:07:48 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
res.setColumns(std::move(res_columns));
|
|
|
|
done = true;
|
|
|
|
return res;
|
|
|
|
}
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (!tables_it || !tables_it->isValid())
|
|
|
|
tables_it = database->getIterator(context);
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
|
2018-08-10 10:15:59 +00:00
|
|
|
{
|
2018-08-25 16:07:48 +00:00
|
|
|
auto table_name = tables_it->name();
|
2019-03-28 19:13:59 +00:00
|
|
|
const StoragePtr & table = tables_it->table();
|
2019-03-28 23:17:22 +00:00
|
|
|
|
|
|
|
TableStructureReadLockHolder lock;
|
|
|
|
|
|
|
|
try
|
|
|
|
{
|
|
|
|
lock = table->lockStructureForShare(false, context.getCurrentQueryId());
|
|
|
|
}
|
|
|
|
catch (const Exception & e)
|
|
|
|
{
|
|
|
|
if (e.code() == ErrorCodes::TABLE_IS_DROPPED)
|
|
|
|
continue;
|
|
|
|
throw;
|
|
|
|
}
|
2019-02-08 14:10:26 +00:00
|
|
|
|
|
|
|
++rows_count;
|
2018-08-10 10:15:59 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
size_t src_index = 0;
|
|
|
|
size_t res_index = 0;
|
2018-08-12 02:55:01 +00:00
|
|
|
|
|
|
|
if (columns_mask[src_index++])
|
2018-08-25 16:07:48 +00:00
|
|
|
res_columns[res_index++]->insert(database_name);
|
2018-08-10 10:15:59 +00:00
|
|
|
|
2018-08-12 02:55:01 +00:00
|
|
|
if (columns_mask[src_index++])
|
2018-08-25 16:07:48 +00:00
|
|
|
res_columns[res_index++]->insert(table_name);
|
2018-08-10 10:15:59 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
2019-03-28 19:13:59 +00:00
|
|
|
res_columns[res_index++]->insert(table->getName());
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-07-07 18:58:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
2018-11-08 17:45:41 +00:00
|
|
|
res_columns[res_index++]->insert(0u); // is_temporary
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2019-04-05 17:37:27 +00:00
|
|
|
if (columns_mask[src_index++])
|
2019-04-05 20:07:24 +00:00
|
|
|
for (const String & path : table->getDataPaths())
|
2019-04-05 12:30:26 +00:00
|
|
|
res_columns[res_index++]->insert(path);
|
2018-08-25 16:07:48 +00:00
|
|
|
|
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(database->getTableMetadataPath(table_name));
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(static_cast<UInt64>(database->getTableMetadataModificationTime(context, table_name)));
|
|
|
|
|
|
|
|
{
|
|
|
|
Array dependencies_table_name_array;
|
|
|
|
Array dependencies_database_name_array;
|
|
|
|
if (columns_mask[src_index] || columns_mask[src_index + 1])
|
2018-01-30 17:47:04 +00:00
|
|
|
{
|
2018-08-25 16:07:48 +00:00
|
|
|
const auto dependencies = context.getDependencies(database_name, table_name);
|
2018-01-30 17:47:04 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
dependencies_table_name_array.reserve(dependencies.size());
|
|
|
|
dependencies_database_name_array.reserve(dependencies.size());
|
|
|
|
for (const auto & dependency : dependencies)
|
|
|
|
{
|
|
|
|
dependencies_table_name_array.push_back(dependency.second);
|
|
|
|
dependencies_database_name_array.push_back(dependency.first);
|
2018-01-30 17:47:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(dependencies_database_name_array);
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(dependencies_table_name_array);
|
|
|
|
}
|
2018-02-02 13:17:45 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index] || columns_mask[src_index + 1])
|
|
|
|
{
|
|
|
|
ASTPtr ast = database->tryGetCreateTableQuery(context, table_name);
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
res_columns[res_index++]->insert(ast ? queryToString(ast) : "");
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
{
|
|
|
|
String engine_full;
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
if (ast)
|
|
|
|
{
|
2019-03-15 17:09:14 +00:00
|
|
|
const auto & ast_create = ast->as<ASTCreateQuery &>();
|
|
|
|
if (ast_create.storage)
|
2018-08-25 16:07:48 +00:00
|
|
|
{
|
2019-03-15 17:09:14 +00:00
|
|
|
engine_full = queryToString(*ast_create.storage);
|
2018-08-25 16:07:48 +00:00
|
|
|
|
|
|
|
static const char * const extra_head = " ENGINE = ";
|
|
|
|
if (startsWith(engine_full, extra_head))
|
|
|
|
engine_full = engine_full.substr(strlen(extra_head));
|
|
|
|
}
|
|
|
|
}
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
res_columns[res_index++]->insert(engine_full);
|
|
|
|
}
|
|
|
|
}
|
2018-11-08 17:45:41 +00:00
|
|
|
else
|
2018-11-19 14:31:16 +00:00
|
|
|
src_index += 2;
|
2018-11-08 17:45:41 +00:00
|
|
|
|
|
|
|
ASTPtr expression_ptr;
|
|
|
|
if (columns_mask[src_index++])
|
|
|
|
{
|
2019-02-08 14:10:26 +00:00
|
|
|
if ((expression_ptr = table->getPartitionKeyAST()))
|
2018-11-08 17:45:41 +00:00
|
|
|
res_columns[res_index++]->insert(queryToString(expression_ptr));
|
|
|
|
else
|
|
|
|
res_columns[res_index++]->insertDefault();
|
|
|
|
}
|
|
|
|
|
2018-11-11 15:44:21 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
{
|
2019-02-08 14:10:26 +00:00
|
|
|
if ((expression_ptr = table->getSortingKeyAST()))
|
2018-11-11 15:44:21 +00:00
|
|
|
res_columns[res_index++]->insert(queryToString(expression_ptr));
|
|
|
|
else
|
|
|
|
res_columns[res_index++]->insertDefault();
|
|
|
|
}
|
|
|
|
|
2018-11-08 17:45:41 +00:00
|
|
|
if (columns_mask[src_index++])
|
|
|
|
{
|
2019-02-08 14:10:26 +00:00
|
|
|
if ((expression_ptr = table->getPrimaryKeyAST()))
|
2018-11-08 17:45:41 +00:00
|
|
|
res_columns[res_index++]->insert(queryToString(expression_ptr));
|
|
|
|
else
|
|
|
|
res_columns[res_index++]->insertDefault();
|
|
|
|
}
|
|
|
|
|
|
|
|
if (columns_mask[src_index++])
|
|
|
|
{
|
2019-02-08 14:10:26 +00:00
|
|
|
if ((expression_ptr = table->getSamplingKeyAST()))
|
2018-11-08 17:45:41 +00:00
|
|
|
res_columns[res_index++]->insert(queryToString(expression_ptr));
|
|
|
|
else
|
|
|
|
res_columns[res_index++]->insertDefault();
|
|
|
|
}
|
2018-08-25 16:07:48 +00:00
|
|
|
}
|
|
|
|
}
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
res.setColumns(std::move(res_columns));
|
|
|
|
return res;
|
|
|
|
}
|
|
|
|
private:
|
|
|
|
std::vector<UInt8> columns_mask;
|
|
|
|
Block header;
|
2019-02-10 16:55:12 +00:00
|
|
|
UInt64 max_block_size;
|
2018-08-25 16:07:48 +00:00
|
|
|
ColumnPtr databases;
|
|
|
|
size_t database_idx = 0;
|
|
|
|
DatabaseIteratorPtr tables_it;
|
|
|
|
const Context context;
|
|
|
|
bool done = false;
|
|
|
|
DatabasePtr database;
|
|
|
|
std::string database_name;
|
|
|
|
};
|
2018-07-07 18:58:48 +00:00
|
|
|
|
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
BlockInputStreams StorageSystemTables::read(
|
|
|
|
const Names & column_names,
|
|
|
|
const SelectQueryInfo & query_info,
|
|
|
|
const Context & context,
|
2018-09-08 11:29:23 +00:00
|
|
|
QueryProcessingStage::Enum /*processed_stage*/,
|
2019-02-18 23:38:44 +00:00
|
|
|
const size_t max_block_size,
|
2018-08-25 16:07:48 +00:00
|
|
|
const unsigned /*num_streams*/)
|
|
|
|
{
|
|
|
|
check(column_names);
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
/// Create a mask of what columns are needed in the result.
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
NameSet names_set(column_names.begin(), column_names.end());
|
2018-08-10 10:15:59 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
Block sample_block = getSampleBlock();
|
|
|
|
Block res_block;
|
2018-07-07 18:58:48 +00:00
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
std::vector<UInt8> columns_mask(sample_block.columns());
|
|
|
|
for (size_t i = 0, size = columns_mask.size(); i < size; ++i)
|
|
|
|
{
|
|
|
|
if (names_set.count(sample_block.getByPosition(i).name))
|
|
|
|
{
|
|
|
|
columns_mask[i] = 1;
|
|
|
|
res_block.insert(sample_block.getByPosition(i));
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-08-25 16:07:48 +00:00
|
|
|
ColumnPtr filtered_databases_column = getFilteredDatabases(query_info.query, context);
|
|
|
|
return {std::make_shared<TablesBlockInputStream>(
|
|
|
|
std::move(columns_mask), std::move(res_block), max_block_size, std::move(filtered_databases_column), context)};
|
2018-01-30 17:47:04 +00:00
|
|
|
}
|
|
|
|
|
2012-05-08 11:19:00 +00:00
|
|
|
}
|