2020-01-03 01:48:15 +00:00
|
|
|
/* Copyright (c) 2018 BlackBerry Limited
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License. */
|
|
|
|
#include <Parsers/ASTSelectQuery.h>
|
|
|
|
#include <Parsers/ASTCreateQuery.h>
|
|
|
|
#include <Parsers/ASTWatchQuery.h>
|
|
|
|
#include <Parsers/ASTLiteral.h>
|
|
|
|
#include <Interpreters/Context.h>
|
2019-08-22 22:41:30 +00:00
|
|
|
#include <Interpreters/InterpreterSelectQuery.h>
|
|
|
|
#include <DataStreams/IBlockOutputStream.h>
|
2020-06-01 16:31:06 +00:00
|
|
|
#include <DataStreams/BlocksSource.h>
|
2019-08-22 22:41:30 +00:00
|
|
|
#include <DataStreams/MaterializingBlockInputStream.h>
|
|
|
|
#include <DataStreams/SquashingBlockInputStream.h>
|
|
|
|
#include <DataStreams/copyData.h>
|
2020-09-14 19:39:33 +00:00
|
|
|
#include <common/logger_useful.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2019-08-22 23:22:57 +00:00
|
|
|
#include <Common/SipHash.h>
|
2021-01-27 00:54:57 +00:00
|
|
|
#include <Common/hex.h>
|
2019-08-22 23:22:57 +00:00
|
|
|
|
|
|
|
#include <Storages/LiveView/StorageLiveView.h>
|
|
|
|
#include <Storages/LiveView/LiveViewBlockInputStream.h>
|
|
|
|
#include <Storages/LiveView/LiveViewBlockOutputStream.h>
|
|
|
|
#include <Storages/LiveView/LiveViewEventsBlockInputStream.h>
|
2019-12-04 23:29:37 +00:00
|
|
|
#include <Storages/LiveView/StorageBlocks.h>
|
2020-09-07 23:08:17 +00:00
|
|
|
#include <Storages/LiveView/TemporaryLiveViewCleaner.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
#include <Storages/StorageFactory.h>
|
|
|
|
#include <Parsers/ASTTablesInSelectQuery.h>
|
|
|
|
#include <Parsers/ASTSubquery.h>
|
2019-12-05 21:42:00 +00:00
|
|
|
#include <Parsers/queryToString.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
#include <Interpreters/DatabaseAndTableWithAlias.h>
|
2020-12-10 23:56:57 +00:00
|
|
|
#include <Interpreters/DatabaseCatalog.h>
|
2019-12-12 15:28:24 +00:00
|
|
|
#include <Interpreters/getTableExpressions.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
#include <Interpreters/AddDefaultDatabaseVisitor.h>
|
2020-01-26 09:49:53 +00:00
|
|
|
#include <Access/AccessFlags.h>
|
2020-02-17 16:35:23 +00:00
|
|
|
#include <Processors/Sources/SourceFromSingleChunk.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2019-12-05 11:53:36 +00:00
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int INCORRECT_QUERY;
|
|
|
|
extern const int TABLE_WAS_NOT_DROPPED;
|
|
|
|
extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW;
|
2019-08-22 23:46:40 +00:00
|
|
|
extern const int SUPPORT_IS_DISABLED;
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2019-08-22 22:41:30 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
static StorageID extractDependentTable(ASTPtr & query, ContextPtr context, const String & table_name, ASTPtr & inner_subquery)
|
2019-12-04 23:29:37 +00:00
|
|
|
{
|
2019-12-05 11:53:36 +00:00
|
|
|
ASTSelectQuery & select_query = typeid_cast<ASTSelectQuery &>(*query);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-01-09 14:33:58 +00:00
|
|
|
if (auto db_and_table = getDatabaseAndTable(select_query, 0))
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
String select_database_name = context->getCurrentDatabase();
|
2020-01-09 14:33:58 +00:00
|
|
|
String select_table_name = db_and_table->table;
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
if (db_and_table->database.empty())
|
|
|
|
{
|
|
|
|
db_and_table->database = select_database_name;
|
|
|
|
AddDefaultDatabaseVisitor visitor(select_database_name);
|
2019-12-04 23:29:37 +00:00
|
|
|
visitor.visit(select_query);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
select_database_name = db_and_table->database;
|
2019-12-05 21:42:00 +00:00
|
|
|
|
2020-01-01 16:44:45 +00:00
|
|
|
select_query.replaceDatabaseAndTable("", table_name + "_blocks");
|
2020-01-09 14:33:58 +00:00
|
|
|
return StorageID(select_database_name, select_table_name);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2020-01-09 14:33:58 +00:00
|
|
|
else if (auto subquery = extractTableExpression(select_query, 0))
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2020-01-09 14:33:58 +00:00
|
|
|
auto * ast_select = subquery->as<ASTSelectWithUnionQuery>();
|
|
|
|
if (!ast_select)
|
2021-06-27 12:40:07 +00:00
|
|
|
throw Exception("LIVE VIEWs are only supported for queries from tables, but there is no table name in select query.",
|
2021-06-27 12:38:08 +00:00
|
|
|
DB::ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW);
|
2019-05-26 22:03:30 +00:00
|
|
|
if (ast_select->list_of_selects->children.size() != 1)
|
|
|
|
throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW);
|
|
|
|
|
2020-01-01 16:44:45 +00:00
|
|
|
inner_subquery = ast_select->list_of_selects->children.at(0)->clone();
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-01-09 14:33:58 +00:00
|
|
|
return extractDependentTable(ast_select->list_of_selects->children.at(0), context, table_name, inner_subquery);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
else
|
2020-01-09 14:33:58 +00:00
|
|
|
{
|
|
|
|
/// If the table is not specified - use the table `system.one`
|
|
|
|
return StorageID("system", "one");
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_context)
|
2020-01-02 17:24:55 +00:00
|
|
|
{
|
2020-01-02 19:31:04 +00:00
|
|
|
ASTPtr mergeable_query = inner_query;
|
|
|
|
|
|
|
|
if (inner_subquery)
|
|
|
|
mergeable_query = inner_subquery;
|
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
MergeableBlocksPtr new_mergeable_blocks = std::make_shared<MergeableBlocks>();
|
|
|
|
BlocksPtrs new_blocks = std::make_shared<std::vector<BlocksPtr>>();
|
|
|
|
BlocksPtr base_blocks = std::make_shared<Blocks>();
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
InterpreterSelectQuery interpreter(mergeable_query->clone(), local_context, SelectQueryOptions(QueryProcessingStage::WithMergeableState), Names());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-05-28 14:11:36 +00:00
|
|
|
auto view_mergeable_stream = std::make_shared<MaterializingBlockInputStream>(interpreter.execute().getInputStream());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
|
|
|
while (Block this_block = view_mergeable_stream->read())
|
2020-01-03 01:48:15 +00:00
|
|
|
base_blocks->push_back(this_block);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
new_blocks->push_back(base_blocks);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
new_mergeable_blocks->blocks = new_blocks;
|
|
|
|
new_mergeable_blocks->sample_block = view_mergeable_stream->getHeader();
|
2020-01-02 19:31:04 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
return new_mergeable_blocks;
|
2020-01-02 17:24:55 +00:00
|
|
|
}
|
|
|
|
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes StorageLiveView::blocksToPipes(BlocksPtrs blocks, Block & sample_block)
|
2020-01-02 17:24:55 +00:00
|
|
|
{
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes pipes;
|
2020-03-23 02:12:31 +00:00
|
|
|
for (auto & blocks_for_source : *blocks)
|
|
|
|
pipes.emplace_back(std::make_shared<BlocksSource>(std::make_shared<BlocksPtr>(blocks_for_source), sample_block));
|
2020-02-17 16:35:23 +00:00
|
|
|
|
|
|
|
return pipes;
|
2020-01-02 17:24:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Complete query using input streams from mergeable blocks
|
2020-02-17 16:35:23 +00:00
|
|
|
BlockInputStreamPtr StorageLiveView::completeQuery(Pipes pipes)
|
2020-01-02 17:24:55 +00:00
|
|
|
{
|
2020-03-03 19:53:18 +00:00
|
|
|
//FIXME it's dangerous to create Context on stack
|
2021-04-10 23:33:54 +00:00
|
|
|
auto block_context = Context::createCopy(getContext());
|
2020-01-02 17:24:55 +00:00
|
|
|
block_context->makeQueryContext();
|
|
|
|
|
2020-03-10 19:36:17 +00:00
|
|
|
auto creator = [&](const StorageID & blocks_id_global)
|
|
|
|
{
|
2020-06-17 16:39:58 +00:00
|
|
|
auto parent_table_metadata = getParentStorage()->getInMemoryMetadataPtr();
|
|
|
|
return StorageBlocks::createStorage(
|
|
|
|
blocks_id_global, parent_table_metadata->getColumns(),
|
|
|
|
std::move(pipes), QueryProcessingStage::WithMergeableState);
|
2020-03-10 19:36:17 +00:00
|
|
|
};
|
2021-04-10 23:33:54 +00:00
|
|
|
block_context->addExternalTable(getBlocksTableName(), TemporaryTableHolder(getContext(), creator));
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
InterpreterSelectQuery select(getInnerBlocksQuery(), block_context, StoragePtr(), nullptr, SelectQueryOptions(QueryProcessingStage::Complete));
|
2020-05-28 14:11:36 +00:00
|
|
|
BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().getInputStream());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
|
|
|
/// Squashing is needed here because the view query can generate a lot of blocks
|
|
|
|
/// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
|
|
|
|
/// and two-level aggregation is triggered).
|
|
|
|
data = std::make_shared<SquashingBlockInputStream>(
|
2021-04-10 23:33:54 +00:00
|
|
|
data, getContext()->getSettingsRef().min_insert_block_size_rows,
|
|
|
|
getContext()->getSettingsRef().min_insert_block_size_bytes);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
|
|
|
return data;
|
|
|
|
}
|
2019-08-22 22:41:30 +00:00
|
|
|
|
|
|
|
void StorageLiveView::writeIntoLiveView(
|
|
|
|
StorageLiveView & live_view,
|
|
|
|
const Block & block,
|
2021-04-10 23:33:54 +00:00
|
|
|
ContextPtr local_context)
|
2019-08-22 22:41:30 +00:00
|
|
|
{
|
|
|
|
BlockOutputStreamPtr output = std::make_shared<LiveViewBlockOutputStream>(live_view);
|
|
|
|
|
|
|
|
/// Check if live view has any readers if not
|
|
|
|
/// just reset blocks to empty and do nothing else
|
|
|
|
/// When first reader comes the blocks will be read.
|
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(live_view.mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
if (!live_view.hasActiveUsers())
|
|
|
|
{
|
|
|
|
live_view.reset();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool is_block_processed = false;
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes from;
|
2020-01-03 01:48:15 +00:00
|
|
|
MergeableBlocksPtr mergeable_blocks;
|
2019-08-22 22:41:30 +00:00
|
|
|
BlocksPtr new_mergeable_blocks = std::make_shared<Blocks>();
|
|
|
|
|
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(live_view.mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
|
|
|
|
mergeable_blocks = live_view.getMergeableBlocks();
|
2021-04-10 23:33:54 +00:00
|
|
|
if (!mergeable_blocks || mergeable_blocks->blocks->size() >= local_context->getGlobalContext()->getSettingsRef().max_live_view_insert_blocks_before_refresh)
|
2019-08-22 22:41:30 +00:00
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
mergeable_blocks = live_view.collectMergeableBlocks(local_context);
|
2019-08-22 22:41:30 +00:00
|
|
|
live_view.setMergeableBlocks(mergeable_blocks);
|
2020-02-17 16:35:23 +00:00
|
|
|
from = live_view.blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block);
|
2019-08-22 22:41:30 +00:00
|
|
|
is_block_processed = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!is_block_processed)
|
|
|
|
{
|
2020-01-03 01:48:15 +00:00
|
|
|
ASTPtr mergeable_query = live_view.getInnerQuery();
|
|
|
|
|
|
|
|
if (live_view.getInnerSubQuery())
|
|
|
|
mergeable_query = live_view.getInnerSubQuery();
|
|
|
|
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes pipes;
|
|
|
|
pipes.emplace_back(std::make_shared<SourceFromSingleChunk>(block.cloneEmpty(), Chunk(block.getColumns(), block.rows())));
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-03-10 19:36:17 +00:00
|
|
|
auto creator = [&](const StorageID & blocks_id_global)
|
|
|
|
{
|
2020-06-17 16:39:58 +00:00
|
|
|
auto parent_metadata = live_view.getParentStorage()->getInMemoryMetadataPtr();
|
|
|
|
return StorageBlocks::createStorage(
|
|
|
|
blocks_id_global, parent_metadata->getColumns(),
|
|
|
|
std::move(pipes), QueryProcessingStage::FetchColumns);
|
2020-03-10 19:36:17 +00:00
|
|
|
};
|
2021-04-10 23:33:54 +00:00
|
|
|
TemporaryTableHolder blocks_storage(local_context, creator);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
InterpreterSelectQuery select_block(mergeable_query, local_context, blocks_storage.getTable(), blocks_storage.getTable()->getInMemoryMetadataPtr(),
|
2019-08-22 22:41:30 +00:00
|
|
|
QueryProcessingStage::WithMergeableState);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2019-08-22 22:41:30 +00:00
|
|
|
auto data_mergeable_stream = std::make_shared<MaterializingBlockInputStream>(
|
2020-05-28 14:11:36 +00:00
|
|
|
select_block.execute().getInputStream());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2019-08-22 22:41:30 +00:00
|
|
|
while (Block this_block = data_mergeable_stream->read())
|
|
|
|
new_mergeable_blocks->push_back(this_block);
|
|
|
|
|
|
|
|
if (new_mergeable_blocks->empty())
|
|
|
|
return;
|
|
|
|
|
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(live_view.mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
|
|
|
|
mergeable_blocks = live_view.getMergeableBlocks();
|
2020-01-03 01:48:15 +00:00
|
|
|
mergeable_blocks->blocks->push_back(new_mergeable_blocks);
|
2020-02-17 16:35:23 +00:00
|
|
|
from = live_view.blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block);
|
2019-08-22 22:41:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-17 16:35:23 +00:00
|
|
|
BlockInputStreamPtr data = live_view.completeQuery(std::move(from));
|
2019-08-22 22:41:30 +00:00
|
|
|
copyData(*data, *output);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
StorageLiveView::StorageLiveView(
|
2019-12-04 16:06:55 +00:00
|
|
|
const StorageID & table_id_,
|
2021-04-10 23:33:54 +00:00
|
|
|
ContextPtr context_,
|
2019-05-26 22:03:30 +00:00
|
|
|
const ASTCreateQuery & query,
|
2019-08-12 11:06:28 +00:00
|
|
|
const ColumnsDescription & columns_)
|
2020-04-27 13:55:30 +00:00
|
|
|
: IStorage(table_id_)
|
2021-04-10 23:33:54 +00:00
|
|
|
, WithContext(context_->getGlobalContext())
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
live_view_context = Context::createCopy(getContext());
|
2019-11-15 14:24:28 +00:00
|
|
|
live_view_context->makeQueryContext();
|
|
|
|
|
2020-09-14 19:39:33 +00:00
|
|
|
log = &Poco::Logger::get("StorageLiveView (" + table_id_.database_name + "." + table_id_.table_name + ")");
|
|
|
|
|
2020-06-19 15:39:41 +00:00
|
|
|
StorageInMemoryMetadata storage_metadata;
|
|
|
|
storage_metadata.setColumns(columns_);
|
|
|
|
setInMemoryMetadata(storage_metadata);
|
2019-08-24 21:20:20 +00:00
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
if (!query.select)
|
|
|
|
throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY);
|
|
|
|
|
|
|
|
/// Default value, if only table name exist in the query
|
|
|
|
if (query.select->list_of_selects->children.size() != 1)
|
|
|
|
throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW);
|
|
|
|
|
|
|
|
inner_query = query.select->list_of_selects->children.at(0);
|
|
|
|
|
2020-01-24 20:14:42 +00:00
|
|
|
auto inner_query_tmp = inner_query->clone();
|
2021-04-10 23:33:54 +00:00
|
|
|
select_table_id = extractDependentTable(inner_query_tmp, getContext(), table_id_.table_name, inner_subquery);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-02-12 16:54:26 +00:00
|
|
|
DatabaseCatalog::instance().addDependency(select_table_id, table_id_);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-02-14 18:20:20 +00:00
|
|
|
if (query.live_view_timeout)
|
|
|
|
{
|
|
|
|
is_temporary = true;
|
2020-09-14 19:39:33 +00:00
|
|
|
temporary_live_view_timeout = Seconds {*query.live_view_timeout};
|
|
|
|
}
|
|
|
|
|
|
|
|
if (query.live_view_periodic_refresh)
|
|
|
|
{
|
|
|
|
is_periodically_refreshed = true;
|
|
|
|
periodic_live_view_refresh = Seconds {*query.live_view_periodic_refresh};
|
2020-02-14 18:20:20 +00:00
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
blocks_ptr = std::make_shared<BlocksPtr>();
|
2019-05-28 21:17:48 +00:00
|
|
|
blocks_metadata_ptr = std::make_shared<BlocksMetadataPtr>();
|
2019-05-26 22:03:30 +00:00
|
|
|
active_ptr = std::make_shared<bool>(true);
|
2020-09-14 19:39:33 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
periodic_refresh_task = getContext()->getSchedulePool().createTask("LieViewPeriodicRefreshTask", [this]{ periodicRefreshTaskFunc(); });
|
2020-09-14 19:39:33 +00:00
|
|
|
periodic_refresh_task->deactivate();
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
Block StorageLiveView::getHeader() const
|
|
|
|
{
|
2019-10-17 14:43:11 +00:00
|
|
|
std::lock_guard lock(sample_block_lock);
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
if (!sample_block)
|
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
sample_block = InterpreterSelectQuery(inner_query->clone(), live_view_context, SelectQueryOptions(QueryProcessingStage::Complete)).getSampleBlock();
|
2019-06-14 20:27:43 +00:00
|
|
|
sample_block.insert({DataTypeUInt64().createColumnConst(
|
|
|
|
sample_block.rows(), 0)->convertToFullColumnIfConst(),
|
|
|
|
std::make_shared<DataTypeUInt64>(),
|
|
|
|
"_version"});
|
|
|
|
/// convert all columns to full columns
|
|
|
|
/// in case some of them are constant
|
|
|
|
for (size_t i = 0; i < sample_block.columns(); ++i)
|
2019-06-14 14:06:15 +00:00
|
|
|
{
|
2019-06-14 20:27:43 +00:00
|
|
|
sample_block.safeGetByPosition(i).column = sample_block.safeGetByPosition(i).column->convertToFullColumnIfConst();
|
2019-06-14 14:06:15 +00:00
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
return sample_block;
|
|
|
|
}
|
|
|
|
|
2020-12-10 23:56:57 +00:00
|
|
|
StoragePtr StorageLiveView::getParentStorage() const
|
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
return DatabaseCatalog::instance().getTable(select_table_id, getContext());
|
2020-12-10 23:56:57 +00:00
|
|
|
}
|
|
|
|
|
2020-01-24 20:14:42 +00:00
|
|
|
ASTPtr StorageLiveView::getInnerBlocksQuery()
|
|
|
|
{
|
|
|
|
std::lock_guard lock(sample_block_lock);
|
|
|
|
if (!inner_blocks_query)
|
|
|
|
{
|
|
|
|
inner_blocks_query = inner_query->clone();
|
|
|
|
/// Rewrite inner query with right aliases for JOIN.
|
|
|
|
/// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table,
|
|
|
|
/// which is not loaded yet during server startup, so we do it lazily
|
2021-04-10 23:33:54 +00:00
|
|
|
InterpreterSelectQuery(inner_blocks_query, live_view_context, SelectQueryOptions().modify().analyze()); // NOLINT
|
2020-01-24 20:14:42 +00:00
|
|
|
auto table_id = getStorageID();
|
2021-04-10 23:33:54 +00:00
|
|
|
extractDependentTable(inner_blocks_query, getContext(), table_id.table_name, inner_subquery);
|
2020-01-24 20:14:42 +00:00
|
|
|
}
|
|
|
|
return inner_blocks_query->clone();
|
|
|
|
}
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
bool StorageLiveView::getNewBlocks()
|
|
|
|
{
|
|
|
|
SipHash hash;
|
|
|
|
UInt128 key;
|
|
|
|
BlocksPtr new_blocks = std::make_shared<Blocks>();
|
2019-05-28 21:17:48 +00:00
|
|
|
BlocksMetadataPtr new_blocks_metadata = std::make_shared<BlocksMetadata>();
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-07-15 11:24:33 +00:00
|
|
|
/// can't set mergeable_blocks here or anywhere else outside the writeIntoLiveView function
|
|
|
|
/// as there could be a race codition when the new block has been inserted into
|
|
|
|
/// the source table by the PushingToViewsBlockOutputStream and this method
|
|
|
|
/// called before writeIntoLiveView function is called which can lead to
|
|
|
|
/// the same block added twice to the mergeable_blocks leading to
|
|
|
|
/// inserted data to be duplicated
|
2021-04-10 23:33:54 +00:00
|
|
|
auto new_mergeable_blocks = collectMergeableBlocks(live_view_context);
|
2020-07-15 14:18:53 +00:00
|
|
|
Pipes from = blocksToPipes(new_mergeable_blocks->blocks, new_mergeable_blocks->sample_block);
|
2020-02-17 16:35:23 +00:00
|
|
|
BlockInputStreamPtr data = completeQuery(std::move(from));
|
2019-06-23 13:51:29 +00:00
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
while (Block block = data->read())
|
|
|
|
{
|
2019-05-30 21:29:30 +00:00
|
|
|
/// calculate hash before virtual column is added
|
|
|
|
block.updateHash(hash);
|
2019-05-29 21:52:13 +00:00
|
|
|
/// add result version meta column
|
|
|
|
block.insert({DataTypeUInt64().createColumnConst(
|
|
|
|
block.rows(), getBlocksVersion() + 1)->convertToFullColumnIfConst(),
|
|
|
|
std::make_shared<DataTypeUInt64>(),
|
|
|
|
"_version"});
|
2019-05-26 22:03:30 +00:00
|
|
|
new_blocks->push_back(block);
|
|
|
|
}
|
|
|
|
|
2021-01-27 00:54:57 +00:00
|
|
|
hash.get128(key);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
/// Update blocks only if hash keys do not match
|
|
|
|
/// NOTE: hash could be different for the same result
|
|
|
|
/// if blocks are not in the same order
|
|
|
|
bool updated = false;
|
|
|
|
{
|
2021-01-27 00:54:57 +00:00
|
|
|
if (getBlocksHashKey() != getHexUIntLowercase(key))
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
|
|
|
if (new_blocks->empty())
|
|
|
|
{
|
|
|
|
new_blocks->push_back(getHeader());
|
|
|
|
}
|
2021-01-27 00:54:57 +00:00
|
|
|
new_blocks_metadata->hash = getHexUIntLowercase(key);
|
2019-05-28 21:17:48 +00:00
|
|
|
new_blocks_metadata->version = getBlocksVersion() + 1;
|
2020-09-14 19:39:33 +00:00
|
|
|
new_blocks_metadata->time = std::chrono::system_clock::now();
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
(*blocks_ptr) = new_blocks;
|
2019-05-28 21:17:48 +00:00
|
|
|
(*blocks_metadata_ptr) = new_blocks_metadata;
|
2020-09-14 19:39:33 +00:00
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
updated = true;
|
|
|
|
}
|
2021-01-30 14:00:42 +00:00
|
|
|
else
|
|
|
|
{
|
2020-09-14 19:39:33 +00:00
|
|
|
new_blocks_metadata->hash = getBlocksHashKey();
|
|
|
|
new_blocks_metadata->version = getBlocksVersion();
|
|
|
|
new_blocks_metadata->time = std::chrono::system_clock::now();
|
|
|
|
|
|
|
|
(*blocks_metadata_ptr) = new_blocks_metadata;
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
return updated;
|
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::checkTableCanBeDropped() const
|
|
|
|
{
|
2019-12-03 16:25:32 +00:00
|
|
|
auto table_id = getStorageID();
|
2020-02-12 16:54:26 +00:00
|
|
|
Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id);
|
2019-05-26 22:03:30 +00:00
|
|
|
if (!dependencies.empty())
|
|
|
|
{
|
2019-12-05 11:42:13 +00:00
|
|
|
StorageID dependent_table_id = dependencies.front();
|
|
|
|
throw Exception("Table has dependency " + dependent_table_id.getNameForLogs(), ErrorCodes::TABLE_WAS_NOT_DROPPED);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::startup()
|
|
|
|
{
|
2020-09-07 23:08:17 +00:00
|
|
|
if (is_temporary)
|
|
|
|
TemporaryLiveViewCleaner::instance().addView(std::static_pointer_cast<StorageLiveView>(shared_from_this()));
|
2020-09-14 19:39:33 +00:00
|
|
|
|
|
|
|
if (is_periodically_refreshed)
|
|
|
|
periodic_refresh_task->activate();
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::shutdown()
|
|
|
|
{
|
2020-09-07 23:08:17 +00:00
|
|
|
shutdown_called = true;
|
2020-09-14 19:39:33 +00:00
|
|
|
|
|
|
|
if (is_periodically_refreshed)
|
|
|
|
periodic_refresh_task->deactivate();
|
|
|
|
|
2020-02-12 16:54:26 +00:00
|
|
|
DatabaseCatalog::instance().removeDependency(select_table_id, getStorageID());
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
StorageLiveView::~StorageLiveView()
|
|
|
|
{
|
|
|
|
shutdown();
|
|
|
|
}
|
|
|
|
|
2020-01-22 11:30:11 +00:00
|
|
|
void StorageLiveView::drop()
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-12-03 16:25:32 +00:00
|
|
|
auto table_id = getStorageID();
|
2020-02-12 16:54:26 +00:00
|
|
|
DatabaseCatalog::instance().removeDependency(select_table_id, table_id);
|
2019-08-22 23:22:57 +00:00
|
|
|
|
|
|
|
std::lock_guard lock(mutex);
|
2019-05-26 22:03:30 +00:00
|
|
|
is_dropped = true;
|
2019-08-22 23:22:57 +00:00
|
|
|
condition.notify_all();
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-09-14 19:39:33 +00:00
|
|
|
void StorageLiveView::scheduleNextPeriodicRefresh()
|
|
|
|
{
|
|
|
|
Seconds current_time = std::chrono::duration_cast<Seconds> (std::chrono::system_clock::now().time_since_epoch());
|
|
|
|
Seconds blocks_time = std::chrono::duration_cast<Seconds> (getBlocksTime().time_since_epoch());
|
|
|
|
|
2021-01-30 14:00:42 +00:00
|
|
|
if ((current_time - periodic_live_view_refresh) >= blocks_time)
|
2020-09-14 19:39:33 +00:00
|
|
|
{
|
|
|
|
refresh(false);
|
|
|
|
blocks_time = std::chrono::duration_cast<Seconds> (getBlocksTime().time_since_epoch());
|
|
|
|
}
|
|
|
|
current_time = std::chrono::duration_cast<Seconds> (std::chrono::system_clock::now().time_since_epoch());
|
|
|
|
|
|
|
|
auto next_refresh_time = blocks_time + periodic_live_view_refresh;
|
|
|
|
|
|
|
|
if (current_time >= next_refresh_time)
|
|
|
|
periodic_refresh_task->scheduleAfter(0);
|
|
|
|
else
|
|
|
|
{
|
|
|
|
auto schedule_time = std::chrono::duration_cast<MilliSeconds> (next_refresh_time - current_time);
|
|
|
|
periodic_refresh_task->scheduleAfter(static_cast<size_t>(schedule_time.count()));
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::periodicRefreshTaskFunc()
|
|
|
|
{
|
|
|
|
LOG_TRACE(log, "periodic refresh task");
|
|
|
|
|
|
|
|
std::lock_guard lock(mutex);
|
|
|
|
|
|
|
|
if (hasActiveUsers())
|
|
|
|
scheduleNextPeriodicRefresh();
|
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::refresh(bool grab_lock)
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2020-09-01 04:42:27 +00:00
|
|
|
// Lock is already acquired exclusively from InterperterAlterQuery.cpp InterpreterAlterQuery::execute() method.
|
|
|
|
// So, reacquiring lock is not needed and will result in an exception.
|
2020-09-14 19:39:33 +00:00
|
|
|
|
|
|
|
if (grab_lock)
|
2020-04-01 12:43:09 +00:00
|
|
|
{
|
|
|
|
std::lock_guard lock(mutex);
|
|
|
|
if (getNewBlocks())
|
|
|
|
condition.notify_all();
|
|
|
|
}
|
2020-09-14 19:39:33 +00:00
|
|
|
else
|
|
|
|
{
|
|
|
|
if (getNewBlocks())
|
|
|
|
condition.notify_all();
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-08-03 11:33:11 +00:00
|
|
|
Pipe StorageLiveView::read(
|
2019-05-26 22:03:30 +00:00
|
|
|
const Names & /*column_names*/,
|
2020-06-15 19:08:58 +00:00
|
|
|
const StorageMetadataPtr & /*metadata_snapshot*/,
|
2020-09-20 17:52:17 +00:00
|
|
|
SelectQueryInfo & /*query_info*/,
|
2021-04-10 23:33:54 +00:00
|
|
|
ContextPtr /*context*/,
|
2019-05-26 22:03:30 +00:00
|
|
|
QueryProcessingStage::Enum /*processed_stage*/,
|
|
|
|
const size_t /*max_block_size*/,
|
|
|
|
const unsigned /*num_streams*/)
|
|
|
|
{
|
2020-09-14 19:39:33 +00:00
|
|
|
std::lock_guard lock(mutex);
|
|
|
|
|
|
|
|
if (!(*blocks_ptr))
|
|
|
|
refresh(false);
|
|
|
|
|
|
|
|
else if (is_periodically_refreshed)
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2021-02-11 20:59:00 +00:00
|
|
|
Seconds current_time = std::chrono::duration_cast<Seconds>(std::chrono::system_clock::now().time_since_epoch());
|
|
|
|
Seconds blocks_time = std::chrono::duration_cast<Seconds>(getBlocksTime().time_since_epoch());
|
2020-09-14 19:39:33 +00:00
|
|
|
|
2021-01-30 14:00:42 +00:00
|
|
|
if ((current_time - periodic_live_view_refresh) >= blocks_time)
|
2020-09-14 19:39:33 +00:00
|
|
|
refresh(false);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2020-09-14 19:39:33 +00:00
|
|
|
|
|
|
|
return Pipe(std::make_shared<BlocksSource>(blocks_ptr, getHeader()));
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BlockInputStreams StorageLiveView::watch(
|
|
|
|
const Names & /*column_names*/,
|
|
|
|
const SelectQueryInfo & query_info,
|
2021-04-10 23:33:54 +00:00
|
|
|
ContextPtr local_context,
|
2019-05-26 22:03:30 +00:00
|
|
|
QueryProcessingStage::Enum & processed_stage,
|
|
|
|
size_t /*max_block_size*/,
|
|
|
|
const unsigned /*num_streams*/)
|
|
|
|
{
|
|
|
|
ASTWatchQuery & query = typeid_cast<ASTWatchQuery &>(*query_info.query);
|
|
|
|
|
2019-07-09 12:41:55 +00:00
|
|
|
bool has_limit = false;
|
|
|
|
UInt64 limit = 0;
|
2020-09-14 19:39:33 +00:00
|
|
|
BlockInputStreamPtr reader;
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
if (query.limit_length)
|
2019-07-09 12:41:55 +00:00
|
|
|
{
|
|
|
|
has_limit = true;
|
|
|
|
limit = safeGet<UInt64>(typeid_cast<ASTLiteral &>(*query.limit_length).value);
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2019-05-28 21:17:48 +00:00
|
|
|
if (query.is_watch_events)
|
2020-09-14 19:39:33 +00:00
|
|
|
reader = std::make_shared<LiveViewEventsBlockInputStream>(
|
2019-08-25 12:27:47 +00:00
|
|
|
std::static_pointer_cast<StorageLiveView>(shared_from_this()),
|
|
|
|
blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
|
2021-04-10 23:33:54 +00:00
|
|
|
local_context->getSettingsRef().live_view_heartbeat_interval.totalSeconds());
|
2019-05-28 21:17:48 +00:00
|
|
|
else
|
2020-09-14 19:39:33 +00:00
|
|
|
reader = std::make_shared<LiveViewBlockInputStream>(
|
2019-08-25 12:27:47 +00:00
|
|
|
std::static_pointer_cast<StorageLiveView>(shared_from_this()),
|
|
|
|
blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
|
2021-04-10 23:33:54 +00:00
|
|
|
local_context->getSettingsRef().live_view_heartbeat_interval.totalSeconds());
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-09-14 19:39:33 +00:00
|
|
|
{
|
|
|
|
std::lock_guard lock(mutex);
|
2019-05-28 21:17:48 +00:00
|
|
|
|
2020-09-14 19:39:33 +00:00
|
|
|
if (!(*blocks_ptr))
|
|
|
|
refresh(false);
|
2019-05-28 21:17:48 +00:00
|
|
|
|
2020-09-14 19:39:33 +00:00
|
|
|
if (is_periodically_refreshed)
|
|
|
|
scheduleNextPeriodicRefresh();
|
2019-05-28 21:17:48 +00:00
|
|
|
}
|
2020-09-14 19:39:33 +00:00
|
|
|
|
|
|
|
processed_stage = QueryProcessingStage::Complete;
|
|
|
|
return { reader };
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-04-28 10:38:57 +00:00
|
|
|
NamesAndTypesList StorageLiveView::getVirtuals() const
|
2020-04-27 13:55:30 +00:00
|
|
|
{
|
2020-04-28 10:38:57 +00:00
|
|
|
return NamesAndTypesList{
|
2020-04-27 13:55:30 +00:00
|
|
|
NameAndTypePair("_version", std::make_shared<DataTypeUInt64>())
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
void registerStorageLiveView(StorageFactory & factory)
|
|
|
|
{
|
|
|
|
factory.registerStorage("LiveView", [](const StorageFactory::Arguments & args)
|
|
|
|
{
|
2021-04-10 23:33:54 +00:00
|
|
|
if (!args.attach && !args.getLocalContext()->getSettingsRef().allow_experimental_live_view)
|
|
|
|
throw Exception(
|
|
|
|
"Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')",
|
|
|
|
ErrorCodes::SUPPORT_IS_DISABLED);
|
2019-08-22 23:46:40 +00:00
|
|
|
|
2021-04-10 23:33:54 +00:00
|
|
|
return StorageLiveView::create(args.table_id, args.getLocalContext(), args.query, args.columns);
|
2019-05-26 22:03:30 +00:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|