2020-01-03 01:48:15 +00:00
|
|
|
/* Copyright (c) 2018 BlackBerry Limited
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
you may not use this file except in compliance with the License.
|
|
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
See the License for the specific language governing permissions and
|
|
|
|
limitations under the License. */
|
|
|
|
#include <Parsers/ASTSelectQuery.h>
|
|
|
|
#include <Parsers/ASTCreateQuery.h>
|
|
|
|
#include <Parsers/ASTWatchQuery.h>
|
|
|
|
#include <Parsers/ASTDropQuery.h>
|
|
|
|
#include <Parsers/ASTLiteral.h>
|
|
|
|
#include <Interpreters/Context.h>
|
|
|
|
#include <Interpreters/InterpreterDropQuery.h>
|
2019-08-22 22:41:30 +00:00
|
|
|
#include <Interpreters/InterpreterSelectQuery.h>
|
|
|
|
#include <DataStreams/IBlockOutputStream.h>
|
|
|
|
#include <DataStreams/OneBlockInputStream.h>
|
2020-06-01 16:31:06 +00:00
|
|
|
#include <DataStreams/BlocksSource.h>
|
2019-08-22 22:41:30 +00:00
|
|
|
#include <DataStreams/MaterializingBlockInputStream.h>
|
|
|
|
#include <DataStreams/SquashingBlockInputStream.h>
|
|
|
|
#include <DataStreams/copyData.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2019-08-22 23:22:57 +00:00
|
|
|
#include <Common/SipHash.h>
|
|
|
|
|
|
|
|
#include <Storages/LiveView/StorageLiveView.h>
|
|
|
|
#include <Storages/LiveView/LiveViewBlockInputStream.h>
|
|
|
|
#include <Storages/LiveView/LiveViewBlockOutputStream.h>
|
|
|
|
#include <Storages/LiveView/LiveViewEventsBlockInputStream.h>
|
2019-12-04 23:29:37 +00:00
|
|
|
#include <Storages/LiveView/StorageBlocks.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
#include <Storages/StorageFactory.h>
|
|
|
|
#include <Parsers/ASTTablesInSelectQuery.h>
|
|
|
|
#include <Parsers/ASTSubquery.h>
|
2019-12-05 21:42:00 +00:00
|
|
|
#include <Parsers/queryToString.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
#include <Interpreters/DatabaseAndTableWithAlias.h>
|
2019-12-12 15:28:24 +00:00
|
|
|
#include <Interpreters/getTableExpressions.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
#include <Interpreters/AddDefaultDatabaseVisitor.h>
|
2020-01-26 09:49:53 +00:00
|
|
|
#include <Access/AccessFlags.h>
|
2020-02-17 16:35:23 +00:00
|
|
|
#include <Processors/Sources/SourceFromSingleChunk.h>
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2019-12-05 11:53:36 +00:00
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
|
|
|
extern const int LOGICAL_ERROR;
|
|
|
|
extern const int INCORRECT_QUERY;
|
|
|
|
extern const int TABLE_WAS_NOT_DROPPED;
|
|
|
|
extern const int QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW;
|
2019-08-22 23:46:40 +00:00
|
|
|
extern const int SUPPORT_IS_DISABLED;
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2019-08-22 22:41:30 +00:00
|
|
|
|
2020-01-09 14:33:58 +00:00
|
|
|
static StorageID extractDependentTable(ASTPtr & query, Context & context, const String & table_name, ASTPtr & inner_subquery)
|
2019-12-04 23:29:37 +00:00
|
|
|
{
|
2019-12-05 11:53:36 +00:00
|
|
|
ASTSelectQuery & select_query = typeid_cast<ASTSelectQuery &>(*query);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-01-09 14:33:58 +00:00
|
|
|
if (auto db_and_table = getDatabaseAndTable(select_query, 0))
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2020-01-09 14:33:58 +00:00
|
|
|
String select_database_name = context.getCurrentDatabase();
|
|
|
|
String select_table_name = db_and_table->table;
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
if (db_and_table->database.empty())
|
|
|
|
{
|
|
|
|
db_and_table->database = select_database_name;
|
|
|
|
AddDefaultDatabaseVisitor visitor(select_database_name);
|
2019-12-04 23:29:37 +00:00
|
|
|
visitor.visit(select_query);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
select_database_name = db_and_table->database;
|
2019-12-05 21:42:00 +00:00
|
|
|
|
2020-01-01 16:44:45 +00:00
|
|
|
select_query.replaceDatabaseAndTable("", table_name + "_blocks");
|
2020-01-09 14:33:58 +00:00
|
|
|
return StorageID(select_database_name, select_table_name);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2020-01-09 14:33:58 +00:00
|
|
|
else if (auto subquery = extractTableExpression(select_query, 0))
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2020-01-09 14:33:58 +00:00
|
|
|
auto * ast_select = subquery->as<ASTSelectWithUnionQuery>();
|
|
|
|
if (!ast_select)
|
|
|
|
throw Exception("Logical error while creating StorageLiveView."
|
|
|
|
" Could not retrieve table name from select query.",
|
|
|
|
DB::ErrorCodes::LOGICAL_ERROR);
|
2019-05-26 22:03:30 +00:00
|
|
|
if (ast_select->list_of_selects->children.size() != 1)
|
|
|
|
throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW);
|
|
|
|
|
2020-01-01 16:44:45 +00:00
|
|
|
inner_subquery = ast_select->list_of_selects->children.at(0)->clone();
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-01-09 14:33:58 +00:00
|
|
|
return extractDependentTable(ast_select->list_of_selects->children.at(0), context, table_name, inner_subquery);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
else
|
2020-01-09 14:33:58 +00:00
|
|
|
{
|
|
|
|
/// If the table is not specified - use the table `system.one`
|
|
|
|
return StorageID("system", "one");
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(const Context & context)
|
2020-01-02 17:24:55 +00:00
|
|
|
{
|
2020-01-02 19:31:04 +00:00
|
|
|
ASTPtr mergeable_query = inner_query;
|
|
|
|
|
|
|
|
if (inner_subquery)
|
|
|
|
mergeable_query = inner_subquery;
|
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
MergeableBlocksPtr new_mergeable_blocks = std::make_shared<MergeableBlocks>();
|
|
|
|
BlocksPtrs new_blocks = std::make_shared<std::vector<BlocksPtr>>();
|
|
|
|
BlocksPtr base_blocks = std::make_shared<Blocks>();
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
InterpreterSelectQuery interpreter(mergeable_query->clone(), context, SelectQueryOptions(QueryProcessingStage::WithMergeableState), Names());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-05-28 14:11:36 +00:00
|
|
|
auto view_mergeable_stream = std::make_shared<MaterializingBlockInputStream>(interpreter.execute().getInputStream());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
|
|
|
while (Block this_block = view_mergeable_stream->read())
|
2020-01-03 01:48:15 +00:00
|
|
|
base_blocks->push_back(this_block);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
new_blocks->push_back(base_blocks);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
new_mergeable_blocks->blocks = new_blocks;
|
|
|
|
new_mergeable_blocks->sample_block = view_mergeable_stream->getHeader();
|
2020-01-02 19:31:04 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
return new_mergeable_blocks;
|
2020-01-02 17:24:55 +00:00
|
|
|
}
|
|
|
|
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes StorageLiveView::blocksToPipes(BlocksPtrs blocks, Block & sample_block)
|
2020-01-02 17:24:55 +00:00
|
|
|
{
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes pipes;
|
2020-03-23 02:12:31 +00:00
|
|
|
for (auto & blocks_for_source : *blocks)
|
|
|
|
pipes.emplace_back(std::make_shared<BlocksSource>(std::make_shared<BlocksPtr>(blocks_for_source), sample_block));
|
2020-02-17 16:35:23 +00:00
|
|
|
|
|
|
|
return pipes;
|
2020-01-02 17:24:55 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/// Complete query using input streams from mergeable blocks
|
2020-02-17 16:35:23 +00:00
|
|
|
BlockInputStreamPtr StorageLiveView::completeQuery(Pipes pipes)
|
2020-01-02 17:24:55 +00:00
|
|
|
{
|
2020-03-03 19:53:18 +00:00
|
|
|
//FIXME it's dangerous to create Context on stack
|
2020-01-02 17:24:55 +00:00
|
|
|
auto block_context = std::make_unique<Context>(global_context);
|
|
|
|
block_context->makeQueryContext();
|
|
|
|
|
2020-03-10 19:36:17 +00:00
|
|
|
auto creator = [&](const StorageID & blocks_id_global)
|
|
|
|
{
|
|
|
|
return StorageBlocks::createStorage(blocks_id_global, getParentStorage()->getColumns(),
|
|
|
|
std::move(pipes), QueryProcessingStage::WithMergeableState);
|
|
|
|
};
|
|
|
|
block_context->addExternalTable(getBlocksTableName(), TemporaryTableHolder(global_context, creator));
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-01-24 20:14:42 +00:00
|
|
|
InterpreterSelectQuery select(getInnerBlocksQuery(), *block_context, StoragePtr(), SelectQueryOptions(QueryProcessingStage::Complete));
|
2020-05-28 14:11:36 +00:00
|
|
|
BlockInputStreamPtr data = std::make_shared<MaterializingBlockInputStream>(select.execute().getInputStream());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
|
|
|
/// Squashing is needed here because the view query can generate a lot of blocks
|
|
|
|
/// even when only one block is inserted into the parent table (e.g. if the query is a GROUP BY
|
|
|
|
/// and two-level aggregation is triggered).
|
|
|
|
data = std::make_shared<SquashingBlockInputStream>(
|
|
|
|
data, global_context.getSettingsRef().min_insert_block_size_rows,
|
|
|
|
global_context.getSettingsRef().min_insert_block_size_bytes);
|
|
|
|
|
|
|
|
return data;
|
|
|
|
}
|
2019-08-22 22:41:30 +00:00
|
|
|
|
|
|
|
void StorageLiveView::writeIntoLiveView(
|
|
|
|
StorageLiveView & live_view,
|
|
|
|
const Block & block,
|
|
|
|
const Context & context)
|
|
|
|
{
|
|
|
|
BlockOutputStreamPtr output = std::make_shared<LiveViewBlockOutputStream>(live_view);
|
|
|
|
|
|
|
|
/// Check if live view has any readers if not
|
|
|
|
/// just reset blocks to empty and do nothing else
|
|
|
|
/// When first reader comes the blocks will be read.
|
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(live_view.mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
if (!live_view.hasActiveUsers())
|
|
|
|
{
|
|
|
|
live_view.reset();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
bool is_block_processed = false;
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes from;
|
2020-01-03 01:48:15 +00:00
|
|
|
MergeableBlocksPtr mergeable_blocks;
|
2019-08-22 22:41:30 +00:00
|
|
|
BlocksPtr new_mergeable_blocks = std::make_shared<Blocks>();
|
|
|
|
|
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(live_view.mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
|
|
|
|
mergeable_blocks = live_view.getMergeableBlocks();
|
2020-01-03 01:48:15 +00:00
|
|
|
if (!mergeable_blocks || mergeable_blocks->blocks->size() >= context.getGlobalContext().getSettingsRef().max_live_view_insert_blocks_before_refresh)
|
2019-08-22 22:41:30 +00:00
|
|
|
{
|
2020-01-02 17:24:55 +00:00
|
|
|
mergeable_blocks = live_view.collectMergeableBlocks(context);
|
2019-08-22 22:41:30 +00:00
|
|
|
live_view.setMergeableBlocks(mergeable_blocks);
|
2020-02-17 16:35:23 +00:00
|
|
|
from = live_view.blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block);
|
2019-08-22 22:41:30 +00:00
|
|
|
is_block_processed = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!is_block_processed)
|
|
|
|
{
|
2020-01-03 01:48:15 +00:00
|
|
|
ASTPtr mergeable_query = live_view.getInnerQuery();
|
|
|
|
|
|
|
|
if (live_view.getInnerSubQuery())
|
|
|
|
mergeable_query = live_view.getInnerSubQuery();
|
|
|
|
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes pipes;
|
|
|
|
pipes.emplace_back(std::make_shared<SourceFromSingleChunk>(block.cloneEmpty(), Chunk(block.getColumns(), block.rows())));
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-03-10 19:36:17 +00:00
|
|
|
auto creator = [&](const StorageID & blocks_id_global)
|
|
|
|
{
|
|
|
|
return StorageBlocks::createStorage(blocks_id_global, live_view.getParentStorage()->getColumns(),
|
|
|
|
std::move(pipes), QueryProcessingStage::FetchColumns);
|
|
|
|
};
|
|
|
|
TemporaryTableHolder blocks_storage(context, creator);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2020-03-10 19:36:17 +00:00
|
|
|
InterpreterSelectQuery select_block(mergeable_query, context, blocks_storage.getTable(),
|
2019-08-22 22:41:30 +00:00
|
|
|
QueryProcessingStage::WithMergeableState);
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2019-08-22 22:41:30 +00:00
|
|
|
auto data_mergeable_stream = std::make_shared<MaterializingBlockInputStream>(
|
2020-05-28 14:11:36 +00:00
|
|
|
select_block.execute().getInputStream());
|
2020-01-02 17:24:55 +00:00
|
|
|
|
2019-08-22 22:41:30 +00:00
|
|
|
while (Block this_block = data_mergeable_stream->read())
|
|
|
|
new_mergeable_blocks->push_back(this_block);
|
|
|
|
|
|
|
|
if (new_mergeable_blocks->empty())
|
|
|
|
return;
|
|
|
|
|
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(live_view.mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
|
|
|
|
mergeable_blocks = live_view.getMergeableBlocks();
|
2020-01-03 01:48:15 +00:00
|
|
|
mergeable_blocks->blocks->push_back(new_mergeable_blocks);
|
2020-02-17 16:35:23 +00:00
|
|
|
from = live_view.blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block);
|
2019-08-22 22:41:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-17 16:35:23 +00:00
|
|
|
BlockInputStreamPtr data = live_view.completeQuery(std::move(from));
|
2019-08-22 22:41:30 +00:00
|
|
|
copyData(*data, *output);
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
StorageLiveView::StorageLiveView(
|
2019-12-04 16:06:55 +00:00
|
|
|
const StorageID & table_id_,
|
2019-05-26 22:03:30 +00:00
|
|
|
Context & local_context,
|
|
|
|
const ASTCreateQuery & query,
|
2019-08-12 11:06:28 +00:00
|
|
|
const ColumnsDescription & columns_)
|
2020-04-27 13:55:30 +00:00
|
|
|
: IStorage(table_id_)
|
2020-04-24 09:20:09 +00:00
|
|
|
, global_context(local_context.getGlobalContext())
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-11-15 14:24:28 +00:00
|
|
|
live_view_context = std::make_unique<Context>(global_context);
|
|
|
|
live_view_context->makeQueryContext();
|
|
|
|
|
2019-08-24 21:20:20 +00:00
|
|
|
setColumns(columns_);
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
if (!query.select)
|
|
|
|
throw Exception("SELECT query is not specified for " + getName(), ErrorCodes::INCORRECT_QUERY);
|
|
|
|
|
|
|
|
/// Default value, if only table name exist in the query
|
|
|
|
if (query.select->list_of_selects->children.size() != 1)
|
|
|
|
throw Exception("UNION is not supported for LIVE VIEW", ErrorCodes::QUERY_IS_NOT_SUPPORTED_IN_LIVE_VIEW);
|
|
|
|
|
|
|
|
inner_query = query.select->list_of_selects->children.at(0);
|
|
|
|
|
2020-01-24 20:14:42 +00:00
|
|
|
auto inner_query_tmp = inner_query->clone();
|
|
|
|
select_table_id = extractDependentTable(inner_query_tmp, global_context, table_id_.table_name, inner_subquery);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-02-12 16:54:26 +00:00
|
|
|
DatabaseCatalog::instance().addDependency(select_table_id, table_id_);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-02-14 18:20:20 +00:00
|
|
|
if (query.live_view_timeout)
|
|
|
|
{
|
|
|
|
is_temporary = true;
|
|
|
|
temporary_live_view_timeout = *query.live_view_timeout;
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
blocks_ptr = std::make_shared<BlocksPtr>();
|
2019-05-28 21:17:48 +00:00
|
|
|
blocks_metadata_ptr = std::make_shared<BlocksMetadataPtr>();
|
2019-05-26 22:03:30 +00:00
|
|
|
active_ptr = std::make_shared<bool>(true);
|
|
|
|
}
|
|
|
|
|
|
|
|
Block StorageLiveView::getHeader() const
|
|
|
|
{
|
2019-10-17 14:43:11 +00:00
|
|
|
std::lock_guard lock(sample_block_lock);
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
if (!sample_block)
|
|
|
|
{
|
2019-12-04 23:29:37 +00:00
|
|
|
sample_block = InterpreterSelectQuery(inner_query->clone(), *live_view_context, SelectQueryOptions(QueryProcessingStage::Complete)).getSampleBlock();
|
2019-06-14 20:27:43 +00:00
|
|
|
sample_block.insert({DataTypeUInt64().createColumnConst(
|
|
|
|
sample_block.rows(), 0)->convertToFullColumnIfConst(),
|
|
|
|
std::make_shared<DataTypeUInt64>(),
|
|
|
|
"_version"});
|
|
|
|
/// convert all columns to full columns
|
|
|
|
/// in case some of them are constant
|
|
|
|
for (size_t i = 0; i < sample_block.columns(); ++i)
|
2019-06-14 14:06:15 +00:00
|
|
|
{
|
2019-06-14 20:27:43 +00:00
|
|
|
sample_block.safeGetByPosition(i).column = sample_block.safeGetByPosition(i).column->convertToFullColumnIfConst();
|
2019-06-14 14:06:15 +00:00
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
return sample_block;
|
|
|
|
}
|
|
|
|
|
2020-01-24 20:14:42 +00:00
|
|
|
ASTPtr StorageLiveView::getInnerBlocksQuery()
|
|
|
|
{
|
|
|
|
std::lock_guard lock(sample_block_lock);
|
|
|
|
if (!inner_blocks_query)
|
|
|
|
{
|
|
|
|
inner_blocks_query = inner_query->clone();
|
|
|
|
/// Rewrite inner query with right aliases for JOIN.
|
|
|
|
/// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table,
|
|
|
|
/// which is not loaded yet during server startup, so we do it lazily
|
2020-03-18 02:02:24 +00:00
|
|
|
InterpreterSelectQuery(inner_blocks_query, *live_view_context, SelectQueryOptions().modify().analyze()); // NOLINT
|
2020-01-24 20:14:42 +00:00
|
|
|
auto table_id = getStorageID();
|
|
|
|
extractDependentTable(inner_blocks_query, global_context, table_id.table_name, inner_subquery);
|
|
|
|
}
|
|
|
|
return inner_blocks_query->clone();
|
|
|
|
}
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
bool StorageLiveView::getNewBlocks()
|
|
|
|
{
|
|
|
|
SipHash hash;
|
|
|
|
UInt128 key;
|
|
|
|
BlocksPtr new_blocks = std::make_shared<Blocks>();
|
2019-05-28 21:17:48 +00:00
|
|
|
BlocksMetadataPtr new_blocks_metadata = std::make_shared<BlocksMetadata>();
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2020-01-03 01:48:15 +00:00
|
|
|
mergeable_blocks = collectMergeableBlocks(*live_view_context);
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes from = blocksToPipes(mergeable_blocks->blocks, mergeable_blocks->sample_block);
|
|
|
|
BlockInputStreamPtr data = completeQuery(std::move(from));
|
2019-06-23 13:51:29 +00:00
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
while (Block block = data->read())
|
|
|
|
{
|
2019-05-30 21:29:30 +00:00
|
|
|
/// calculate hash before virtual column is added
|
|
|
|
block.updateHash(hash);
|
2019-05-29 21:52:13 +00:00
|
|
|
/// add result version meta column
|
|
|
|
block.insert({DataTypeUInt64().createColumnConst(
|
|
|
|
block.rows(), getBlocksVersion() + 1)->convertToFullColumnIfConst(),
|
|
|
|
std::make_shared<DataTypeUInt64>(),
|
|
|
|
"_version"});
|
2019-05-26 22:03:30 +00:00
|
|
|
new_blocks->push_back(block);
|
|
|
|
}
|
|
|
|
|
|
|
|
hash.get128(key.low, key.high);
|
|
|
|
|
|
|
|
/// Update blocks only if hash keys do not match
|
|
|
|
/// NOTE: hash could be different for the same result
|
|
|
|
/// if blocks are not in the same order
|
|
|
|
bool updated = false;
|
|
|
|
{
|
2019-05-28 21:17:48 +00:00
|
|
|
if (getBlocksHashKey() != key.toHexString())
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
|
|
|
if (new_blocks->empty())
|
|
|
|
{
|
|
|
|
new_blocks->push_back(getHeader());
|
|
|
|
}
|
2019-05-28 21:17:48 +00:00
|
|
|
new_blocks_metadata->hash = key.toHexString();
|
|
|
|
new_blocks_metadata->version = getBlocksVersion() + 1;
|
2019-05-26 22:03:30 +00:00
|
|
|
(*blocks_ptr) = new_blocks;
|
2019-05-28 21:17:48 +00:00
|
|
|
(*blocks_metadata_ptr) = new_blocks_metadata;
|
2019-05-26 22:03:30 +00:00
|
|
|
updated = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return updated;
|
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::checkTableCanBeDropped() const
|
|
|
|
{
|
2019-12-03 16:25:32 +00:00
|
|
|
auto table_id = getStorageID();
|
2020-02-12 16:54:26 +00:00
|
|
|
Dependencies dependencies = DatabaseCatalog::instance().getDependencies(table_id);
|
2019-05-26 22:03:30 +00:00
|
|
|
if (!dependencies.empty())
|
|
|
|
{
|
2019-12-05 11:42:13 +00:00
|
|
|
StorageID dependent_table_id = dependencies.front();
|
|
|
|
throw Exception("Table has dependency " + dependent_table_id.getNameForLogs(), ErrorCodes::TABLE_WAS_NOT_DROPPED);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-08-25 01:40:24 +00:00
|
|
|
void StorageLiveView::noUsersThread(std::shared_ptr<StorageLiveView> storage, const UInt64 & timeout)
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
|
|
|
bool drop_table = false;
|
|
|
|
|
2019-08-25 11:36:08 +00:00
|
|
|
if (storage->shutdown_called)
|
2019-08-25 01:40:24 +00:00
|
|
|
return;
|
|
|
|
|
2019-12-03 16:25:32 +00:00
|
|
|
auto table_id = storage->getStorageID();
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2020-03-08 21:29:00 +00:00
|
|
|
while (true)
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
std::unique_lock lock(storage->no_users_thread_wakeup_mutex);
|
2019-08-25 12:27:47 +00:00
|
|
|
if (!storage->no_users_thread_condition.wait_for(lock, std::chrono::seconds(timeout), [&] { return storage->no_users_thread_wakeup; }))
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-08-25 01:40:24 +00:00
|
|
|
storage->no_users_thread_wakeup = false;
|
2019-08-25 11:36:08 +00:00
|
|
|
if (storage->shutdown_called)
|
2019-05-26 22:03:30 +00:00
|
|
|
return;
|
2019-08-25 01:40:24 +00:00
|
|
|
if (storage->hasUsers())
|
2019-05-26 22:03:30 +00:00
|
|
|
return;
|
2020-02-12 16:54:26 +00:00
|
|
|
if (!DatabaseCatalog::instance().getDependencies(table_id).empty())
|
2019-05-26 22:03:30 +00:00
|
|
|
continue;
|
|
|
|
drop_table = true;
|
|
|
|
}
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (drop_table)
|
|
|
|
{
|
2020-05-28 23:01:18 +00:00
|
|
|
if (DatabaseCatalog::instance().tryGetTable(table_id, storage->global_context))
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
|
|
|
/// We create and execute `drop` query for this table
|
|
|
|
auto drop_query = std::make_shared<ASTDropQuery>();
|
2019-12-03 16:25:32 +00:00
|
|
|
drop_query->database = table_id.database_name;
|
|
|
|
drop_query->table = table_id.table_name;
|
2019-06-03 19:53:11 +00:00
|
|
|
drop_query->kind = ASTDropQuery::Kind::Drop;
|
2019-05-26 22:03:30 +00:00
|
|
|
ASTPtr ast_drop_query = drop_query;
|
2019-08-25 01:40:24 +00:00
|
|
|
InterpreterDropQuery drop_interpreter(ast_drop_query, storage->global_context);
|
2019-05-26 22:03:30 +00:00
|
|
|
drop_interpreter.execute();
|
|
|
|
}
|
|
|
|
catch (...)
|
|
|
|
{
|
2019-12-03 16:25:32 +00:00
|
|
|
tryLogCurrentException(__PRETTY_FUNCTION__);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-07 01:10:56 +00:00
|
|
|
void StorageLiveView::startNoUsersThread(const UInt64 & timeout)
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
|
|
|
bool expected = false;
|
2019-08-22 22:41:30 +00:00
|
|
|
if (!start_no_users_thread_called.compare_exchange_strong(expected, true))
|
2019-05-26 22:03:30 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
if (is_temporary)
|
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
std::lock_guard no_users_thread_lock(no_users_thread_mutex);
|
|
|
|
|
|
|
|
if (shutdown_called)
|
|
|
|
return;
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
if (no_users_thread.joinable())
|
|
|
|
{
|
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
std::lock_guard lock(no_users_thread_wakeup_mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
no_users_thread_wakeup = true;
|
2019-08-22 23:22:57 +00:00
|
|
|
no_users_thread_condition.notify_one();
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
no_users_thread.join();
|
|
|
|
}
|
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
std::lock_guard lock(no_users_thread_wakeup_mutex);
|
2019-08-22 22:41:30 +00:00
|
|
|
no_users_thread_wakeup = false;
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
if (!is_dropped)
|
2019-08-25 12:27:47 +00:00
|
|
|
no_users_thread = std::thread(&StorageLiveView::noUsersThread,
|
|
|
|
std::static_pointer_cast<StorageLiveView>(shared_from_this()), timeout);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2019-08-25 12:27:47 +00:00
|
|
|
|
2019-08-22 22:41:30 +00:00
|
|
|
start_no_users_thread_called = false;
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::startup()
|
|
|
|
{
|
2019-06-07 01:10:56 +00:00
|
|
|
startNoUsersThread(temporary_live_view_timeout);
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void StorageLiveView::shutdown()
|
|
|
|
{
|
2020-02-12 16:54:26 +00:00
|
|
|
DatabaseCatalog::instance().removeDependency(select_table_id, getStorageID());
|
2019-05-26 22:03:30 +00:00
|
|
|
bool expected = false;
|
|
|
|
if (!shutdown_called.compare_exchange_strong(expected, true))
|
|
|
|
return;
|
|
|
|
|
2020-05-12 18:10:10 +00:00
|
|
|
/// WATCH queries should be stopped after setting shutdown_called to true.
|
|
|
|
/// Otherwise livelock is possible for LiveView table in Atomic database:
|
|
|
|
/// WATCH query will wait for table to be dropped and DatabaseCatalog will wait for queries to finish
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
std::lock_guard no_users_thread_lock(no_users_thread_mutex);
|
|
|
|
if (no_users_thread.joinable())
|
2019-08-25 01:40:24 +00:00
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
{
|
|
|
|
std::lock_guard lock(no_users_thread_wakeup_mutex);
|
|
|
|
no_users_thread_wakeup = true;
|
|
|
|
no_users_thread_condition.notify_one();
|
|
|
|
}
|
2019-08-25 01:40:24 +00:00
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
StorageLiveView::~StorageLiveView()
|
|
|
|
{
|
|
|
|
shutdown();
|
2019-10-17 14:47:36 +00:00
|
|
|
|
|
|
|
{
|
|
|
|
std::lock_guard lock(no_users_thread_mutex);
|
|
|
|
if (no_users_thread.joinable())
|
|
|
|
no_users_thread.detach();
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-01-22 11:30:11 +00:00
|
|
|
void StorageLiveView::drop()
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-12-03 16:25:32 +00:00
|
|
|
auto table_id = getStorageID();
|
2020-02-12 16:54:26 +00:00
|
|
|
DatabaseCatalog::instance().removeDependency(select_table_id, table_id);
|
2019-08-22 23:22:57 +00:00
|
|
|
|
|
|
|
std::lock_guard lock(mutex);
|
2019-05-26 22:03:30 +00:00
|
|
|
is_dropped = true;
|
2019-08-22 23:22:57 +00:00
|
|
|
condition.notify_all();
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-04-01 12:43:09 +00:00
|
|
|
void StorageLiveView::refresh(const Context & context)
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2020-04-09 18:10:27 +00:00
|
|
|
auto alter_lock = lockAlterIntention(context.getCurrentQueryId(), context.getSettingsRef().lock_acquire_timeout);
|
2020-04-01 12:43:09 +00:00
|
|
|
{
|
|
|
|
std::lock_guard lock(mutex);
|
|
|
|
if (getNewBlocks())
|
|
|
|
condition.notify_all();
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-02-19 16:07:28 +00:00
|
|
|
Pipes StorageLiveView::read(
|
2019-05-26 22:03:30 +00:00
|
|
|
const Names & /*column_names*/,
|
|
|
|
const SelectQueryInfo & /*query_info*/,
|
|
|
|
const Context & /*context*/,
|
|
|
|
QueryProcessingStage::Enum /*processed_stage*/,
|
|
|
|
const size_t /*max_block_size*/,
|
|
|
|
const unsigned /*num_streams*/)
|
|
|
|
{
|
2020-02-17 16:35:23 +00:00
|
|
|
Pipes pipes;
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(mutex);
|
2019-05-26 22:03:30 +00:00
|
|
|
if (!(*blocks_ptr))
|
|
|
|
{
|
|
|
|
if (getNewBlocks())
|
2019-08-22 23:22:57 +00:00
|
|
|
condition.notify_all();
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2020-02-17 16:35:23 +00:00
|
|
|
pipes.emplace_back(std::make_shared<BlocksSource>(blocks_ptr, getHeader()));
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2020-02-17 16:35:23 +00:00
|
|
|
return pipes;
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
BlockInputStreams StorageLiveView::watch(
|
|
|
|
const Names & /*column_names*/,
|
|
|
|
const SelectQueryInfo & query_info,
|
|
|
|
const Context & context,
|
|
|
|
QueryProcessingStage::Enum & processed_stage,
|
|
|
|
size_t /*max_block_size*/,
|
|
|
|
const unsigned /*num_streams*/)
|
|
|
|
{
|
|
|
|
ASTWatchQuery & query = typeid_cast<ASTWatchQuery &>(*query_info.query);
|
|
|
|
|
2019-07-09 12:41:55 +00:00
|
|
|
bool has_limit = false;
|
|
|
|
UInt64 limit = 0;
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
if (query.limit_length)
|
2019-07-09 12:41:55 +00:00
|
|
|
{
|
|
|
|
has_limit = true;
|
|
|
|
limit = safeGet<UInt64>(typeid_cast<ASTLiteral &>(*query.limit_length).value);
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2019-05-28 21:17:48 +00:00
|
|
|
if (query.is_watch_events)
|
2019-05-26 22:03:30 +00:00
|
|
|
{
|
2019-08-25 12:27:47 +00:00
|
|
|
auto reader = std::make_shared<LiveViewEventsBlockInputStream>(
|
|
|
|
std::static_pointer_cast<StorageLiveView>(shared_from_this()),
|
|
|
|
blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
|
|
|
|
context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
|
2020-02-14 18:20:20 +00:00
|
|
|
temporary_live_view_timeout);
|
2019-05-28 21:17:48 +00:00
|
|
|
|
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
std::lock_guard no_users_thread_lock(no_users_thread_mutex);
|
|
|
|
if (no_users_thread.joinable())
|
|
|
|
{
|
|
|
|
std::lock_guard lock(no_users_thread_wakeup_mutex);
|
|
|
|
no_users_thread_wakeup = true;
|
|
|
|
no_users_thread_condition.notify_one();
|
|
|
|
}
|
2019-05-28 21:17:48 +00:00
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
|
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(mutex);
|
2019-05-28 21:17:48 +00:00
|
|
|
if (!(*blocks_ptr))
|
|
|
|
{
|
2019-05-30 21:29:30 +00:00
|
|
|
if (getNewBlocks())
|
2019-08-22 23:22:57 +00:00
|
|
|
condition.notify_all();
|
2019-05-28 21:17:48 +00:00
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2019-05-28 21:17:48 +00:00
|
|
|
|
|
|
|
processed_stage = QueryProcessingStage::Complete;
|
|
|
|
|
|
|
|
return { reader };
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
2019-05-28 21:17:48 +00:00
|
|
|
else
|
|
|
|
{
|
2019-08-25 12:27:47 +00:00
|
|
|
auto reader = std::make_shared<LiveViewBlockInputStream>(
|
|
|
|
std::static_pointer_cast<StorageLiveView>(shared_from_this()),
|
|
|
|
blocks_ptr, blocks_metadata_ptr, active_ptr, has_limit, limit,
|
|
|
|
context.getSettingsRef().live_view_heartbeat_interval.totalSeconds(),
|
2020-02-14 18:20:20 +00:00
|
|
|
temporary_live_view_timeout);
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2019-05-28 21:17:48 +00:00
|
|
|
{
|
2019-10-17 14:47:36 +00:00
|
|
|
std::lock_guard no_users_thread_lock(no_users_thread_mutex);
|
|
|
|
if (no_users_thread.joinable())
|
|
|
|
{
|
|
|
|
std::lock_guard lock(no_users_thread_wakeup_mutex);
|
|
|
|
no_users_thread_wakeup = true;
|
|
|
|
no_users_thread_condition.notify_one();
|
|
|
|
}
|
2019-05-28 21:17:48 +00:00
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
|
2019-05-28 21:17:48 +00:00
|
|
|
{
|
2019-08-22 23:22:57 +00:00
|
|
|
std::lock_guard lock(mutex);
|
2019-05-28 21:17:48 +00:00
|
|
|
if (!(*blocks_ptr))
|
|
|
|
{
|
2019-05-30 21:29:30 +00:00
|
|
|
if (getNewBlocks())
|
2019-08-22 23:22:57 +00:00
|
|
|
condition.notify_all();
|
2019-05-28 21:17:48 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
processed_stage = QueryProcessingStage::Complete;
|
|
|
|
|
|
|
|
return { reader };
|
|
|
|
}
|
2019-05-26 22:03:30 +00:00
|
|
|
}
|
|
|
|
|
2020-04-28 10:38:57 +00:00
|
|
|
NamesAndTypesList StorageLiveView::getVirtuals() const
|
2020-04-27 13:55:30 +00:00
|
|
|
{
|
2020-04-28 10:38:57 +00:00
|
|
|
return NamesAndTypesList{
|
2020-04-27 13:55:30 +00:00
|
|
|
NameAndTypePair("_version", std::make_shared<DataTypeUInt64>())
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2019-05-26 22:03:30 +00:00
|
|
|
void registerStorageLiveView(StorageFactory & factory)
|
|
|
|
{
|
|
|
|
factory.registerStorage("LiveView", [](const StorageFactory::Arguments & args)
|
|
|
|
{
|
2019-08-30 23:38:03 +00:00
|
|
|
if (!args.attach && !args.local_context.getSettingsRef().allow_experimental_live_view)
|
2019-08-22 23:46:40 +00:00
|
|
|
throw Exception("Experimental LIVE VIEW feature is not enabled (the setting 'allow_experimental_live_view')", ErrorCodes::SUPPORT_IS_DISABLED);
|
|
|
|
|
2019-12-04 16:06:55 +00:00
|
|
|
return StorageLiveView::create(args.table_id, args.local_context, args.query, args.columns);
|
2019-05-26 22:03:30 +00:00
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|