thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-17 13:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Primary key in storage metadata

Browse Source
This commit is contained in:
alesapin 2020-06-17 15:39:20 +03:00
parent 1da393b218
commit 1afdebeebd
28 changed files with 114 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -330,7 +330,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
current_info.query = query_ptr;
current_info.syntax_analyzer_result = syntax_analyzer_result;
MergeTreeWhereOptimizer{current_info, *context, *merge_tree, syntax_analyzer_result->requiredSourceColumns(), log};
MergeTreeWhereOptimizer{current_info, *context, *merge_tree, metadata_snapshot, syntax_analyzer_result->requiredSourceColumns(), log};
}
}

29
src/Storages/IStorage.cpp
View File

@ -319,35 +319,6 @@ NamesAndTypesList IStorage::getVirtuals() const
return {};
}
const KeyDescription & IStorage::getPrimaryKey() const
{
return metadata->primary_key;
}
bool IStorage::isPrimaryKeyDefined() const
{
return metadata->primary_key.definition_ast != nullptr;
}
bool IStorage::hasPrimaryKey() const
{
return !metadata->primary_key.column_names.empty();
}
Names IStorage::getColumnsRequiredForPrimaryKey() const
{
if (hasPrimaryKey())
return metadata->primary_key.expression->getRequiredColumns();
return {};
}
Names IStorage::getPrimaryKeyColumns() const
{
if (!metadata->primary_key.column_names.empty())
return metadata->primary_key.column_names;
return {};
}
TTLTableDescription IStorage::getTableTTLs() const
{
std::lock_guard lock(ttl_mutex);

15
src/Storages/IStorage.h
View File

@ -427,21 +427,6 @@ public:
/// Returns data paths if storage supports it, empty vector otherwise.
virtual Strings getDataPaths() const { return {}; }
/// Returns structure with primary key.
const KeyDescription & getPrimaryKey() const;
/// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none.
ASTPtr getPrimaryKeyAST() const { return metadata->primary_key.definition_ast; }
/// Storage has user-defined (in CREATE query) sorting key.
bool isPrimaryKeyDefined() const;
/// Storage has primary key (maybe part of some other key). It means, that
/// it contains at least one column.
bool hasPrimaryKey() const;
/// Returns column names that need to be read to calculate primary key.
Names getColumnsRequiredForPrimaryKey() const;
/// Returns columns names in sorting key specified by. For example: 'a', 'x
/// * y', 'toStartOfMonth(date)', etc.
Names getPrimaryKeyColumns() const;
/// Returns storage policy if storage supports it.
virtual StoragePolicyPtr getStoragePolicy() const { return {}; }

5
src/Storages/MergeTree/IMergeTreeDataPart.cpp
View File

@ -437,7 +437,8 @@ void IMergeTreeDataPart::loadIndex()
if (!index_granularity.isInitialized())
throw Exception("Index granularity is not loaded before index loading", ErrorCodes::LOGICAL_ERROR);
const auto & primary_key = storage.getPrimaryKey();
auto metadata_snapshot = storage.getInMemoryMetadataPtr();
const auto & primary_key = metadata_snapshot->getPrimaryKey();
size_t key_size = primary_key.column_names.size();
if (key_size)
@ -842,7 +843,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
String path = getFullRelativePath();
auto metadata_snapshot = storage.getInMemoryMetadataPtr();
const auto & pk = storage.getPrimaryKey();
const auto & pk = metadata_snapshot->getPrimaryKey();
if (!checksums.empty())
{
if (!pk.column_names.empty() && !checksums.files.count("primary.idx"))

1
src/Storages/MergeTree/IMergeTreeDataPart.h
View File

@ -86,6 +86,7 @@ public:
virtual MergeTreeWriterPtr getWriter(
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const CompressionCodecPtr & default_codec_,
const MergeTreeWriterSettings & writer_settings,

6
src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp
View File

@ -65,6 +65,7 @@ void IMergeTreeDataPartWriter::Stream::addToChecksums(MergeTreeData::DataPart::C
IMergeTreeDataPartWriter::IMergeTreeDataPartWriter(
const MergeTreeData::DataPartPtr & data_part_,
const NamesAndTypesList & columns_list_,
const StorageMetadataPtr & metadata_snapshot_,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc_,
const String & marks_file_extension_,
const CompressionCodecPtr & default_codec_,
@ -73,6 +74,7 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter(
: data_part(data_part_)
, part_path(data_part_->getFullRelativePath())
, storage(data_part_->storage)
, metadata_snapshot(metadata_snapshot_)
, columns_list(columns_list_)
, marks_file_extension(marks_file_extension_)
, index_granularity(index_granularity_)
@ -162,7 +164,7 @@ void IMergeTreeDataPartWriter::fillIndexGranularity(size_t index_granularity_for
void IMergeTreeDataPartWriter::initPrimaryIndex()
{
if (storage.hasPrimaryKey())
if (metadata_snapshot->hasPrimaryKey())
{
index_file_stream = data_part->volume->getDisk()->writeFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite);
index_stream = std::make_unique<HashingWriteBuffer>(*index_file_stream);
@ -221,7 +223,7 @@ void IMergeTreeDataPartWriter::calculateAndSerializePrimaryIndex(const Block & p
while (index_mark < total_marks && current_row < rows)
{
if (storage.hasPrimaryKey())
if (metadata_snapshot->hasPrimaryKey())
{
for (size_t j = 0; j < primary_columns_num; ++j)
{

2
src/Storages/MergeTree/IMergeTreeDataPartWriter.h
View File

@ -63,6 +63,7 @@ public:
IMergeTreeDataPartWriter(
const MergeTreeData::DataPartPtr & data_part,
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot_,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const String & marks_file_extension,
const CompressionCodecPtr & default_codec,
@ -119,6 +120,7 @@ protected:
MergeTreeData::DataPartPtr data_part;
String part_path;
const MergeTreeData & storage;
StorageMetadataPtr metadata_snapshot;
NamesAndTypesList columns_list;
const String marks_file_extension;

19
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -173,11 +173,11 @@ MergeTreeData::MergeTreeData(
const auto settings = getSettings();
/// NOTE: using the same columns list as is read when performing actual merges.
merging_params.check(getColumns().getAllPhysical());
merging_params.check(metadata_.getColumns().getAllPhysical());
if (metadata_.sampling_key.definition_ast != nullptr)
{
const auto & pk_sample_block = getPrimaryKey().sample_block;
const auto & pk_sample_block = metadata_.getPrimaryKey().sample_block;
if (!pk_sample_block.has(metadata_.sampling_key.column_names[0]) && !attach
&& !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility.
throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS);
@ -410,7 +410,7 @@ ExpressionActionsPtr getCombinedIndicesExpression(
ExpressionActionsPtr MergeTreeData::getPrimaryKeyAndSkipIndicesExpression(const StorageMetadataPtr & metadata_snapshot) const
{
return getCombinedIndicesExpression(getPrimaryKey(), metadata_snapshot->getSecondaryIndices(), metadata_snapshot->getColumns(), global_context);
return getCombinedIndicesExpression(metadata_snapshot->getPrimaryKey(), metadata_snapshot->getSecondaryIndices(), metadata_snapshot->getColumns(), global_context);
}
ExpressionActionsPtr MergeTreeData::getSortingKeyAndSkipIndicesExpression(const StorageMetadataPtr & metadata_snapshot) const
@ -2915,11 +2915,12 @@ MergeTreeData::DataPartsVector MergeTreeData::Transaction::commit(MergeTreeData:
return total_covered_parts;
}
bool MergeTreeData::isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node) const
bool MergeTreeData::isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(
const ASTPtr & node, const StorageMetadataPtr & metadata_snapshot) const
{
const String column_name = node->getColumnName();
for (const auto & name : getPrimaryKeyColumns())
for (const auto & name : metadata_snapshot->getPrimaryKeyColumns())
if (column_name == name)
return true;
@ -2929,7 +2930,7 @@ bool MergeTreeData::isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const A
if (const auto * func = node->as<ASTFunction>())
if (func->arguments->children.size() == 1)
return isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(func->arguments->children.front());
return isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(func->arguments->children.front(), metadata_snapshot);
return false;
}
@ -2946,14 +2947,14 @@ bool MergeTreeData::mayBenefitFromIndexForIn(
{
for (const auto & item : left_in_operand_tuple->arguments->children)
{
if (isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(item))
if (isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(item, metadata_snapshot))
return true;
for (const auto & index : metadata_snapshot->getSecondaryIndices())
if (index_wrapper_factory.get(index)->mayBenefitFromIndexForIn(item))
return true;
}
/// The tuple itself may be part of the primary key, so check that as a last resort.
return isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(left_in_operand);
return isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(left_in_operand, metadata_snapshot);
}
else
{
@ -2961,7 +2962,7 @@ bool MergeTreeData::mayBenefitFromIndexForIn(
if (index_wrapper_factory.get(index)->mayBenefitFromIndexForIn(left_in_operand))
return true;
return isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(left_in_operand);
return isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(left_in_operand, metadata_snapshot);
}
}

2
src/Storages/MergeTree/MergeTreeData.h
View File

@ -812,7 +812,7 @@ protected:
DataPartsLock & data_parts_lock) const;
/// Checks whether the column is in the primary key, possibly wrapped in a chain of functions with single argument.
bool isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node) const;
bool isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const ASTPtr & node, const StorageMetadataPtr & metadata_snapshot) const;
/// Common part for |freezePartition()| and |freezeAll()|.
using MatcherFn = std::function<bool(const DataPartPtr &)>;

2
src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
View File

@ -1604,7 +1604,7 @@ void MergeTreeDataMergerMutator::mutateAllPartColumns(
if (mutating_stream == nullptr)
throw Exception("Cannot mutate part columns with uninitialized mutations stream. It's a bug", ErrorCodes::LOGICAL_ERROR);
if (data.hasPrimaryKey() || metadata_snapshot->hasSecondaryIndices())
if (metadata_snapshot->hasPrimaryKey() || metadata_snapshot->hasSecondaryIndices())
mutating_stream = std::make_shared<MaterializingBlockInputStream>(
std::make_shared<ExpressionBlockInputStream>(mutating_stream, data.getPrimaryKeyAndSkipIndicesExpression(metadata_snapshot)));

5
src/Storages/MergeTree/MergeTreeDataPartCompact.cpp
View File

@ -54,6 +54,7 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartCompact::getReader(
IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter(
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const CompressionCodecPtr & default_codec,
const MergeTreeWriterSettings & writer_settings,
@ -68,8 +69,8 @@ IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartCompact::getWriter(
{ return *getColumnPosition(lhs.name) < *getColumnPosition(rhs.name); });
return std::make_unique<MergeTreeDataPartWriterCompact>(
shared_from_this(), ordered_columns_list, indices_to_recalc,
index_granularity_info.marks_file_extension,
shared_from_this(), ordered_columns_list, metadata_snapshot,
indices_to_recalc, index_granularity_info.marks_file_extension,
default_codec, writer_settings, computed_index_granularity);
}

1
src/Storages/MergeTree/MergeTreeDataPartCompact.h
View File

@ -46,6 +46,7 @@ public:
MergeTreeWriterPtr getWriter(
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const CompressionCodecPtr & default_codec_,
const MergeTreeWriterSettings & writer_settings,

3
src/Storages/MergeTree/MergeTreeDataPartWide.cpp
View File

@ -53,13 +53,14 @@ IMergeTreeDataPart::MergeTreeReaderPtr MergeTreeDataPartWide::getReader(
IMergeTreeDataPart::MergeTreeWriterPtr MergeTreeDataPartWide::getWriter(
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const CompressionCodecPtr & default_codec,
const MergeTreeWriterSettings & writer_settings,
const MergeTreeIndexGranularity & computed_index_granularity) const
{
return std::make_unique<MergeTreeDataPartWriterWide>(
shared_from_this(), columns_list, indices_to_recalc,
shared_from_this(), columns_list, metadata_snapshot, indices_to_recalc,
index_granularity_info.marks_file_extension,
default_codec, writer_settings, computed_index_granularity);
}

1
src/Storages/MergeTree/MergeTreeDataPartWide.h
View File

@ -39,6 +39,7 @@ public:
MergeTreeWriterPtr getWriter(
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const CompressionCodecPtr & default_codec_,
const MergeTreeWriterSettings & writer_settings,

8
src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp
View File

@ -3,19 +3,17 @@
namespace DB
{
MergeTreeDataPartWriterCompact::MergeTreeDataPartWriterCompact(
const MergeTreeData::DataPartPtr & data_part_,
const NamesAndTypesList & columns_list_,
const StorageMetadataPtr & metadata_snapshot_,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc_,
const String & marks_file_extension_,
const CompressionCodecPtr & default_codec_,
const MergeTreeWriterSettings & settings_,
const MergeTreeIndexGranularity & index_granularity_)
: IMergeTreeDataPartWriter(data_part_, columns_list_,
indices_to_recalc_, marks_file_extension_,
default_codec_, settings_, index_granularity_)
: IMergeTreeDataPartWriter(
data_part_, columns_list_, metadata_snapshot_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_)
{
using DataPart = MergeTreeDataPartCompact;
String data_file_name = DataPart::DATA_FILE_NAME;

1
src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h
View File

@ -10,6 +10,7 @@ public:
MergeTreeDataPartWriterCompact(
const MergeTreeData::DataPartPtr & data_part,
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot_,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const String & marks_file_extension,
const CompressionCodecPtr & default_codec,

8
src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp
View File

@ -16,16 +16,16 @@ namespace
MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide(
const MergeTreeData::DataPartPtr & data_part_,
const NamesAndTypesList & columns_list_,
const StorageMetadataPtr & metadata_snapshot_,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc_,
const String & marks_file_extension_,
const CompressionCodecPtr & default_codec_,
const MergeTreeWriterSettings & settings_,
const MergeTreeIndexGranularity & index_granularity_)
: IMergeTreeDataPartWriter(data_part_, columns_list_,
indices_to_recalc_, marks_file_extension_,
default_codec_, settings_, index_granularity_)
: IMergeTreeDataPartWriter(
data_part_, columns_list_, metadata_snapshot_, indices_to_recalc_, marks_file_extension_, default_codec_, settings_, index_granularity_)
{
const auto & columns = storage.getColumns();
const auto & columns = metadata_snapshot->getColumns();
for (const auto & it : columns_list)
addStreams(it.name, *it.type, columns.getCodecOrDefault(it.name, default_codec), settings.estimated_size);
}

1
src/Storages/MergeTree/MergeTreeDataPartWriterWide.h
View File

@ -13,6 +13,7 @@ public:
MergeTreeDataPartWriterWide(
const MergeTreeData::DataPartPtr & data_part,
const NamesAndTypesList & columns_list,
const StorageMetadataPtr & metadata_snapshot,
const std::vector<MergeTreeIndexPtr> & indices_to_recalc,
const String & marks_file_extension,
const CompressionCodecPtr & default_codec,

22
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -100,7 +100,10 @@ static Block getBlockWithPartColumn(const MergeTreeData::DataPartsVector & parts
size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead(
const MergeTreeData::DataPartsVector & parts, const KeyCondition & key_condition, const Settings & settings) const
const MergeTreeData::DataPartsVector & parts,
const StorageMetadataPtr & metadata_snapshot,
const KeyCondition & key_condition,
const Settings & settings) const
{
size_t rows_count = 0;
@ -109,7 +112,7 @@ size_t MergeTreeDataSelectExecutor::getApproximateTotalRowsToRead(
for (const auto & part : parts)
{
MarkRanges ranges = markRangesFromPKRange(part, key_condition, settings);
MarkRanges ranges = markRangesFromPKRange(part, metadata_snapshot, key_condition, settings);
/** In order to get a lower bound on the number of rows that match the condition on PK,
* consider only guaranteed full marks.
@ -224,7 +227,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
data.check(real_column_names);
const Settings & settings = context.getSettingsRef();
const auto & primary_key = data.getPrimaryKey();
const auto & primary_key = metadata_snapshot->getPrimaryKey();
Names primary_key_columns = primary_key.column_names;
KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expression);
@ -326,7 +329,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
/// Convert absolute value of the sampling (in form `SAMPLE 1000000` - how many rows to read) into the relative `SAMPLE 0.1` (how much data to read).
size_t approx_total_rows = 0;
if (relative_sample_size > 1 || relative_sample_offset > 1)
approx_total_rows = getApproximateTotalRowsToRead(parts, key_condition, settings);
approx_total_rows = getApproximateTotalRowsToRead(parts, metadata_snapshot, key_condition, settings);
if (relative_sample_size > 1)
{
@ -565,8 +568,8 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
{
RangesInDataPart ranges(part, part_index++);
if (data.hasPrimaryKey())
ranges.ranges = markRangesFromPKRange(part, key_condition, settings);
if (metadata_snapshot->hasPrimaryKey())
ranges.ranges = markRangesFromPKRange(part, metadata_snapshot, key_condition, settings);
else
{
size_t total_marks_count = part->getMarksCount();
@ -1297,7 +1300,10 @@ void MergeTreeDataSelectExecutor::createPositiveSignCondition(
/// Calculates a set of mark ranges, that could possibly contain keys, required by condition.
/// In other words, it removes subranges from whole range, that definitely could not contain required keys.
MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
const MergeTreeData::DataPartPtr & part, const KeyCondition & key_condition, const Settings & settings) const
const MergeTreeData::DataPartPtr & part,
const StorageMetadataPtr & metadata_snapshot,
const KeyCondition & key_condition,
const Settings & settings) const
{
MarkRanges res;
@ -1335,7 +1341,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
std::function<void(size_t, size_t, FieldRef &)> create_field_ref;
/// If there are no monotonic functions, there is no need to save block reference.
/// Passing explicit field to FieldRef allows to optimize ranges and shows better performance.
const auto & primary_key = data.getPrimaryKey();
const auto & primary_key = metadata_snapshot->getPrimaryKey();
if (key_condition.hasMonotonicFunctionsChain())
{
auto index_block = std::make_shared<Block>();

2
src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
View File

@ -91,6 +91,7 @@ private:
/// Get the approximate value (bottom estimate - only by full marks) of the number of rows falling under the index.
size_t getApproximateTotalRowsToRead(
const MergeTreeData::DataPartsVector & parts,
const StorageMetadataPtr & metadata_snapshot,
const KeyCondition & key_condition,
const Settings & settings) const;
@ -102,6 +103,7 @@ private:
MarkRanges markRangesFromPKRange(
const MergeTreeData::DataPartPtr & part,
const StorageMetadataPtr & metadata_snapshot,
const KeyCondition & key_condition,
const Settings & settings) const;

13
src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
View File

@ -31,15 +31,16 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
SelectQueryInfo & query_info,
const Context & context,
const MergeTreeData & data,
const StorageMetadataPtr & metadata_snapshot,
const Names & queried_columns_,
Poco::Logger * log_)
: table_columns{ext::map<std::unordered_set>(data.getColumns().getAllPhysical(),
[] (const NameAndTypePair & col) { return col.name; })},
queried_columns{queried_columns_},
block_with_constants{KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context)},
log{log_}
: table_columns{ext::map<std::unordered_set>(
metadata_snapshot->getColumns().getAllPhysical(), [](const NameAndTypePair & col) { return col.name; })}
, queried_columns{queried_columns_}
, block_with_constants{KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context)}
, log{log_}
{
const auto & primary_key = data.getPrimaryKey();
const auto & primary_key = metadata_snapshot->getPrimaryKey();
if (!primary_key.column_names.empty())
first_primary_key_column = primary_key.column_names[0];

3
src/Storages/MergeTree/MergeTreeWhereOptimizer.h
View File

@ -16,6 +16,8 @@ namespace DB
class ASTSelectQuery;
class ASTFunction;
class MergeTreeData;
struct StorageInMemoryMetadata;
using StorageMetadataPtr = std::shared_ptr<StorageInMemoryMetadata>;
/** Identifies WHERE expressions that can be placed in PREWHERE by calculating respective
* sizes of columns used in particular expression and identifying "good" conditions of
@ -31,6 +33,7 @@ public:
SelectQueryInfo & query_info,
const Context & context,
const MergeTreeData & data,
const StorageMetadataPtr & metadata_snapshot,
const Names & queried_columns_,
Poco::Logger * log_);

4
src/Storages/MergeTree/MergedBlockOutputStream.cpp
View File

@ -59,7 +59,7 @@ MergedBlockOutputStream::MergedBlockOutputStream(
volume->getDisk()->createDirectories(part_path);
writer = data_part->getWriter(columns_list, skip_indices, default_codec, writer_settings);
writer = data_part->getWriter(columns_list, metadata_snapshot, skip_indices, default_codec, writer_settings);
writer->initPrimaryIndex();
writer->initSkipIndices();
}
@ -169,7 +169,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm
std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end()));
Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end());
Block primary_key_block = getBlockAndPermute(block, storage.getPrimaryKeyColumns(), permutation);
Block primary_key_block = getBlockAndPermute(block, metadata_snapshot->getPrimaryKeyColumns(), permutation);
Block skip_indexes_block = getBlockAndPermute(block, skip_indexes_column_names, permutation);
writer->write(block, permutation, primary_key_block, skip_indexes_block);

1
src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp
View File

@ -28,6 +28,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream(
writer = data_part->getWriter(
header.getNamesAndTypesList(),
metadata_snapshot_,
indices_to_recalc,
default_codec,
std::move(writer_settings),

4
src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp
View File

@ -40,11 +40,11 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
/// So rules in zookeeper metadata is following:
/// - When we have only ORDER BY, than store it in "primary key:" row of /metadata
/// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata
if (!data.isPrimaryKeyDefined())
if (!metadata_snapshot->isPrimaryKeyDefined())
primary_key = formattedAST(metadata_snapshot->getSortingKey().expression_list_ast);
else
{
primary_key = formattedAST(data.getPrimaryKey().expression_list_ast);
primary_key = formattedAST(metadata_snapshot->getPrimaryKey().expression_list_ast);
sorting_key = formattedAST(metadata_snapshot->getSortingKey().expression_list_ast);
}

28
src/Storages/StorageInMemoryMetadata.cpp
View File

@ -365,4 +365,32 @@ Names StorageInMemoryMetadata::getColumnsRequiredForSampling() const
return {};
}
const KeyDescription & StorageInMemoryMetadata::getPrimaryKey() const
{
return primary_key;
}
bool StorageInMemoryMetadata::isPrimaryKeyDefined() const
{
return primary_key.definition_ast != nullptr;
}
bool StorageInMemoryMetadata::hasPrimaryKey() const
{
return !primary_key.column_names.empty();
}
Names StorageInMemoryMetadata::getColumnsRequiredForPrimaryKey() const
{
if (hasPrimaryKey())
return primary_key.expression->getRequiredColumns();
return {};
}
Names StorageInMemoryMetadata::getPrimaryKeyColumns() const
{
if (!primary_key.column_names.empty())
return primary_key.column_names;
return {};
}
}

15
src/Storages/StorageInMemoryMetadata.h
View File

@ -151,6 +151,21 @@ struct StorageInMemoryMetadata
bool hasSamplingKey() const;
/// Returns column names that need to be read to calculate sampling key.
Names getColumnsRequiredForSampling() const;
/// Returns structure with primary key.
const KeyDescription & getPrimaryKey() const;
/// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none.
ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; }
/// Storage has user-defined (in CREATE query) sorting key.
bool isPrimaryKeyDefined() const;
/// Storage has primary key (maybe part of some other key). It means, that
/// it contains at least one column.
bool hasPrimaryKey() const;
/// Returns column names that need to be read to calculate primary key.
Names getColumnsRequiredForPrimaryKey() const;
/// Returns columns names in sorting key specified by. For example: 'a', 'x
/// * y', 'toStartOfMonth(date)', etc.
Names getPrimaryKeyColumns() const;
};
using StorageMetadataPtr = std::shared_ptr<StorageInMemoryMetadata>;

2
src/Storages/System/StorageSystemTables.cpp
View File

@ -384,7 +384,7 @@ protected:
if (columns_mask[src_index++])
{
assert(table != nullptr);
if ((expression_ptr = table->getPrimaryKey().expression_list_ast))
if ((expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
res_columns[res_index++]->insert(queryToString(expression_ptr));
else
res_columns[res_index++]->insertDefault();