mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
Sorting and primary key (broken)
This commit is contained in:
parent
9fb28f5ac0
commit
616902a995
@ -224,10 +224,8 @@ static NameSet getKeyColumns(const StoragePtr & storage)
|
||||
for (const String & col : merge_tree_data->getColumnsRequiredForPartitionKey())
|
||||
key_columns.insert(col);
|
||||
|
||||
auto sorting_key_expr = merge_tree_data->sorting_key_expr;
|
||||
if (sorting_key_expr)
|
||||
for (const String & col : sorting_key_expr->getRequiredColumns())
|
||||
key_columns.insert(col);
|
||||
for (const String & col : merge_tree_data->getColumnsRequiredForSortingKey())
|
||||
key_columns.insert(col);
|
||||
/// We don't process sample_by_ast separately because it must be among the primary key columns.
|
||||
|
||||
if (!merge_tree_data->merging_params.sign_column.empty())
|
||||
|
@ -460,7 +460,7 @@ Names IStorage::getColumnsRequiredForSampling() const
|
||||
|
||||
bool IStorage::hasPartitionKey() const
|
||||
{
|
||||
return partition_key.expressions != nullptr;
|
||||
return partition_key.definition_ast != nullptr;
|
||||
}
|
||||
|
||||
|
||||
@ -469,6 +469,21 @@ bool IStorage::supportsSampling() const
|
||||
return hasSamplingKey();
|
||||
}
|
||||
|
||||
Names IStorage::getColumnsRequiredForSortingKey() const
|
||||
{
|
||||
if (hasSortingKey())
|
||||
return sorting_key.expressions->getRequiredColumns();
|
||||
return {};
|
||||
}
|
||||
|
||||
|
||||
Names IStorage::getSortingKeyColumns() const
|
||||
{
|
||||
if (hasSortingKey())
|
||||
return sorting_key.expression_column_names;
|
||||
return {};
|
||||
}
|
||||
|
||||
const StorageMetadataKeyField & IStorage::getSamplingKey() const
|
||||
{
|
||||
return sampling_key;
|
||||
@ -480,7 +495,43 @@ void IStorage::setSamplingKey(const StorageMetadataKeyField & sampling_key_)
|
||||
|
||||
bool IStorage::hasSamplingKey() const
|
||||
{
|
||||
return sampling_key.expressions != nullptr;
|
||||
return sampling_key.definition_ast != nullptr;
|
||||
}
|
||||
|
||||
bool IStorage::hasSortingKey() const
|
||||
{
|
||||
return sorting_key.definition_ast != nullptr;
|
||||
}
|
||||
|
||||
void IStorage::setSortingKey(const StorageMetadataKeyField & sorting_key_)
|
||||
{
|
||||
sorting_key = sorting_key_;
|
||||
}
|
||||
|
||||
const StorageMetadataKeyField & IStorage::getSortingKey() const
|
||||
{
|
||||
return sorting_key;
|
||||
}
|
||||
|
||||
const StorageMetadataKeyField & IStorage::getPrimaryKey() const
|
||||
{
|
||||
return primary_key;
|
||||
}
|
||||
|
||||
void IStorage::setPrimaryKey(const StorageMetadataKeyField & primary_key_)
|
||||
{
|
||||
primary_key = primary_key_;
|
||||
}
|
||||
|
||||
bool IStorage::hasPrimaryKey() const
|
||||
{
|
||||
return primary_key.definition_ast != nullptr;
|
||||
}
|
||||
|
||||
Names IStorage::getColumnsRequiredForPrimaryKey() const
|
||||
{
|
||||
if (hasPrimaryKey())
|
||||
return primary_key.expressions->getRequiredColumns();
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
@ -192,6 +192,14 @@ public: /// thread-unsafe part. lockStructure must be acquired
|
||||
void setSamplingKey(const StorageMetadataKeyField & sampling_key_);
|
||||
bool hasSamplingKey() const;
|
||||
|
||||
const StorageMetadataKeyField & getSortingKey() const;
|
||||
void setSortingKey(const StorageMetadataKeyField & sorting_key_);
|
||||
bool hasSortingKey() const;
|
||||
|
||||
const StorageMetadataKeyField & getPrimaryKey() const;
|
||||
void setPrimaryKey(const StorageMetadataKeyField & primary_key_);
|
||||
bool hasPrimaryKey() const;
|
||||
|
||||
|
||||
protected: /// still thread-unsafe part.
|
||||
void setIndices(IndicesDescription indices_);
|
||||
@ -210,8 +218,8 @@ private:
|
||||
ConstraintsDescription constraints;
|
||||
|
||||
StorageMetadataKeyField partition_key;
|
||||
//StorageMetadataKeyField primary_key;
|
||||
//StorageMetadataKeyField sorting_key;
|
||||
StorageMetadataKeyField primary_key;
|
||||
StorageMetadataKeyField sorting_key;
|
||||
StorageMetadataKeyField sampling_key;
|
||||
|
||||
//StorageMetadataField rows_ttl_entry;
|
||||
@ -464,10 +472,10 @@ public:
|
||||
virtual ASTPtr getPartitionKeyAST() const { return partition_key.definition_ast; }
|
||||
|
||||
/// Returns ASTExpressionList of sorting key expression for storage or nullptr if there is none.
|
||||
virtual ASTPtr getSortingKeyAST() const { return nullptr; }
|
||||
virtual ASTPtr getSortingKeyAST() const { return sorting_key.definition_ast; }
|
||||
|
||||
/// Returns ASTExpressionList of primary key expression for storage or nullptr if there is none.
|
||||
virtual ASTPtr getPrimaryKeyAST() const { return nullptr; }
|
||||
virtual ASTPtr getPrimaryKeyAST() const { return primary_key.definition_ast; }
|
||||
|
||||
/// Returns sampling expression AST for storage or nullptr if there is none.
|
||||
virtual ASTPtr getSamplingKeyAST() const { return sampling_key.definition_ast; }
|
||||
@ -476,20 +484,20 @@ public:
|
||||
virtual Names getColumnsRequiredForPartitionKey() const;
|
||||
|
||||
/// Returns column names that need to be read to calculate sorting key.
|
||||
virtual Names getColumnsRequiredForSortingKey() const { return {}; }
|
||||
virtual Names getColumnsRequiredForSortingKey() const;
|
||||
|
||||
/// Returns column names that need to be read to calculate primary key.
|
||||
virtual Names getColumnsRequiredForPrimaryKey() const { return {}; }
|
||||
virtual Names getColumnsRequiredForPrimaryKey() const;
|
||||
|
||||
/// Returns column names that need to be read to calculate sampling key.
|
||||
virtual Names getColumnsRequiredForSampling() const;
|
||||
|
||||
/// Returns column names that need to be read for FINAL to work.
|
||||
virtual Names getColumnsRequiredForFinal() const { return {}; }
|
||||
virtual Names getColumnsRequiredForFinal() const { return getColumnsRequiredForSortingKey(); }
|
||||
|
||||
/// Returns columns names in sorting key specified by user in ORDER BY
|
||||
/// expression. For example: 'a', 'x * y', 'toStartOfMonth(date)', etc.
|
||||
virtual Names getSortingKeyColumns() const { return {}; }
|
||||
virtual Names getSortingKeyColumns() const;
|
||||
|
||||
/// Returns columns, which will be needed to calculate dependencies
|
||||
/// (skip indices, TTL expressions) if we update @updated_columns set of columns.
|
||||
|
@ -438,7 +438,8 @@ void IMergeTreeDataPart::loadIndex()
|
||||
if (!index_granularity.isInitialized())
|
||||
throw Exception("Index granularity is not loaded before index loading", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
size_t key_size = storage.primary_key_columns.size();
|
||||
const auto & primary_key = storage.getPrimaryKey();
|
||||
size_t key_size = primary_key.expression_column_names.size();
|
||||
|
||||
if (key_size)
|
||||
{
|
||||
@ -447,7 +448,7 @@ void IMergeTreeDataPart::loadIndex()
|
||||
|
||||
for (size_t i = 0; i < key_size; ++i)
|
||||
{
|
||||
loaded_index[i] = storage.primary_key_data_types[i]->createColumn();
|
||||
loaded_index[i] = primary_key.data_types[i]->createColumn();
|
||||
loaded_index[i]->reserve(index_granularity.getMarksCount());
|
||||
}
|
||||
|
||||
@ -456,7 +457,7 @@ void IMergeTreeDataPart::loadIndex()
|
||||
|
||||
for (size_t i = 0; i < index_granularity.getMarksCount(); ++i) //-V756
|
||||
for (size_t j = 0; j < key_size; ++j)
|
||||
storage.primary_key_data_types[j]->deserializeBinary(*loaded_index[j], *index_file);
|
||||
primary_key.data_types[j]->deserializeBinary(*loaded_index[j], *index_file);
|
||||
|
||||
for (size_t i = 0; i < key_size; ++i)
|
||||
{
|
||||
@ -844,7 +845,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
|
||||
|
||||
if (!checksums.empty())
|
||||
{
|
||||
if (!storage.primary_key_columns.empty() && !checksums.files.count("primary.idx"))
|
||||
if (!storage.getPrimaryKey().expression_column_names.empty() && !checksums.files.count("primary.idx"))
|
||||
throw Exception("No checksum for primary.idx", ErrorCodes::NO_FILE_IN_DATA_PART);
|
||||
|
||||
if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
|
||||
@ -878,7 +879,7 @@ void IMergeTreeDataPart::checkConsistencyBase() const
|
||||
};
|
||||
|
||||
/// Check that the primary key index is not empty.
|
||||
if (!storage.primary_key_columns.empty())
|
||||
if (!storage.getPrimaryKey().expression_column_names.empty())
|
||||
check_file_not_empty(volume->getDisk(), path + "primary.idx");
|
||||
|
||||
if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
|
||||
|
@ -154,7 +154,8 @@ MergeTreeData::MergeTreeData(
|
||||
{
|
||||
StorageMetadataKeyField sampling_key = StorageMetadataKeyField::getKeyFromAST(metadata.sample_by_ast, getColumns(), global_context);
|
||||
|
||||
if (!primary_key_sample.has(sampling_key.expression_column_names[0])
|
||||
const auto & primary_key = getPrimaryKey();
|
||||
if (!primary_key.sample_block.has(sampling_key.expression_column_names[0])
|
||||
&& !attach && !settings->compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility.
|
||||
throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
@ -252,11 +253,11 @@ StorageInMemoryMetadata MergeTreeData::getInMemoryMetadata() const
|
||||
if (hasPartitionKey())
|
||||
metadata.partition_by_ast = getPartitionKeyAST()->clone();
|
||||
|
||||
if (order_by_ast)
|
||||
metadata.order_by_ast = order_by_ast->clone();
|
||||
if (hasSortingKey())
|
||||
metadata.order_by_ast = getSortingKeyAST()->clone();
|
||||
|
||||
if (primary_key_ast)
|
||||
metadata.primary_key_ast = primary_key_ast->clone();
|
||||
if (hasPrimaryKey())
|
||||
metadata.primary_key_ast = getPrimaryKeyAST()->clone();
|
||||
|
||||
if (ttl_table_ast)
|
||||
metadata.ttl_for_table_ast = ttl_table_ast->clone();
|
||||
@ -349,17 +350,18 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
|
||||
auto all_columns = metadata.columns.getAllPhysical();
|
||||
|
||||
/// Order by check AST
|
||||
if (order_by_ast && only_check)
|
||||
if (hasSortingKey() && only_check)
|
||||
{
|
||||
/// This is ALTER, not CREATE/ATTACH TABLE. Let us check that all new columns used in the sorting key
|
||||
/// expression have just been added (so that the sorting order is guaranteed to be valid with the new key).
|
||||
|
||||
ASTPtr added_key_column_expr_list = std::make_shared<ASTExpressionList>();
|
||||
const auto & old_sorting_key_columns = getSortingKeyColumns();
|
||||
for (size_t new_i = 0, old_i = 0; new_i < sorting_key_size; ++new_i)
|
||||
{
|
||||
if (old_i < sorting_key_columns.size())
|
||||
if (old_i < old_sorting_key_columns.size())
|
||||
{
|
||||
if (new_sorting_key_columns[new_i] != sorting_key_columns[old_i])
|
||||
if (new_sorting_key_columns[new_i] != old_sorting_key_columns[old_i])
|
||||
added_key_column_expr_list->children.push_back(new_sorting_key_expr_list->children[new_i]);
|
||||
else
|
||||
++old_i;
|
||||
@ -414,6 +416,12 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
|
||||
new_primary_key_data_types.push_back(elem.type);
|
||||
}
|
||||
|
||||
DataTypes new_sorting_key_data_types;
|
||||
for (size_t i = 0; i < sorting_key_size; ++i)
|
||||
{
|
||||
new_sorting_key_data_types.push_back(new_sorting_key_sample.getByPosition(i).type);
|
||||
}
|
||||
|
||||
ASTPtr skip_indices_with_primary_key_expr_list = new_primary_key_expr_list->clone();
|
||||
ASTPtr skip_indices_with_sorting_key_expr_list = new_sorting_key_expr_list->clone();
|
||||
|
||||
@ -463,17 +471,23 @@ void MergeTreeData::setProperties(const StorageInMemoryMetadata & metadata, bool
|
||||
{
|
||||
setColumns(std::move(metadata.columns));
|
||||
|
||||
order_by_ast = metadata.order_by_ast;
|
||||
sorting_key_columns = std::move(new_sorting_key_columns);
|
||||
sorting_key_expr_ast = std::move(new_sorting_key_expr_list);
|
||||
sorting_key_expr = std::move(new_sorting_key_expr);
|
||||
StorageMetadataKeyField new_sorting_key;
|
||||
new_sorting_key.definition_ast = metadata.order_by_ast;
|
||||
new_sorting_key.expression_column_names = std::move(new_sorting_key_columns);
|
||||
new_sorting_key.expression_ast = std::move(new_sorting_key_expr_list);
|
||||
new_sorting_key.expressions = std::move(new_sorting_key_expr);
|
||||
new_sorting_key.sample_block = std::move(new_sorting_key_sample);
|
||||
new_sorting_key.data_types = std::move(new_sorting_key_data_types);
|
||||
setSortingKey(new_sorting_key);
|
||||
|
||||
primary_key_ast = metadata.primary_key_ast;
|
||||
primary_key_columns = std::move(new_primary_key_columns);
|
||||
primary_key_expr_ast = std::move(new_primary_key_expr_list);
|
||||
primary_key_expr = std::move(new_primary_key_expr);
|
||||
primary_key_sample = std::move(new_primary_key_sample);
|
||||
primary_key_data_types = std::move(new_primary_key_data_types);
|
||||
StorageMetadataKeyField new_primary_key;
|
||||
new_primary_key.definition_ast = metadata.primary_key_ast;
|
||||
new_primary_key.expression_column_names = std::move(new_primary_key_columns);
|
||||
new_primary_key.expression_ast = std::move(new_primary_key_expr_list);
|
||||
new_primary_key.expressions = std::move(new_primary_key_expr);
|
||||
new_primary_key.sample_block = std::move(new_primary_key_sample);
|
||||
new_primary_key.data_types = std::move(new_primary_key_data_types);
|
||||
setPrimaryKey(new_primary_key);
|
||||
|
||||
setIndices(metadata.indices);
|
||||
skip_indices = std::move(new_indices);
|
||||
@ -622,8 +636,8 @@ void MergeTreeData::setTTLExpressions(const ColumnsDescription & new_columns,
|
||||
for (const auto & col : getPartitionKey().expressions->getRequiredColumns())
|
||||
columns_ttl_forbidden.insert(col);
|
||||
|
||||
if (sorting_key_expr)
|
||||
for (const auto & col : sorting_key_expr->getRequiredColumns())
|
||||
if (hasSortingKey())
|
||||
for (const auto & col : getColumnsRequiredForSortingKey())
|
||||
columns_ttl_forbidden.insert(col);
|
||||
|
||||
for (const auto & [name, ast] : new_column_ttls)
|
||||
@ -1421,8 +1435,9 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, const S
|
||||
columns_alter_type_forbidden.insert(col);
|
||||
}
|
||||
|
||||
if (sorting_key_expr)
|
||||
if (hasSortingKey())
|
||||
{
|
||||
auto sorting_key_expr = getSortingKey().expressions;
|
||||
for (const ExpressionAction & action : sorting_key_expr->getActions())
|
||||
{
|
||||
auto action_columns = action.getNeededColumns();
|
||||
@ -3085,7 +3100,7 @@ bool MergeTreeData::isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(const A
|
||||
{
|
||||
const String column_name = node->getColumnName();
|
||||
|
||||
for (const auto & name : primary_key_columns)
|
||||
for (const auto & name : getPrimaryKey().expression_column_names)
|
||||
if (column_name == name)
|
||||
return true;
|
||||
|
||||
@ -3145,10 +3160,10 @@ MergeTreeData & MergeTreeData::checkStructureAndGetMergeTreeData(IStorage & sour
|
||||
return ast ? queryToString(ast) : "";
|
||||
};
|
||||
|
||||
if (query_to_string(order_by_ast) != query_to_string(src_data->order_by_ast))
|
||||
if (query_to_string(getSortingKeyAST()) != query_to_string(src_data->getSortingKeyAST()))
|
||||
throw Exception("Tables have different ordering", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (query_to_string(getPartitionKey().definition_ast) != query_to_string(src_data->getPartitionKey().definition_ast))
|
||||
if (query_to_string(getPartitionKeyAST()) != query_to_string(src_data->getPartitionKeyAST()))
|
||||
throw Exception("Tables have different partition key", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
||||
if (format_version != src_data->format_version)
|
||||
|
@ -334,14 +334,14 @@ public:
|
||||
/// See comments about methods below in IStorage interface
|
||||
StorageInMemoryMetadata getInMemoryMetadata() const override;
|
||||
|
||||
ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; }
|
||||
ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; }
|
||||
// ASTPtr getSortingKeyAST() const override { return sorting_key_expr_ast; }
|
||||
//ASTPtr getPrimaryKeyAST() const override { return primary_key_expr_ast; }
|
||||
|
||||
//Names getColumnsRequiredForPartitionKey() const override { return (partition_key_expr ? partition_key_expr->getRequiredColumns() : Names{}); }
|
||||
Names getColumnsRequiredForSortingKey() const override { return sorting_key_expr->getRequiredColumns(); }
|
||||
Names getColumnsRequiredForPrimaryKey() const override { return primary_key_expr->getRequiredColumns(); }
|
||||
Names getColumnsRequiredForFinal() const override { return sorting_key_expr->getRequiredColumns(); }
|
||||
Names getSortingKeyColumns() const override { return sorting_key_columns; }
|
||||
//Names getColumnsRequiredForSortingKey() const override { return sorting_key_expr->getRequiredColumns(); }
|
||||
//Names getColumnsRequiredForPrimaryKey() const override { return primary_key_expr->getRequiredColumns(); }
|
||||
//Names getColumnsRequiredForFinal() const override { return sorting_key_expr->getRequiredColumns(); }
|
||||
//Names getSortingKeyColumns() const override { return sorting_key_columns; }
|
||||
|
||||
ColumnDependencies getColumnDependencies(const NameSet & updated_columns) const override;
|
||||
|
||||
@ -525,8 +525,7 @@ public:
|
||||
*/
|
||||
static ASTPtr extractKeyExpressionList(const ASTPtr & node);
|
||||
|
||||
bool hasSortingKey() const { return !sorting_key_columns.empty(); }
|
||||
bool hasPrimaryKey() const { return !primary_key_columns.empty(); }
|
||||
//bool hasPrimaryKey() const { return !primary_key_columns.empty(); }
|
||||
bool hasSkipIndices() const { return !skip_indices.empty(); }
|
||||
|
||||
bool hasAnyColumnTTL() const { return !column_ttl_entries_by_name.empty(); }
|
||||
@ -659,16 +658,16 @@ public:
|
||||
|
||||
/// Names of sorting key columns in ORDER BY expression. For example: 'a',
|
||||
/// 'x * y', 'toStartOfMonth(date)', etc.
|
||||
Names sorting_key_columns;
|
||||
ASTPtr sorting_key_expr_ast;
|
||||
ExpressionActionsPtr sorting_key_expr;
|
||||
//Names sorting_key_columns;
|
||||
//ASTPtr sorting_key_expr_ast;
|
||||
//ExpressionActionsPtr sorting_key_expr;
|
||||
|
||||
/// Names of columns for primary key.
|
||||
Names primary_key_columns;
|
||||
ASTPtr primary_key_expr_ast;
|
||||
ExpressionActionsPtr primary_key_expr;
|
||||
Block primary_key_sample;
|
||||
DataTypes primary_key_data_types;
|
||||
//Names primary_key_columns;
|
||||
//ASTPtr primary_key_expr_ast;
|
||||
//ExpressionActionsPtr primary_key_expr;
|
||||
//Block primary_key_sample;
|
||||
//DataTypes primary_key_data_types;
|
||||
|
||||
struct TTLEntry
|
||||
{
|
||||
@ -729,8 +728,8 @@ protected:
|
||||
friend struct ReplicatedMergeTreeTableMetadata;
|
||||
friend class StorageReplicatedMergeTree;
|
||||
|
||||
ASTPtr order_by_ast;
|
||||
ASTPtr primary_key_ast;
|
||||
///ASTPtr order_by_ast;
|
||||
///ASTPtr primary_key_ast;
|
||||
ASTPtr ttl_table_ast;
|
||||
ASTPtr settings_ast;
|
||||
|
||||
|
@ -607,7 +607,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
|
||||
NamesAndTypesList merging_columns;
|
||||
Names gathering_column_names, merging_column_names;
|
||||
extractMergingAndGatheringColumns(
|
||||
storage_columns, data.sorting_key_expr, data.skip_indices,
|
||||
storage_columns, data.getSortingKey().expressions, data.skip_indices,
|
||||
data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names);
|
||||
|
||||
auto single_disk_volume = std::make_shared<SingleDiskVolume>("volume_" + future_part.name, disk);
|
||||
@ -726,7 +726,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
|
||||
pipes.emplace_back(std::move(pipe));
|
||||
}
|
||||
|
||||
Names sort_columns = data.sorting_key_columns;
|
||||
Names sort_columns = data.getSortingKey().expression_column_names;
|
||||
SortDescription sort_description;
|
||||
size_t sort_columns_size = sort_columns.size();
|
||||
sort_description.reserve(sort_columns_size);
|
||||
|
@ -222,9 +222,10 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
|
||||
data.check(real_column_names);
|
||||
|
||||
const Settings & settings = context.getSettingsRef();
|
||||
Names primary_key_columns = data.primary_key_columns;
|
||||
const auto & primary_key = data.getPrimaryKey();
|
||||
Names primary_key_columns = primary_key.expression_column_names;
|
||||
|
||||
KeyCondition key_condition(query_info, context, primary_key_columns, data.primary_key_expr);
|
||||
KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expressions);
|
||||
|
||||
if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue())
|
||||
{
|
||||
@ -613,7 +614,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
|
||||
if (select.final())
|
||||
{
|
||||
/// Add columns needed to calculate the sorting expression and the sign.
|
||||
std::vector<String> add_columns = data.sorting_key_expr->getRequiredColumns();
|
||||
std::vector<String> add_columns = data.getColumnsRequiredForSortingKey();
|
||||
column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end());
|
||||
|
||||
if (!data.merging_params.sign_column.empty())
|
||||
@ -639,7 +640,7 @@ Pipes MergeTreeDataSelectExecutor::readFromParts(
|
||||
else if (settings.optimize_read_in_order && query_info.input_sorting_info)
|
||||
{
|
||||
size_t prefix_size = query_info.input_sorting_info->order_key_prefix_descr.size();
|
||||
auto order_key_prefix_ast = data.sorting_key_expr_ast->clone();
|
||||
auto order_key_prefix_ast = data.getSortingKey().expression_ast->clone();
|
||||
order_key_prefix_ast->children.resize(prefix_size);
|
||||
|
||||
auto syntax_result = SyntaxAnalyzer(context).analyze(order_key_prefix_ast, data.getColumns().getAllPhysical());
|
||||
@ -1024,7 +1025,7 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsWithOrder(
|
||||
{
|
||||
SortDescription sort_description;
|
||||
for (size_t j = 0; j < input_sorting_info->order_key_prefix_descr.size(); ++j)
|
||||
sort_description.emplace_back(data.sorting_key_columns[j],
|
||||
sort_description.emplace_back(data.getSortingKey().expression_column_names[j],
|
||||
input_sorting_info->direction, 1);
|
||||
|
||||
/// Drop temporary columns, added by 'sorting_key_prefix_expr'
|
||||
@ -1097,11 +1098,11 @@ Pipes MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal(
|
||||
if (!out_projection)
|
||||
out_projection = createProjection(pipe, data);
|
||||
|
||||
pipe.addSimpleTransform(std::make_shared<ExpressionTransform>(pipe.getHeader(), data.sorting_key_expr));
|
||||
pipe.addSimpleTransform(std::make_shared<ExpressionTransform>(pipe.getHeader(), data.getSortingKey().expressions));
|
||||
pipes.emplace_back(std::move(pipe));
|
||||
}
|
||||
|
||||
Names sort_columns = data.sorting_key_columns;
|
||||
Names sort_columns = data.getSortingKey().expression_column_names;
|
||||
SortDescription sort_description;
|
||||
size_t sort_columns_size = sort_columns.size();
|
||||
sort_description.reserve(sort_columns_size);
|
||||
@ -1294,11 +1295,12 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
|
||||
std::function<void(size_t, size_t, FieldRef &)> create_field_ref;
|
||||
/// If there are no monotonic functions, there is no need to save block reference.
|
||||
/// Passing explicit field to FieldRef allows to optimize ranges and shows better performance.
|
||||
const auto & primary_key = data.getPrimaryKey();
|
||||
if (key_condition.hasMonotonicFunctionsChain())
|
||||
{
|
||||
auto index_block = std::make_shared<Block>();
|
||||
for (size_t i = 0; i < used_key_size; ++i)
|
||||
index_block->insert({index[i], data.primary_key_data_types[i], data.primary_key_columns[i]});
|
||||
index_block->insert({index[i], primary_key.data_types[i], primary_key.expression_column_names[i]});
|
||||
|
||||
create_field_ref = [index_block](size_t row, size_t column, FieldRef & field)
|
||||
{
|
||||
@ -1329,7 +1331,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
|
||||
create_field_ref(range.begin, i, index_left[i]);
|
||||
|
||||
may_be_true = key_condition.mayBeTrueAfter(
|
||||
used_key_size, index_left.data(), data.primary_key_data_types);
|
||||
used_key_size, index_left.data(), primary_key.data_types);
|
||||
}
|
||||
else
|
||||
{
|
||||
@ -1343,7 +1345,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
|
||||
}
|
||||
|
||||
may_be_true = key_condition.mayBeTrueInRange(
|
||||
used_key_size, index_left.data(), index_right.data(), data.primary_key_data_types);
|
||||
used_key_size, index_left.data(), index_right.data(), primary_key.data_types);
|
||||
}
|
||||
|
||||
if (!may_be_true)
|
||||
|
@ -262,7 +262,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
|
||||
if (data.hasSortingKey() || data.hasSkipIndices())
|
||||
data.sorting_key_and_skip_indices_expr->execute(block);
|
||||
|
||||
Names sort_columns = data.sorting_key_columns;
|
||||
Names sort_columns = data.getSortingKey().expression_column_names;
|
||||
SortDescription sort_description;
|
||||
size_t sort_columns_size = sort_columns.size();
|
||||
sort_description.reserve(sort_columns_size);
|
||||
|
@ -39,8 +39,9 @@ MergeTreeWhereOptimizer::MergeTreeWhereOptimizer(
|
||||
block_with_constants{KeyCondition::getBlockWithConstants(query_info.query, query_info.syntax_analyzer_result, context)},
|
||||
log{log_}
|
||||
{
|
||||
if (!data.primary_key_columns.empty())
|
||||
first_primary_key_column = data.primary_key_columns[0];
|
||||
const auto & primary_key = data.getPrimaryKey();
|
||||
if (!primary_key.expression_column_names.empty())
|
||||
first_primary_key_column = primary_key.expression_column_names[0];
|
||||
|
||||
calculateColumnSizes(data, queried_columns);
|
||||
determineArrayJoinedNames(query_info.query->as<ASTSelectQuery &>());
|
||||
|
@ -161,7 +161,7 @@ void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Perm
|
||||
std::inserter(skip_indexes_column_names_set, skip_indexes_column_names_set.end()));
|
||||
Names skip_indexes_column_names(skip_indexes_column_names_set.begin(), skip_indexes_column_names_set.end());
|
||||
|
||||
Block primary_key_block = getBlockAndPermute(block, storage.primary_key_columns, permutation);
|
||||
Block primary_key_block = getBlockAndPermute(block, storage.getPrimaryKey().expression_column_names, permutation);
|
||||
Block skip_indexes_block = getBlockAndPermute(block, skip_indexes_column_names, permutation);
|
||||
|
||||
writer->write(block, permutation, primary_key_block, skip_indexes_block);
|
||||
|
@ -40,12 +40,12 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
|
||||
/// So rules in zookeeper metadata is following:
|
||||
/// - When we have only ORDER BY, than store it in "primary key:" row of /metadata
|
||||
/// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata
|
||||
if (!data.primary_key_ast)
|
||||
primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast));
|
||||
if (!data.hasPrimaryKey())
|
||||
primary_key = formattedAST(data.getSortingKey().expression_ast);
|
||||
else
|
||||
{
|
||||
primary_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.primary_key_ast));
|
||||
sorting_key = formattedAST(MergeTreeData::extractKeyExpressionList(data.order_by_ast));
|
||||
primary_key = formattedAST(data.getPrimaryKey().expression_ast);
|
||||
sorting_key = formattedAST(data.getSortingKey().expression_ast);
|
||||
}
|
||||
|
||||
data_format_version = data.format_version;
|
||||
|
@ -493,11 +493,11 @@ void StorageReplicatedMergeTree::setTableStructure(ColumnsDescription new_column
|
||||
metadata.order_by_ast = tuple;
|
||||
}
|
||||
|
||||
if (!primary_key_ast)
|
||||
if (!hasPrimaryKey())
|
||||
{
|
||||
/// Primary and sorting key become independent after this ALTER so we have to
|
||||
/// save the old ORDER BY expression as the new primary key.
|
||||
metadata.primary_key_ast = order_by_ast->clone();
|
||||
metadata.primary_key_ast = getSortingKeyAST()->clone();
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user