renames [#CLICKHOUSE-3859]

This commit is contained in:
Alexey Zatelepin 2018-10-12 22:00:43 +03:00
parent 0fa86162d9
commit f4c26e78d2
11 changed files with 88 additions and 85 deletions

View File

@ -102,9 +102,9 @@ static NameSet getKeyColumns(const StoragePtr & storage)
for (const String & col : merge_tree_data->partition_expr->getRequiredColumns())
key_columns.insert(col);
auto primary_expr = merge_tree_data->getSortExpression();
if (primary_expr)
for (const String & col : primary_expr->getRequiredColumns())
auto sorting_key_expr = merge_tree_data->getSortingKeyExpression();
if (sorting_key_expr)
for (const String & col : sorting_key_expr->getRequiredColumns())
key_columns.insert(col);
/// We don't process sampling_expression separately because it must be among the primary key columns.

View File

@ -86,8 +86,8 @@ MergeTreeData::MergeTreeData(
const String & database_, const String & table_,
const String & full_path_, const ColumnsDescription & columns_,
Context & context_,
const ASTPtr & primary_key_expr_ast_,
const ASTPtr & sort_expr_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sorting_key_ast_,
const String & date_column_name,
const ASTPtr & partition_expr_ast_,
const ASTPtr & sampling_expression_,
@ -114,7 +114,7 @@ MergeTreeData::MergeTreeData(
/// NOTE: using the same columns list as is read when performing actual merges.
merging_params.check(getColumns().getAllPhysical());
setPrimaryKey(primary_key_expr_ast_, sort_expr_ast_);
setPrimaryKey(primary_key_ast_, sorting_key_ast_);
if (sampling_expression && (!primary_key_sample.has(sampling_expression->getColumnName()))
&& !attach && !settings.compatibility_allow_sampling_expression_not_in_primary_key) /// This is for backward compatibility.
@ -209,41 +209,41 @@ static void checkKeyExpression(const ExpressionActions & expr, const Block & sam
}
void MergeTreeData::setPrimaryKey(ASTPtr new_primary_key_expr_ast, const ASTPtr & new_sort_expr_ast)
void MergeTreeData::setPrimaryKey(ASTPtr new_primary_key_ast, const ASTPtr & new_sorting_key_ast)
{
if (!new_sort_expr_ast)
if (!new_sorting_key_ast)
throw Exception("Sorting key cannot be empty", ErrorCodes::BAD_ARGUMENTS);
if (!new_primary_key_expr_ast)
new_primary_key_expr_ast = new_sort_expr_ast->clone();
if (!new_primary_key_ast)
new_primary_key_ast = new_sorting_key_ast->clone();
if (new_sort_expr_ast.get() != sort_expr_ast.get()
if (new_sorting_key_ast.get() != sorting_key_ast.get()
&& merging_params.mode == MergeTreeData::MergingParams::VersionedCollapsing)
{
new_sort_expr_ast->children.push_back(std::make_shared<ASTIdentifier>(merging_params.version_column));
new_sorting_key_ast->children.push_back(std::make_shared<ASTIdentifier>(merging_params.version_column));
}
size_t primary_key_size = new_primary_key_expr_ast->children.size();
size_t sort_key_size = new_sort_expr_ast->children.size();
if (primary_key_size > sort_key_size)
size_t primary_key_size = new_primary_key_ast->children.size();
size_t sorting_key_size = new_sorting_key_ast->children.size();
if (primary_key_size > sorting_key_size)
throw Exception("Primary key must be a prefix of the sorting key, but its length: "
+ toString(primary_key_size) + " is greater than the sorting key length: " + toString(sort_key_size),
+ toString(primary_key_size) + " is greater than the sorting key length: " + toString(sorting_key_size),
ErrorCodes::BAD_ARGUMENTS);
Names new_primary_key_columns;
Names new_sort_key_columns;
Names new_sorting_key_columns;
for (size_t i = 0; i < sort_key_size; ++i)
for (size_t i = 0; i < sorting_key_size; ++i)
{
String sort_key_column = new_sort_expr_ast->children[i]->getColumnName();
new_sort_key_columns.push_back(sort_key_column);
String sorting_key_column = new_sorting_key_ast->children[i]->getColumnName();
new_sorting_key_columns.push_back(sorting_key_column);
if (i < primary_key_size)
{
String pk_column = new_primary_key_expr_ast->children[i]->getColumnName();
if (pk_column != sort_key_column)
String pk_column = new_primary_key_ast->children[i]->getColumnName();
if (pk_column != sorting_key_column)
throw Exception("Primary key must be a prefix of the sorting key, but in position "
+ toString(i) + " its column is " + pk_column + ", not " + sort_key_column,
+ toString(i) + " its column is " + pk_column + ", not " + sorting_key_column,
ErrorCodes::BAD_ARGUMENTS);
new_primary_key_columns.push_back(pk_column);
@ -252,31 +252,31 @@ void MergeTreeData::setPrimaryKey(ASTPtr new_primary_key_expr_ast, const ASTPtr
auto all_columns = getColumns().getAllPhysical();
auto new_sort_expr = ExpressionAnalyzer(new_sort_expr_ast, context, nullptr, all_columns)
auto new_sorting_key_expr = ExpressionAnalyzer(new_sorting_key_ast, context, nullptr, all_columns)
.getActions(false);
auto new_sort_expr_sample =
ExpressionAnalyzer(new_sort_expr_ast, context, nullptr, all_columns)
auto new_sorting_key_sample =
ExpressionAnalyzer(new_sorting_key_ast, context, nullptr, all_columns)
.getActions(true)->getSampleBlock();
checkKeyExpression(*new_sort_expr, new_sort_expr_sample, "Sorting");
checkKeyExpression(*new_sorting_key_expr, new_sorting_key_sample, "Sorting");
auto new_primary_key_expr = ExpressionAnalyzer(new_primary_key_expr_ast, context, nullptr, all_columns)
auto new_primary_key_expr = ExpressionAnalyzer(new_primary_key_ast, context, nullptr, all_columns)
.getActions(false);
Block new_primary_key_sample;
DataTypes new_primary_key_data_types;
for (size_t i = 0; i < primary_key_size; ++i)
{
const auto & elem = new_sort_expr_sample.getByPosition(i);
const auto & elem = new_sorting_key_sample.getByPosition(i);
new_primary_key_sample.insert(elem);
new_primary_key_data_types.push_back(elem.type);
}
sort_expr_ast = new_sort_expr_ast;
sort_columns = std::move(new_sort_key_columns);
sort_expr = std::move(new_sort_expr);
sorting_key_ast = new_sorting_key_ast;
sorting_key_columns = std::move(new_sorting_key_columns);
sorting_key_expr = std::move(new_sorting_key_expr);
primary_key_expr_ast = new_primary_key_expr_ast;
primary_key_ast = new_primary_key_ast;
primary_key_columns = std::move(new_primary_key_columns);
primary_key_expr = std::move(new_primary_key_expr);
primary_key_sample = std::move(new_primary_key_sample);
@ -908,18 +908,18 @@ void MergeTreeData::checkAlter(const AlterCommands & commands)
columns_alter_forbidden.insert(col);
}
if (sort_expr)
if (sorting_key_expr)
{
for (const ExpressionAction & action : sort_expr->getActions())
for (const ExpressionAction & action : sorting_key_expr->getActions())
{
auto action_columns = action.getNeededColumns();
columns_alter_forbidden.insert(action_columns.begin(), action_columns.end());
}
for (const String & col : sort_expr->getRequiredColumns())
for (const String & col : sorting_key_expr->getRequiredColumns())
columns_alter_metadata_only.insert(col);
/// We don't process sampling_expression separately because it must be among the primary key columns
/// and we don't process primary_key_expr separately because it is a prefix of sort_expr.
/// and we don't process primary_key_expr separately because it is a prefix of sorting_key_expr.
}
if (!merging_params.sign_column.empty())
@ -1168,7 +1168,7 @@ MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart(
MergeTreeDataPartChecksum::uint128 new_primary_key_hash{};
/// TODO: Check the order of secondary sorting key columns.
if (new_primary_key.get() != primary_key_expr_ast.get())
if (new_primary_key.get() != primary_key_ast.get())
{
ExpressionActionsPtr new_primary_expr = ExpressionAnalyzer(new_primary_key, context, nullptr, new_columns).getActions(true);
Block new_primary_key_sample = new_primary_expr->getSampleBlock();
@ -2421,7 +2421,7 @@ MergeTreeData * MergeTreeData::checkStructureAndGetMergeTreeData(const StoragePt
return ast ? queryToString(ast) : "";
};
if (query_to_string(sort_expr_ast) != query_to_string(src_data->sort_expr_ast))
if (query_to_string(sorting_key_ast) != query_to_string(src_data->sorting_key_ast))
throw Exception("Tables have different ordering", ErrorCodes::BAD_ARGUMENTS);
if (query_to_string(partition_expr_ast) != query_to_string(src_data->partition_expr_ast))

View File

@ -284,7 +284,7 @@ public:
/// Attach the table corresponding to the directory in full_path (must end with /), with the given columns.
/// Correctness of names and paths is not checked.
///
/// primary_expr_ast - expression used for sorting;
/// primary_key_ast - expression used for sorting;
/// date_column_name - if not empty, the name of the Date column used for partitioning by month.
/// Otherwise, partition_expr_ast is used for partitioning.
/// require_part_metadata - should checksums.txt and columns.txt exist in the part directory.
@ -293,8 +293,8 @@ public:
const String & full_path_,
const ColumnsDescription & columns_,
Context & context_,
const ASTPtr & primary_expr_ast_,
const ASTPtr & sort_expr_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sorting_key_ast_,
const String & date_column_name,
const ASTPtr & partition_expr_ast_,
const ASTPtr & sampling_expression_, /// nullptr, if sampling is not supported.
@ -488,10 +488,11 @@ public:
bool hasPrimaryKey() const { return !primary_key_columns.empty(); }
ExpressionActionsPtr getPrimaryKeyExpression() const { return primary_key_expr; }
bool hasSortExpression() const { return !sort_columns.empty(); }
ExpressionActionsPtr getSortExpression() const { return sort_expr; } /// may return nullptr
Names getPrimaryKeyColumns() const { return primary_key_columns; }
Names getSortColumns() const { return sort_columns; }
bool hasSortingKey() const { return !sorting_key_columns.empty(); }
ExpressionActionsPtr getSortingKeyExpression() const { return sorting_key_expr; }
Names getSortingKeyColumns() const { return sorting_key_columns; }
/// Check that the part is not broken and calculate the checksums for it if they are not present.
MutableDataPartPtr loadPartAndFixMetadata(const String & relative_path);
@ -549,11 +550,12 @@ public:
const MergeTreeSettings settings;
ASTPtr primary_key_expr_ast;
ASTPtr sort_expr_ast;
ASTPtr primary_key_ast;
Block primary_key_sample;
DataTypes primary_key_data_types;
ASTPtr sorting_key_ast;
ASTPtr partition_expr_ast;
ExpressionActionsPtr partition_expr;
Block partition_key_sample;
@ -580,11 +582,12 @@ private:
bool require_part_metadata;
ExpressionActionsPtr primary_key_expr;
ExpressionActionsPtr sort_expr;
/// Names of columns for primary key. Is the prefix of sort_columns.
/// Names of columns for primary key.
Names primary_key_columns;
ExpressionActionsPtr sorting_key_expr;
/// Names of columns for primary key + secondary sorting columns.
Names sort_columns;
Names sorting_key_columns;
String database_name;
String table_name;
@ -687,7 +690,7 @@ private:
/// The same for clearOldTemporaryDirectories.
std::mutex clear_old_temporary_directories_mutex;
void setPrimaryKey(ASTPtr new_primary_key_expr_ast, const ASTPtr & new_sort_expr_ast);
void setPrimaryKey(ASTPtr new_primary_key_ast, const ASTPtr & new_sorting_key_ast);
void initPartitionKey();

View File

@ -335,12 +335,12 @@ MergeTreeData::DataPartsVector MergeTreeDataMergerMutator::selectAllPartsFromPar
/// PK columns are sorted and merged, ordinary columns are gathered using info from merge step
static void extractMergingAndGatheringColumns(
const NamesAndTypesList & all_columns,
const ExpressionActionsPtr & sort_key_expressions,
const ExpressionActionsPtr & sorting_key_expr,
const MergeTreeData::MergingParams & merging_params,
NamesAndTypesList & gathering_columns, Names & gathering_column_names,
NamesAndTypesList & merging_columns, Names & merging_column_names)
{
Names sort_key_columns_vec = sort_key_expressions->getRequiredColumns();
Names sort_key_columns_vec = sorting_key_expr->getRequiredColumns();
std::set<String> key_columns(sort_key_columns_vec.cbegin(), sort_key_columns_vec.cend());
/// Force sign column for Collapsing mode
@ -549,7 +549,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
NamesAndTypesList gathering_columns, merging_columns;
Names gathering_column_names, merging_column_names;
extractMergingAndGatheringColumns(
all_columns, data.getSortExpression(),
all_columns, data.getSortingKeyExpression(),
data.merging_params, gathering_columns, gathering_column_names, merging_columns, merging_column_names);
MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared<MergeTreeData::DataPart>(
@ -632,12 +632,12 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor
if (data.hasPrimaryKey())
src_streams.emplace_back(std::make_shared<MaterializingBlockInputStream>(
std::make_shared<ExpressionBlockInputStream>(BlockInputStreamPtr(std::move(input)), data.getSortExpression())));
std::make_shared<ExpressionBlockInputStream>(BlockInputStreamPtr(std::move(input)), data.getSortingKeyExpression())));
else
src_streams.emplace_back(std::move(input));
}
Names sort_columns = data.getSortColumns();
Names sort_columns = data.getSortingKeyColumns();
SortDescription sort_description;
size_t sort_columns_size = sort_columns.size();
sort_description.reserve(sort_columns_size);

View File

@ -554,7 +554,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::readFromParts(
if (select.final())
{
/// Add columns needed to calculate the sorting expression and the sign.
std::vector<String> add_columns = data.getSortExpression()->getRequiredColumns();
std::vector<String> add_columns = data.getSortingKeyExpression()->getRequiredColumns();
column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end());
if (!data.merging_params.sign_column.empty())
@ -782,10 +782,10 @@ BlockInputStreams MergeTreeDataSelectExecutor::spreadMarkRangesAmongStreamsFinal
prewhere_info, true, settings.min_bytes_to_use_direct_io, settings.max_read_buffer_size, true,
virt_columns, part.part_index_in_query);
to_merge.emplace_back(std::make_shared<ExpressionBlockInputStream>(source_stream, data.getSortExpression()));
to_merge.emplace_back(std::make_shared<ExpressionBlockInputStream>(source_stream, data.getSortingKeyExpression()));
}
Names sort_columns = data.getSortColumns();
Names sort_columns = data.getSortingKeyColumns();
SortDescription sort_description;
size_t sort_columns_size = sort_columns.size();
sort_description.reserve(sort_columns_size);

View File

@ -175,10 +175,10 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart(BlockWithPa
dir.createDirectories();
/// If we need to calculate some columns to sort.
if (data.hasSortExpression())
data.getSortExpression()->execute(block);
if (data.hasSortingKey())
data.getSortingKeyExpression()->execute(block);
Names sort_columns = data.getSortColumns();
Names sort_columns = data.getSortingKeyColumns();
SortDescription sort_description;
size_t sort_columns_size = sort_columns.size();
sort_description.reserve(sort_columns_size);

View File

@ -95,7 +95,7 @@ void ReplicatedMergeTreeAlterThread::run()
storage.setColumns(std::move(columns_in_zk));
/// Reinitialize primary key because primary key column types might have changed.
storage.data.setPrimaryKey(storage.data.primary_key_expr_ast, storage.data.sort_expr_ast);
storage.data.setPrimaryKey(storage.data.primary_key_ast, storage.data.sorting_key_ast);
LOG_INFO(log, "Applied changes to table.");
}
@ -131,7 +131,7 @@ void ReplicatedMergeTreeAlterThread::run()
/// TODO: You can skip checking for too large changes if ZooKeeper has, for example,
/// node /flags/force_alter.
auto transaction = storage.data.alterDataPart(
part, columns_for_parts, storage.data.primary_key_expr_ast, false);
part, columns_for_parts, storage.data.primary_key_ast, false);
if (!transaction)
continue;

View File

@ -46,8 +46,8 @@ StorageMergeTree::StorageMergeTree(
const ColumnsDescription & columns_,
bool attach,
Context & context_,
const ASTPtr & primary_key_expr_ast_,
const ASTPtr & sort_expr_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sorting_key_ast_,
const String & date_column_name,
const ASTPtr & partition_expr_ast_,
const ASTPtr & sampling_expression_, /// nullptr, if sampling is not supported.
@ -58,7 +58,7 @@ StorageMergeTree::StorageMergeTree(
context(context_), background_pool(context_.getBackgroundPool()),
data(database_name, table_name,
full_path, columns_,
context_, primary_key_expr_ast_, sort_expr_ast_, date_column_name, partition_expr_ast_,
context_, primary_key_ast_, sorting_key_ast_, date_column_name, partition_expr_ast_,
sampling_expression_, merging_params_,
settings_, false, attach),
reader(data), writer(data), merger_mutator(data, context.getBackgroundPool()),
@ -202,8 +202,8 @@ void StorageMergeTree::alter(
bool primary_key_is_modified = false;
ASTPtr new_primary_key_ast = data.primary_key_expr_ast;
ASTPtr new_sort_expr_ast = data.sort_expr_ast;
ASTPtr new_primary_key_ast = data.primary_key_ast;
ASTPtr new_sorting_key_ast = data.sorting_key_ast;
for (const AlterCommand & param : params)
{
@ -211,7 +211,7 @@ void StorageMergeTree::alter(
{
primary_key_is_modified = true;
new_primary_key_ast = param.primary_key;
new_sort_expr_ast = param.primary_key->clone();
new_sorting_key_ast = param.primary_key->clone();
}
}
@ -249,7 +249,7 @@ void StorageMergeTree::alter(
setColumns(std::move(new_columns));
/// Reinitialize primary key because primary key column types might have changed.
data.setPrimaryKey(new_primary_key_ast, new_sort_expr_ast);
data.setPrimaryKey(new_primary_key_ast, new_sorting_key_ast);
for (auto & transaction : transactions)
transaction->commit();
@ -713,7 +713,7 @@ void StorageMergeTree::clearColumnInPartition(const ASTPtr & partition, const Fi
if (part->info.partition_id != partition_id)
throw Exception("Unexpected partition ID " + part->info.partition_id + ". This is a bug.", ErrorCodes::LOGICAL_ERROR);
if (auto transaction = data.alterDataPart(part, columns_for_parts, data.primary_key_expr_ast, false))
if (auto transaction = data.alterDataPart(part, columns_for_parts, data.primary_key_ast, false))
transactions.push_back(std::move(transaction));
LOG_DEBUG(log, "Removing column " << get<String>(column_name) << " from part " << part->name);

View File

@ -96,7 +96,7 @@ public:
ASTPtr getSamplingExpression() const override { return data.sampling_expression; }
ASTPtr getPrimaryExpression() const override { return data.primary_key_expr_ast; }
ASTPtr getPrimaryExpression() const override { return data.primary_key_ast; }
private:
String path;
@ -157,7 +157,7 @@ protected:
* (correctness of names and paths are not checked)
* consisting of the specified columns.
*
* primary_expr_ast - expression for sorting;
* primary_key_ast - expression for sorting;
* date_column_name - if not empty, the name of the column with the date used for partitioning by month;
otherwise, partition_expr_ast is used as the partitioning expression;
*/
@ -168,8 +168,8 @@ protected:
const ColumnsDescription & columns_,
bool attach,
Context & context_,
const ASTPtr & primary_key_expr_ast_,
const ASTPtr & sort_expr_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sorting_key_ast_,
const String & date_column_name,
const ASTPtr & partition_expr_ast_,
const ASTPtr & sampling_expression_, /// nullptr, if sampling is not supported.

View File

@ -197,8 +197,8 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
const String & path_, const String & database_name_, const String & name_,
const ColumnsDescription & columns_,
Context & context_,
const ASTPtr & primary_key_expr_ast_,
const ASTPtr & sort_expr_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sorting_key_ast_,
const String & date_column_name,
const ASTPtr & partition_expr_ast_,
const ASTPtr & sampling_expression_,
@ -212,7 +212,7 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree(
replica_name(context.getMacros()->expand(replica_name_, database_name, table_name)),
data(database_name, table_name,
full_path, columns_,
context_, primary_key_expr_ast_, sort_expr_ast_, date_column_name, partition_expr_ast_,
context_, primary_key_ast_, sorting_key_ast_, date_column_name, partition_expr_ast_,
sampling_expression_, merging_params_,
settings_, true, attach,
[this] (const std::string & name) { enqueuePartForCheck(name); }),
@ -353,7 +353,7 @@ namespace
<< "index granularity: " << data.index_granularity << "\n"
<< "mode: " << static_cast<int>(data.merging_params.mode) << "\n"
<< "sign column: " << data.merging_params.sign_column << "\n"
<< "primary key: " << formattedAST(data.primary_key_expr_ast) << "\n";
<< "primary key: " << formattedAST(data.primary_key_ast) << "\n";
if (data.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING)
{
@ -433,7 +433,7 @@ namespace
in >> "\nprimary key: ";
String read_primary_key;
String local_primary_key = formattedAST(data.primary_key_expr_ast);
String local_primary_key = formattedAST(data.primary_key_ast);
in >> read_primary_key;
/// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes
@ -1610,7 +1610,7 @@ void StorageReplicatedMergeTree::executeClearColumnInPartition(const LogEntry &
LOG_DEBUG(log, "Clearing column " << entry.column_name << " in part " << part->name);
auto transaction = data.alterDataPart(part, columns_for_parts, data.primary_key_expr_ast, false);
auto transaction = data.alterDataPart(part, columns_for_parts, data.primary_key_ast, false);
if (!transaction)
continue;

View File

@ -195,7 +195,7 @@ public:
ASTPtr getSamplingExpression() const override { return data.sampling_expression; }
ASTPtr getPrimaryExpression() const override { return data.primary_key_expr_ast; }
ASTPtr getPrimaryExpression() const override { return data.primary_key_ast; }
private:
/// Delete old parts from disk and from ZooKeeper.
@ -522,8 +522,8 @@ protected:
const String & path_, const String & database_name_, const String & name_,
const ColumnsDescription & columns_,
Context & context_,
const ASTPtr & primary_expr_ast_,
const ASTPtr & sort_expr_ast_,
const ASTPtr & primary_key_ast_,
const ASTPtr & sorting_key_ast_,
const String & date_column_name,
const ASTPtr & partition_expr_ast_,
const ASTPtr & sampling_expression_,