Renames for compact parts

This commit is contained in:
alesapin 2020-04-02 21:24:11 +03:00
parent 5b133dd1ce
commit 0606b7e4df
8 changed files with 88 additions and 12 deletions

View File

@ -3608,13 +3608,8 @@ MergeTreeData::AlterConversions MergeTreeData::getAlterConversionsForPart(const
AlterConversions result{}; AlterConversions result{};
for (const auto & command : commands) for (const auto & command : commands)
{
if (command.type == MutationCommand::Type::RENAME_COLUMN) if (command.type == MutationCommand::Type::RENAME_COLUMN)
{
result.rename_map[command.rename_to] = command.column_name; result.rename_map[command.rename_to] = command.column_name;
LOG_DEBUG(log, "Add to rename map:" << command.column_name);
}
}
return result; return result;
} }

View File

@ -1261,6 +1261,20 @@ void MergeTreeDataMergerMutator::splitMutationCommands(
{ {
removed_columns_from_compact_part.emplace(command.column_name); removed_columns_from_compact_part.emplace(command.column_name);
} }
else if (command.type == MutationCommand::Type::RENAME_COLUMN)
{
if (is_compact_part)
{
for_interpreter.push_back(
{
.type = MutationCommand::Type::READ_COLUMN,
.column_name = command.rename_to,
});
already_changed_columns.emplace(command.column_name);
}
else
for_file_renames.push_back(command);
}
else else
{ {
for_file_renames.push_back(command); for_file_renames.push_back(command);
@ -1273,7 +1287,8 @@ void MergeTreeDataMergerMutator::splitMutationCommands(
/// we just don't read dropped columns /// we just don't read dropped columns
for (const auto & column : part->getColumns()) for (const auto & column : part->getColumns())
{ {
if (!removed_columns_from_compact_part.count(column.name) && !already_changed_columns.count(column.name)) if (!removed_columns_from_compact_part.count(column.name)
&& !already_changed_columns.count(column.name))
{ {
for_interpreter.emplace_back(MutationCommand for_interpreter.emplace_back(MutationCommand
{ {

View File

@ -81,14 +81,23 @@ MergeTreeReaderCompact::MergeTreeReaderCompact(
const auto & [name, type] = *name_and_type; const auto & [name, type] = *name_and_type;
auto position = data_part->getColumnPosition(name); auto position = data_part->getColumnPosition(name);
/// If array of Nested column is missing in part, if (!position)
/// we have to read it's offsets if they exists. {
auto renamed_it = alter_conversions.rename_map.find(name);
if (renamed_it != alter_conversions.rename_map.end())
position = data_part->getColumnPosition(renamed_it->second);
}
if (!position && typeid_cast<const DataTypeArray *>(type.get())) if (!position && typeid_cast<const DataTypeArray *>(type.get()))
{ {
/// If array of Nested column is missing in part,
/// we have to read it's offsets if they exists.
position = findColumnForOffsets(name); position = findColumnForOffsets(name);
read_only_offsets[i] = (position != std::nullopt); read_only_offsets[i] = (position != std::nullopt);
} }
column_positions[i] = std::move(position); column_positions[i] = std::move(position);
} }
@ -125,7 +134,15 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading,
if (!res_columns[pos]) if (!res_columns[pos])
continue; continue;
const auto & [name, type] = *name_and_type; auto [name, type] = *name_and_type;
if (alter_conversions.rename_map.count(name))
{
String original_name = alter_conversions.rename_map[name];
if (!data_part->getColumnPosition(name) && data_part->getColumnPosition(original_name))
name = original_name;
}
auto & column = mutable_columns[pos]; auto & column = mutable_columns[pos];
try try

View File

@ -52,12 +52,10 @@ public:
return part->storage.getInMemoryMetadata(); return part->storage.getInMemoryMetadata();
} }
bool hasSortingKey() const { return part->storage.hasSortingKey(); } bool hasSortingKey() const { return part->storage.hasSortingKey(); }
Names getSortingKeyColumns() const override { return part->storage.getSortingKeyColumns(); } Names getSortingKeyColumns() const override { return part->storage.getSortingKeyColumns(); }
protected: protected:
StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_)
: IStorage(getIDFromPart(part_), part_->storage.getVirtuals()) : IStorage(getIDFromPart(part_), part_->storage.getVirtuals())

View File

@ -1260,4 +1260,15 @@ CheckResults StorageMergeTree::checkData(const ASTPtr & query, const Context & c
return results; return results;
} }
MutationCommands StorageMergeTree::getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const
{
std::lock_guard lock(currently_processing_in_background_mutex);
auto it = current_mutations_by_version.upper_bound(part->info.getDataVersion());
if (it == current_mutations_by_version.end())
return {};
return it->second.commands;
}
} }

View File

@ -166,7 +166,7 @@ protected:
std::unique_ptr<MergeTreeSettings> settings_, std::unique_ptr<MergeTreeSettings> settings_,
bool has_force_restore_data_flag); bool has_force_restore_data_flag);
MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & /* part */) const override { return {}; } MutationCommands getFirtsAlterMutationCommandsForPart(const DataPartPtr & part) const override;
}; };
} }

View File

@ -0,0 +1,7 @@
1
1
date key renamed_value1 value2 value3
2019-10-02 1 1 1 1
7 7
date key renamed_value1 renamed_value2 renamed_value3
2019-10-02 7 7 7 7

View File

@ -0,0 +1,33 @@
DROP TABLE IF EXISTS table_with_compact_parts;
CREATE TABLE table_with_compact_parts
(
date Date,
key UInt64,
value1 String,
value2 String,
value3 String
)
ENGINE = MergeTree()
PARTITION BY date
ORDER BY key
settings index_granularity = 8,
min_rows_for_wide_part = 10;
INSERT INTO table_with_compact_parts SELECT toDate('2019-10-01') + number % 3, number, toString(number), toString(number), toString(number) from numbers(9);
SELECT value1 FROM table_with_compact_parts WHERE key = 1;
ALTER TABLE table_with_compact_parts RENAME COLUMN value1 to renamed_value1;
SELECT renamed_value1 FROM table_with_compact_parts WHERE key = 1;
SELECT * FROM table_with_compact_parts WHERE key = 1 FORMAT TSVWithNames;
ALTER TABLE table_with_compact_parts RENAME COLUMN value2 TO renamed_value2, RENAME COLUMN value3 TO renamed_value3;
SELECT renamed_value2, renamed_value3 FROM table_with_compact_parts WHERE key = 7;
SELECT * FROM table_with_compact_parts WHERE key = 7 FORMAT TSVWithNames;
DROP TABLE IF EXISTS table_with_compact_parts;