thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-21 09:10:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Altering primary key [#METR-21119].

Browse Source
This commit is contained in:
Alexey Milovidov 2016-05-14 00:08:19 +03:00
parent a078cfa609
commit 6942819013
14 changed files with 147 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dbms/include/DB/Databases/DatabaseCloud.h
View File

@ -105,6 +105,18 @@ public:
void shutdown() override;
void drop() override;
void alterTable(
const Context & context,
const String & name,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
const ASTModifier & engine_modifier) override
{
throw Exception("ALTER TABLE is not supported by database engine " + getEngineName(), ErrorCodes::NOT_IMPLEMENTED);
}
using Hash = UInt128;
private:

9
dbms/include/DB/Databases/DatabaseOrdinary.h
View File

@ -45,6 +45,15 @@ public:
void shutdown() override;
void drop() override;
void alterTable(
const Context & context,
const String & name,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
const ASTModifier & engine_modifier) override;
};
}

13
dbms/include/DB/Databases/IDatabase.h
View File

@ -68,6 +68,19 @@ public:
/// Переименовать таблицу и, возможно, переместить таблицу в другую БД.
virtual void renameTable(const Context & context, const String & name, IDatabase & to_database, const String & to_name) = 0;
using ASTModifier = std::function<void(ASTPtr &)>;
/// Изменить структуру таблицы в метаданных.
/// Нужно вызывать под TableStructureLock соответствующей таблицы. Если engine_modifier пустой, то engine не изменяется.
virtual void alterTable(
const Context & context,
const String & name,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
const ASTModifier & engine_modifier);
/// Получить запрос CREATE TABLE для таблицы. Может выдавать информацию и для detached таблиц, для которых есть метаданные.
virtual ASTPtr getCreateQuery(const String & name) const = 0;

10
dbms/include/DB/Interpreters/InterpreterAlterQuery.h
View File

@ -20,16 +20,6 @@ public:
BlockIO execute() override;
/** Изменяет список столбцов в метаданных таблицы на диске. Нужно вызывать под TableStructureLock соответствующей таблицы.
*/
static void updateMetadata(const String & database,
const String & table,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
const Context & context);
private:
struct PartitionCommand
{

6
dbms/include/DB/Storages/MergeTree/MergeTreeData.h
View File

@ -405,7 +405,11 @@ public:
* Если измененных столбцов подозрительно много, и !skip_sanity_checks, бросает исключение.
* Если никаких действий над данными не требуется, возвращает nullptr.
*/
AlterDataPartTransactionPtr alterDataPart(const DataPartPtr & part, const NamesAndTypesList & new_columns, bool skip_sanity_checks = false);
AlterDataPartTransactionPtr alterDataPart(
const DataPartPtr & part,
const NamesAndTypesList & new_columns,
const NamesAndTypesList & new_primary_key,
bool skip_sanity_checks);
/// Нужно вызывать под залоченным lockStructureForAlter().
void setColumnsList(const NamesAndTypesList & new_columns) { columns = new NamesAndTypesList(new_columns); }

62
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -8,6 +8,7 @@
#include <DB/Parsers/parseQuery.h>
#include <DB/Parsers/ParserCreateQuery.h>
#include <DB/Interpreters/Context.h>
#include <DB/Interpreters/InterpreterCreateQuery.h>
#include <DB/IO/WriteBufferFromFile.h>
#include <DB/IO/ReadBufferFromFile.h>
#include <DB/IO/copyData.h>
@ -23,6 +24,7 @@ namespace ErrorCodes
extern const int TABLE_METADATA_DOESNT_EXIST;
extern const int CANNOT_CREATE_TABLE_FROM_METADATA;
extern const int INCORRECT_FILE_NAME;
extern const int LOGICAL_ERROR;
}
@ -434,4 +436,64 @@ void DatabaseOrdinary::drop()
/// Дополнительных действий по удалению не требуется.
}
void DatabaseOrdinary::alterTable(
const Context & context,
const String & name,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
const ASTModifier & engine_modifier)
{
/// Считываем определение таблицы и заменяем в нём нужные части на новые.
String table_name_escaped = escapeForFileName(name);
String table_metadata_tmp_path = path + "/" + table_name_escaped + ".sql.tmp";
String table_metadata_path = path + "/" + table_name_escaped + ".sql";
String statement;
{
char in_buf[METADATA_FILE_BUFFER_SIZE];
ReadBufferFromFile in(table_metadata_path, METADATA_FILE_BUFFER_SIZE, -1, in_buf);
WriteBufferFromString out(statement);
copyData(in, out);
}
ParserCreateQuery parser;
ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path);
ASTCreateQuery & ast_create_query = typeid_cast<ASTCreateQuery &>(*ast);
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns, materialized_columns, alias_columns, column_defaults);
auto it = std::find(ast_create_query.children.begin(), ast_create_query.children.end(), ast_create_query.columns);
if (it == ast_create_query.children.end())
throw Exception("Logical error: cannot find columns child in ASTCreateQuery", ErrorCodes::LOGICAL_ERROR);
*it = new_columns;
if (engine_modifier)
engine_modifier(ast_create_query.storage);
statement = getTableDefinitionFromCreateQuery(ast_create_query);
{
WriteBufferFromFile out(table_metadata_tmp_path, statement.size(), O_WRONLY | O_CREAT | O_EXCL);
writeString(statement, out);
out.next();
out.sync();
out.close();
}
try
{
/// rename атомарно заменяет старый файл новым.
Poco::File(table_metadata_tmp_path).renameTo(table_metadata_path);
}
catch (...)
{
Poco::File(table_metadata_tmp_path).remove();
throw;
}
}
}

43
dbms/src/Interpreters/InterpreterAlterQuery.cpp
View File

@ -212,47 +212,4 @@ void InterpreterAlterQuery::parseAlter(
}
}
void InterpreterAlterQuery::updateMetadata(
const String & database_name,
const String & table_name,
const NamesAndTypesList & columns,
const NamesAndTypesList & materialized_columns,
const NamesAndTypesList & alias_columns,
const ColumnDefaults & column_defaults,
const Context & context)
{
String path = context.getPath();
String database_name_escaped = escapeForFileName(database_name);
String table_name_escaped = escapeForFileName(table_name);
String metadata_path = path + "metadata/" + database_name_escaped + "/" + table_name_escaped + ".sql";
String metadata_temp_path = metadata_path + ".tmp";
StringPtr query = new String();
{
ReadBufferFromFile in(metadata_path);
WriteBufferFromString out(*query);
copyData(in, out);
}
ParserCreateQuery parser;
ASTPtr ast = parseQuery(parser, query->data(), query->data() + query->size(), "in file " + metadata_path);
ast->query_string = query;
ASTCreateQuery & attach = typeid_cast<ASTCreateQuery &>(*ast);
ASTPtr new_columns = InterpreterCreateQuery::formatColumns(columns, materialized_columns, alias_columns, column_defaults);
*std::find(attach.children.begin(), attach.children.end(), attach.columns) = new_columns;
attach.columns = new_columns;
{
Poco::FileOutputStream ostr(metadata_temp_path);
formatAST(attach, ostr, 0, false);
}
Poco::File(metadata_temp_path).renameTo(metadata_path);
}
}

15
dbms/src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -515,7 +515,8 @@ void MergeTreeData::checkAlter(const AlterCommands & params)
if (primary_expr)
keys = primary_expr->getRequiredColumns();
keys.push_back(merging_params.sign_column);
if (!merging_params.sign_column.empty())
keys.push_back(merging_params.sign_column);
std::sort(keys.begin(), keys.end());
@ -533,7 +534,10 @@ void MergeTreeData::checkAlter(const AlterCommands & params)
/// augment plain columns with materialized columns for convert expression creation
new_columns.insert(std::end(new_columns),
std::begin(new_materialized_columns), std::end(new_materialized_columns));
createConvertExpression(nullptr, getColumnsList(), new_columns, unused_expression, unused_map, unused_bool);
}
void MergeTreeData::createConvertExpression(const DataPartPtr & part, const NamesAndTypesList & old_columns, const NamesAndTypesList & new_columns,
@ -632,7 +636,10 @@ void MergeTreeData::createConvertExpression(const DataPartPtr & part, const Name
}
MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart(
const DataPartPtr & part, const NamesAndTypesList & new_columns, bool skip_sanity_checks)
const DataPartPtr & part,
const NamesAndTypesList & new_columns,
const NamesAndTypesList & new_primary_key,
bool skip_sanity_checks)
{
ExpressionActionsPtr expression;
AlterDataPartTransactionPtr transaction(new AlterDataPartTransaction(part)); /// Блокирует изменение куска.
@ -643,8 +650,8 @@ MergeTreeData::AlterDataPartTransactionPtr MergeTreeData::alterDataPart(
{
transaction->clear();
throw Exception("Suspiciously many (" + toString(transaction->rename_map.size()) + ") files need to be modified in part " + part->name
+ ". Aborting just in case");
throw Exception("Suspiciously many (" + toString(transaction->rename_map.size())
+ ") files need to be modified in part " + part->name + ". Aborting just in case");
}
if (transaction->rename_map.empty() && !force_update_metadata)

10
dbms/src/Storages/MergeTree/ReplicatedMergeTreeAlterThread.cpp
View File

@ -4,6 +4,7 @@
#include <DB/Storages/ColumnsDescription.h>
#include <DB/Storages/StorageReplicatedMergeTree.h>
#include <DB/Storages/MergeTree/ReplicatedMergeTreeAlterThread.h>
#include <DB/Databases/IDatabase.h>
namespace DB
@ -95,8 +96,9 @@ void ReplicatedMergeTreeAlterThread::run()
{
LOG_INFO(log, "Columns list changed in ZooKeeper. Applying changes locally.");
InterpreterAlterQuery::updateMetadata(storage.database_name, storage.table_name, columns,
materialized_columns, alias_columns, column_defaults, storage.context);
storage.context.getDatabase(storage.database_name)->alterTable(
storage.context, storage.table_name,
columns, materialized_columns, alias_columns, column_defaults, {});
if (columns_changed)
{
@ -157,7 +159,7 @@ void ReplicatedMergeTreeAlterThread::run()
/// Обновим кусок и запишем результат во временные файлы.
/// TODO: Можно пропускать проверку на слишком большие изменения, если в ZooKeeper есть, например,
/// нода /flags/force_alter.
auto transaction = storage.data.alterDataPart(part, columns_plus_materialized);
auto transaction = storage.data.alterDataPart(part, columns_plus_materialized, false);
if (!transaction)
continue;
@ -195,7 +197,7 @@ void ReplicatedMergeTreeAlterThread::run()
for (const MergeTreeData::DataPartPtr & part : parts)
{
auto transaction = storage.unreplicated_data->alterDataPart(part, columns_plus_materialized);
auto transaction = storage.unreplicated_data->alterDataPart(part, columns_plus_materialized, false);
if (!transaction)
continue;

7
dbms/src/Storages/StorageBuffer.cpp
View File

@ -2,6 +2,7 @@
#include <DB/Interpreters/InterpreterInsertQuery.h>
#include <DB/Interpreters/InterpreterAlterQuery.h>
#include <DB/DataStreams/IProfilingBlockInputStream.h>
#include <DB/Databases/IDatabase.h>
#include <DB/Storages/StorageBuffer.h>
#include <DB/Parsers/ASTInsertQuery.h>
#include <DB/Parsers/ASTIdentifier.h>
@ -508,8 +509,10 @@ void StorageBuffer::alter(const AlterCommands & params, const String & database_
optimize(context.getSettings());
params.apply(*columns, materialized_columns, alias_columns, column_defaults);
InterpreterAlterQuery::updateMetadata(database_name, table_name,
*columns, materialized_columns, alias_columns, column_defaults, context);
context.getDatabase(database_name)->alterTable(
context, table_name,
*columns, materialized_columns, alias_columns, column_defaults, {});
}
}

10
dbms/src/Storages/StorageDistributed.cpp
View File

@ -2,12 +2,16 @@
#include <DB/DataStreams/BlockExtraInfoInputStream.h>
#include <DB/DataStreams/UnionBlockInputStream.h>
#include <DB/Databases/IDatabase.h>
#include <DB/Storages/StorageDistributed.h>
#include <DB/Storages/VirtualColumnFactory.h>
#include <DB/Storages/Distributed/DistributedBlockOutputStream.h>
#include <DB/Storages/Distributed/DirectoryMonitor.h>
#include <DB/Storages/MergeTree/ReshardingWorker.h>
#include <DB/Common/escapeForFileName.h>
#include <DB/Parsers/ASTInsertQuery.h>
#include <DB/Parsers/ASTSelectQuery.h>
#include <DB/Parsers/ASTIdentifier.h>
@ -222,8 +226,10 @@ void StorageDistributed::alter(const AlterCommands & params, const String & data
auto lock = lockStructureForAlter();
params.apply(*columns, materialized_columns, alias_columns, column_defaults);
InterpreterAlterQuery::updateMetadata(database_name, table_name,
*columns, materialized_columns, alias_columns, column_defaults, context);
context.getDatabase(database_name)->alterTable(
context, table_name,
*columns, materialized_columns, alias_columns, column_defaults, {});
}
void StorageDistributed::shutdown()

6
dbms/src/Storages/StorageMerge.cpp
View File

@ -230,8 +230,10 @@ void StorageMerge::alter(const AlterCommands & params, const String & database_n
auto lock = lockStructureForAlter();
params.apply(*columns, materialized_columns, alias_columns, column_defaults);
InterpreterAlterQuery::updateMetadata(database_name, table_name, *columns,
materialized_columns, alias_columns, column_defaults, context);
context.getDatabase(database_name)->alterTable(
context, table_name,
*columns, materialized_columns, alias_columns, column_defaults, {});
}
}

13
dbms/src/Storages/StorageMergeTree.cpp
View File

@ -4,6 +4,7 @@
#include <DB/Storages/MergeTree/DiskSpaceMonitor.h>
#include <DB/Storages/MergeTree/MergeList.h>
#include <DB/Storages/MergeTree/MergeTreeWhereOptimizer.h>
#include <DB/Databases/IDatabase.h>
#include <DB/Common/escapeForFileName.h>
#include <DB/Interpreters/InterpreterAlterQuery.h>
#include <Poco/DirectoryIterator.h>
@ -180,16 +181,16 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa
MergeTreeData::DataParts parts = data.getDataParts();
std::vector<MergeTreeData::AlterDataPartTransactionPtr> transactions;
for (const MergeTreeData::DataPartPtr & part : parts)
{
if (auto transaction = data.alterDataPart(part, columns_for_parts))
if (auto transaction = data.alterDataPart(part, columns_for_parts, false))
transactions.push_back(std::move(transaction));
}
auto table_hard_lock = lockStructureForAlter();
InterpreterAlterQuery::updateMetadata(database_name, table_name, new_columns,
new_materialized_columns, new_alias_columns, new_column_defaults, context);
context.getDatabase(database_name)->alterTable(
context, table_name,
new_columns, new_materialized_columns, new_alias_columns, new_column_defaults, {});
materialized_columns = new_materialized_columns;
alias_columns = new_alias_columns;
@ -201,9 +202,7 @@ void StorageMergeTree::alter(const AlterCommands & params, const String & databa
data.column_defaults = std::move(new_column_defaults);
for (auto & transaction : transactions)
{
transaction->commit();
}
}
bool StorageMergeTree::merge(size_t aio_threshold, bool aggressive, BackgroundProcessingPool::Context * pool_context)

8
dbms/src/Storages/StorageReplicatedMergeTree.cpp
View File

@ -12,6 +12,8 @@
#include <DB/Storages/MergeTree/ReplicatedMergeTreeAddress.h>
#include <DB/Storages/MergeTree/ReshardingWorker.h>
#include <DB/Databases/IDatabase.h>
#include <DB/Parsers/formatAST.h>
#include <DB/Parsers/ASTInsertQuery.h>
@ -581,8 +583,10 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo
LOG_WARNING(log, "Table structure in ZooKeeper is a little different from local table structure. Assuming ALTER.");
/// Без всяких блокировок, потому что таблица еще не создана.
InterpreterAlterQuery::updateMetadata(database_name, table_name, columns,
materialized_columns, alias_columns, column_defaults, context);
context.getDatabase(database_name)->alterTable(
context, table_name,
columns, materialized_columns, alias_columns, column_defaults, {});
data.setColumnsList(columns);
data.materialized_columns = std::move(materialized_columns);
data.alias_columns = std::move(alias_columns);