Fix some more tests.

This commit is contained in:
Nikolai Kochetov 2022-05-05 09:19:12 +00:00
parent fc3645a483
commit b3d38301c9
5 changed files with 70 additions and 20 deletions

View File

@ -189,6 +189,10 @@ void DataPartStorageOnDisk::remove(
{ {
std::string proj_dir_name = projection.name + ".proj"; std::string proj_dir_name = projection.name + ".proj";
projection_directories.emplace(proj_dir_name); projection_directories.emplace(proj_dir_name);
clearDirectory(
fs::path(to) / proj_dir_name,
can_remove_shared_data, names_not_to_remove, projection.checksums, {}, log, true);
} }
clearDirectory(to, can_remove_shared_data, names_not_to_remove, checksums, projection_directories, log, false); clearDirectory(to, can_remove_shared_data, names_not_to_remove, checksums, projection_directories, log, false);
@ -552,6 +556,13 @@ void DataPartStorageOnDisk::rename(const String & new_relative_path, Poco::Logge
"Part directory {} doesn't exist. Most likely it is a logical error.", "Part directory {} doesn't exist. Most likely it is a logical error.",
std::string(fs::path(volume->getDisk()->getPath()) / root_path / part_dir)); std::string(fs::path(volume->getDisk()->getPath()) / root_path / part_dir));
auto new_path = fs::path(root_path) / new_relative_path;
if (!new_path.has_filename())
throw Exception(ErrorCodes::LOGICAL_ERROR,
"Cannot rename from {} to {}. Destination should not contain trailing slash",
getFullRelativePath(),
new_relative_path);
/// Why "" ? /// Why "" ?
String to = fs::path(root_path) / new_relative_path / ""; String to = fs::path(root_path) / new_relative_path / "";
@ -585,7 +596,8 @@ void DataPartStorageOnDisk::rename(const String & new_relative_path, Poco::Logge
/// Why? /// Why?
volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr))); volume->getDisk()->setLastModified(from, Poco::Timestamp::fromEpochTime(time(nullptr)));
volume->getDisk()->moveDirectory(from, to); volume->getDisk()->moveDirectory(from, to);
part_dir = new_relative_path; part_dir = new_path.filename();
root_path = new_path.remove_filename();
// metadata_manager->updateAll(true); // metadata_manager->updateAll(true);
SyncGuardPtr sync_guard; SyncGuardPtr sync_guard;
@ -593,6 +605,32 @@ void DataPartStorageOnDisk::rename(const String & new_relative_path, Poco::Logge
sync_guard = volume->getDisk()->getDirectorySyncGuard(getFullRelativePath()); sync_guard = volume->getDisk()->getDirectorySyncGuard(getFullRelativePath());
} }
void DataPartStorageOnDisk::changeRootPath(const std::string & from_root, const std::string & to_root)
{
/// This is a very dumb implementation, here for root path like
/// "some/current/path/to/part" and change like
/// "some/current" -> "other/different", we just replace prefix to make new root like
/// "other/different/path/to/part".
/// Here we expect that actual move was done by somebody else.
size_t prefix_size = from_root.size();
if (prefix_size > 0 && from_root.back() == '/')
--prefix_size;
if (prefix_size > root_path.size()
|| std::string_view(from_root).substr(0, prefix_size) != std::string_view(root_path).substr(0, prefix_size))
throw Exception(
ErrorCodes::LOGICAL_ERROR,
"Cannot change part root to {} because it is not a prefix of current root {}",
from_root, root_path);
size_t dst_size = to_root.size();
if (dst_size > 0 && to_root.back() == '/')
--dst_size;
root_path = to_root.substr(0, dst_size) + root_path.substr(prefix_size);
}
bool DataPartStorageOnDisk::shallParticipateInMerges(const IStoragePolicy & storage_policy) const bool DataPartStorageOnDisk::shallParticipateInMerges(const IStoragePolicy & storage_policy) const
{ {
/// `IMergeTreeDataPart::volume` describes space where current part belongs, and holds /// `IMergeTreeDataPart::volume` describes space where current part belongs, and holds

View File

@ -99,6 +99,8 @@ public:
void rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync_part_dir) override; void rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync_part_dir) override;
void changeRootPath(const std::string & from_root, const std::string & to_root) override;
std::string getName() const override; std::string getName() const override;
std::string getDiskType() const override; std::string getDiskType() const override;

View File

@ -165,7 +165,11 @@ public:
const std::string & dir_path, const std::string & dir_path,
Poco::Logger * log) const = 0; Poco::Logger * log) const = 0;
virtual void rename(const String & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync_part_dir) = 0; virtual void rename(const std::string & new_relative_path, Poco::Logger * log, bool remove_new_dir_if_exists, bool fsync_part_dir) = 0;
/// Change part's root. From should be a prefix path of current root path.
/// Right now, this is needed for rename table query.
virtual void changeRootPath(const std::string & from_root, const std::string & to_root) = 0;
/// Disk name /// Disk name
virtual std::string getName() const = 0; virtual std::string getName() const = 0;

View File

@ -1376,16 +1376,19 @@ try
{ {
assertOnDisk(); assertOnDisk();
std::string relative_path = storage.relative_data_path;
bool fsync_dir = storage.getSettings()->fsync_part_directory;
if (parent_part) if (parent_part)
throw Exception( {
ErrorCodes::LOGICAL_ERROR, /// For projections, move is only possible inside parent part dir.
"Move is not supported for projection parts: moving form {} to {}", relative_path = parent_part->data_part_storage->getFullRelativePath();
data_part_storage->getFullPath(), new_relative_path); }
String from = data_part_storage->getFullRelativePath(); String from = data_part_storage->getFullRelativePath();
String to = fs::path(storage.relative_data_path) / new_relative_path / ""; String to = fs::path(relative_path) / new_relative_path / "";
data_part_storage->rename(new_relative_path, storage.log, remove_new_dir_if_exists, storage.getSettings()->fsync_part_directory); data_part_storage->rename(new_relative_path, storage.log, remove_new_dir_if_exists, fsync_dir);
metadata_manager->move(from, to); metadata_manager->move(from, to);
for (const auto & [p_name, part] : projection_parts) for (const auto & [p_name, part] : projection_parts)

View File

@ -1893,6 +1893,9 @@ void MergeTreeData::rename(const String & new_table_path, const StorageID & new_
if (!getStorageID().hasUUID()) if (!getStorageID().hasUUID())
getContext()->dropCaches(); getContext()->dropCaches();
for (const auto & part : data_parts_by_info)
part->data_part_storage->changeRootPath(relative_data_path, new_table_path);
relative_data_path = new_table_path; relative_data_path = new_table_path;
renameInMemory(new_table_id); renameInMemory(new_table_id);
} }