thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Minor refactor

Browse Source
This commit is contained in:
Julia Kartseva 2024-06-28 01:30:49 +00:00
parent 98ad45ba96
commit 27392fee6e
5 changed files with 28 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
View File

@ -16,15 +16,15 @@ CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(
}
ObjectStorageKey
CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & /* key_prefix */) const
CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const
{
const auto & [object_key_prefix, suffix_parts] = getLongestObjectKeyPrefix(path);
auto key = std::filesystem::path(object_key_prefix.empty() ? std::string() : object_key_prefix);
auto key = std::filesystem::path(object_key_prefix);
/// The longest prefix is the same as path, meaning that the path is already mapped.
if (suffix_parts.empty())
return ObjectStorageKey::createAsRelative(storage_key_prefix, std::move(key));
return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, std::move(key));
/// File and top-level directory paths are mapped as is.
if (!is_directory || object_key_prefix.empty())
@ -40,7 +40,7 @@ CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, c
key /= getRandomASCIIString(part_size);
}
return ObjectStorageKey::createAsRelative(storage_key_prefix, key);
return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, key);
}
std::tuple<std::string, std::vector<std::string>> CommonPathPrefixKeyGenerator::getLongestObjectKeyPrefix(const std::string & path) const

38
src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
View File

@ -80,9 +80,20 @@ std::vector<std::string> MetadataStorageFromPlainObjectStorage::listDirectory(co
object_storage->listObjects(abs_key, files, 0);
std::unordered_set<std::string> directories;
getDirectChildrenOnDisk(abs_key, object_storage->getCommonKeyPrefix(), files, path, directories);
return std::vector<std::string>(std::make_move_iterator(directories.begin()), std::make_move_iterator(directories.end()));
std::unordered_set<std::string> result;
for (const auto & elem : files)
{
const auto & p = elem->relative_path;
chassert(p.find(abs_key) == 0);
const auto child_pos = abs_key.size();
/// string::npos is ok.
const auto slash_pos = p.find('/', child_pos);
if (slash_pos == std::string::npos)
result.emplace(p.substr(child_pos));
else
result.emplace(p.substr(child_pos, slash_pos - child_pos));
}
return std::vector<std::string>(std::make_move_iterator(result.begin()), std::make_move_iterator(result.end()));
}
DirectoryIteratorPtr MetadataStorageFromPlainObjectStorage::iterateDirectory(const std::string & path) const
@ -102,27 +113,6 @@ StoredObjects MetadataStorageFromPlainObjectStorage::getStorageObjects(const std
return {StoredObject(object_key.serialize(), path, object_size)};
}
void MetadataStorageFromPlainObjectStorage::getDirectChildrenOnDisk(
const std::string & storage_key,
const std::string & /* storage_key_perfix */,
const RelativePathsWithMetadata & remote_paths,
const std::string & /* local_path */,
std::unordered_set<std::string> & result) const
{
for (const auto & elem : remote_paths)
{
const auto & path = elem->relative_path;
chassert(path.find(storage_key) == 0);
const auto child_pos = storage_key.size();
/// string::npos is ok.
const auto slash_pos = path.find('/', child_pos);
if (slash_pos == std::string::npos)
result.emplace(path.substr(child_pos));
else
result.emplace(path.substr(child_pos, slash_pos - child_pos));
}
}
const IMetadataStorage & MetadataStorageFromPlainObjectStorageTransaction::getStorageForNonTransactionalReads() const
{
return metadata_storage;

8
src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
View File

@ -86,14 +86,6 @@ protected:
/// Returns a map of local paths to paths in object storage.
virtual std::shared_ptr<PathMap> getPathMap() const { throwNotImplemented(); }
/// Retrieves the immediate files and directories within a given directory on a disk.
virtual void getDirectChildrenOnDisk(
const std::string & storage_key,
const std::string & storage_key_perfix,
const RelativePathsWithMetadata & remote_paths,
const std::string & local_path,
std::unordered_set<std::string> & result) const;
};
class MetadataStorageFromPlainObjectStorageTransaction final : public IMetadataTransaction, private MetadataOperationsHolder

26
src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
View File

@ -121,7 +121,7 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
return result;
}
void getDirectChildrenOnRewritableDisk(
void getDirectChildrenOnDiskImpl(
const std::string & storage_key,
const std::string & storage_key_perfix,
const RelativePathsWithMetadata & remote_paths,
@ -215,10 +215,8 @@ bool MetadataStorageFromPlainRewritableObjectStorage::exists(const std::string &
if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
{
auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
chassert(key_prefix.starts_with(object_storage->getCommonKeyPrefix()));
auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix.substr(object_storage->getCommonKeyPrefix().size());
return object_storage->existsOrHasAnyChild(metadata_key);
auto key_prefix = object_storage->generateObjectKeyForPath(path, getMetadataKeyPrefix()).serialize();
return object_storage->existsOrHasAnyChild(key_prefix);
}
return false;
@ -228,11 +226,8 @@ bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::str
{
if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
{
auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize()) / "";
chassert(directory.string().starts_with(object_storage->getCommonKeyPrefix()));
auto metadata_key
= std::filesystem::path(getMetadataKeyPrefix()) / directory.string().substr(object_storage->getCommonKeyPrefix().size());
return object_storage->existsOrHasAnyChild(metadata_key);
auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path, getMetadataKeyPrefix()).serialize()) / "";
return object_storage->existsOrHasAnyChild(directory);
}
else
return MetadataStorageFromPlainObjectStorage::isDirectory(path);
@ -240,12 +235,10 @@ bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::str
std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDirectory(const std::string & path) const
{
auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
auto key_prefix = object_storage->generateObjectKeyForPath(path, "" /* key_prefix */).serialize();
RelativePathsWithMetadata files;
std::string abs_key = key_prefix;
if (!abs_key.ends_with('/'))
abs_key += '/';
auto abs_key = std::filesystem::path(object_storage->getCommonKeyPrefix()) / key_prefix / "";
object_storage->listObjects(abs_key, files, 0);
@ -255,8 +248,7 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
/// metadata along with regular files.
if (object_storage->getCommonKeyPrefix() != getMetadataKeyPrefix())
{
chassert(abs_key.starts_with(object_storage->getCommonKeyPrefix()));
auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / abs_key.substr(object_storage->getCommonKeyPrefix().size());
auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix / "";
RelativePathsWithMetadata metadata_files;
object_storage->listObjects(metadata_key, metadata_files, 0);
getDirectChildrenOnDisk(metadata_key, getMetadataKeyPrefix(), metadata_files, path, directories);
@ -272,7 +264,7 @@ void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
const std::string & local_path,
std::unordered_set<std::string> & result) const
{
getDirectChildrenOnRewritableDisk(storage_key, storage_key_perfix, remote_paths, local_path, *getPathMap(), metadata_mutex, result);
getDirectChildrenOnDiskImpl(storage_key, storage_key_perfix, remote_paths, local_path, *getPathMap(), metadata_mutex, result);
}
}

2
src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
View File

@ -33,7 +33,7 @@ protected:
const std::string & storage_key_perfix,
const RelativePathsWithMetadata & remote_paths,
const std::string & local_path,
std::unordered_set<std::string> & result) const override;
std::unordered_set<std::string> & result) const;
};
}