thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Test for multi-layer cache

Browse Source
This commit is contained in:
kssenii 2022-07-14 17:30:44 +02:00
parent 31f0abe672
commit 9aa7e8aed8
13 changed files with 81 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Disks/DiskDecorator.cpp
View File

@ -236,9 +236,9 @@ void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & c
delegate->applyNewSettings(config, context, config_prefix, map);
}
DiskObjectStoragePtr DiskDecorator::createDiskObjectStorage(const String & name)
DiskObjectStoragePtr DiskDecorator::createDiskObjectStorage()
{
return delegate->createDiskObjectStorage(name);
return delegate->createDiskObjectStorage();
}
}

6
src/Disks/DiskDecorator.h
View File

@ -88,8 +88,8 @@ public:
StoredObjects getStorageObjects(const String & path) const override { return delegate->getStorageObjects(path); }
void getRemotePathsRecursive(const String & path, std::vector<LocalPathWithObjectStoragePaths> & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); }
DiskObjectStoragePtr createDiskObjectStorage(const String &) override;
const std::unordered_set<String> & getCacheLayersNames() const override { return delegate->getCacheLayersNames(); }
DiskObjectStoragePtr createDiskObjectStorage() override;
NameSet getCacheLayersNames() const override { return delegate->getCacheLayersNames(); }
MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); }
@ -97,8 +97,6 @@ public:
UInt32 getRefCount(const String & path) const override { return delegate->getRefCount(path); }
DiskPtr getWrappedDisk() const override { return delegate; }
void syncRevision(UInt64 revision) override { delegate->syncRevision(revision); }
UInt64 getRevision() const override { return delegate->getRevision(); }

6
src/Disks/DiskLocal.cpp
View File

@ -596,7 +596,7 @@ catch (...)
return false;
}
DiskObjectStoragePtr DiskLocal::createDiskObjectStorage(const String & name_)
DiskObjectStoragePtr DiskLocal::createDiskObjectStorage()
{
auto object_storage = std::make_shared<LocalObjectStorage>();
auto metadata_storage = std::make_shared<FakeMetadataStorageFromDisk>(
@ -605,9 +605,9 @@ DiskObjectStoragePtr DiskLocal::createDiskObjectStorage(const String & name_)
/* object_storage_root_path */getPath());
return std::make_shared<DiskObjectStorage>(
name_,
getName(),
disk_path,
"DiskLocalObjectStorage",
"Local",
metadata_storage,
object_storage,
DiskType::Local,

2
src/Disks/DiskLocal.h
View File

@ -122,7 +122,7 @@ public:
bool canRead() const noexcept;
bool canWrite() const noexcept;
DiskObjectStoragePtr createDiskObjectStorage(const String & name_) override;
DiskObjectStoragePtr createDiskObjectStorage() override;
private:
std::optional<UInt64> tryReserve(UInt64 bytes);

9
src/Disks/IDisk.h
View File

@ -222,16 +222,11 @@ public:
virtual bool supportsCache() const { return false; }
virtual const std::unordered_set<String> & getCacheLayersNames() const
virtual NameSet getCacheLayersNames() const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getCacheLayersNames()` is not implemented for disk: {}", getType());
}
virtual DiskPtr getWrappedDisk() const
{
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getWrappedDisk()` is not implemented for disk: {}", getType());
}
/// Returns a list of storage objects (contains path, size, ...).
/// (A list is returned because for Log family engines there might
/// be multiple files in remote fs for single clickhouse file.
@ -351,7 +346,7 @@ public:
/// Return current disk revision.
virtual UInt64 getRevision() const { return 0; }
virtual DiskObjectStoragePtr createDiskObjectStorage(const String &)
virtual DiskObjectStoragePtr createDiskObjectStorage()
{
throw Exception(
ErrorCodes::NOT_IMPLEMENTED,

8
src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
View File

@ -21,10 +21,14 @@ namespace ErrorCodes
extern const int CANNOT_STAT;
}
CachedObjectStorage:: CachedObjectStorage(ObjectStoragePtr object_storage_, FileCachePtr cache_)
CachedObjectStorage:: CachedObjectStorage(
ObjectStoragePtr object_storage_,
FileCachePtr cache_,
const std::string & cache_config_name_)
: object_storage(object_storage_)
, cache(cache_)
, log(&Poco::Logger::get(getName() + "(" + object_storage_->getName() +")"))
, cache_config_name(cache_config_name_)
, log(&Poco::Logger::get(getName()))
{
cache->initialize();
}

7
src/Disks/ObjectStorages/Cached/CachedObjectStorage.h
View File

@ -14,7 +14,7 @@ namespace DB
class CachedObjectStorage : public IObjectStorage
{
public:
CachedObjectStorage(ObjectStoragePtr object_storage_, FileCachePtr cache_);
CachedObjectStorage(ObjectStoragePtr object_storage_, FileCachePtr cache_, const String & cache_config_name_);
std::string getName() const override { return "CachedObjectStorage(" + object_storage->getName() + ")"; }
@ -95,6 +95,10 @@ public:
bool isReadOnly() const override { return object_storage->isReadOnly(); }
const std::string & getCacheConfigName() const { return cache_config_name; }
ObjectStoragePtr getWrappedObjectStorage() { return object_storage; }
private:
IFileCache::Key getCacheKey(const std::string & path) const;
String getCachePath(const std::string & path) const;
@ -102,6 +106,7 @@ private:
ObjectStoragePtr object_storage;
FileCachePtr cache;
std::string cache_config_name;
Poco::Logger * log;
};

4
src/Disks/ObjectStorages/Cached/registerDiskCache.cpp
View File

@ -35,6 +35,7 @@ void registerDiskCache(DiskFactory & factory)
"There is not disk with name `{}`, disk name should be initialized before cache disk",
disk_name);
}
/// TODO: Add a check that there underlying storage is unique among cached storages.
FileCacheSettings file_cache_settings;
file_cache_settings.loadFromConfig(config, config_prefix);
@ -44,9 +45,10 @@ void registerDiskCache(DiskFactory & factory)
fs::create_directories(cache_base_path);
auto disk = disk_it->second;
auto disk_object_storage = disk->createDiskObjectStorage(disk_name);
auto cache = FileCacheFactory::instance().getOrCreate(cache_base_path, file_cache_settings, name);
auto disk_object_storage = disk->createDiskObjectStorage();
disk_object_storage->wrapWithCache(cache, name);
LOG_TEST(

23
src/Disks/ObjectStorages/DiskObjectStorage.cpp
View File

@ -445,12 +445,12 @@ bool DiskObjectStorage::isReadOnly() const
return object_storage->isReadOnly();
}
DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage(const String & name_)
DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage()
{
return std::make_shared<DiskObjectStorage>(
name_,
getName(),
object_storage_root_path,
name,
getName(),
metadata_storage,
object_storage,
disk_type,
@ -460,8 +460,21 @@ DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage(const String & n
void DiskObjectStorage::wrapWithCache(FileCachePtr cache, const String & layer_name)
{
object_storage = std::make_shared<CachedObjectStorage>(object_storage, cache);
cache_layers.insert(layer_name);
object_storage = std::make_shared<CachedObjectStorage>(object_storage, cache, layer_name);
}
NameSet DiskObjectStorage::getCacheLayersNames() const
{
NameSet cache_layers;
auto current_object_storage = object_storage;
while (current_object_storage->supportsCache())
{
auto * cached_object_storage = assert_cast<CachedObjectStorage *>(current_object_storage.get());
cache_layers.insert(cached_object_storage->getCacheConfigName());
std::cerr << "\n\n" << "having cache layer: " << cached_object_storage->getCacheConfigName() << " for " << object_storage->getName() << "\n\n";
current_object_storage = cached_object_storage->getWrappedObjectStorage();
}
return cache_layers;
}
template <typename T>

6
src/Disks/ObjectStorages/DiskObjectStorage.h
View File

@ -164,7 +164,7 @@ public:
UInt64 getRevision() const override;
DiskObjectStoragePtr createDiskObjectStorage(const String & name_) override;
DiskObjectStoragePtr createDiskObjectStorage() override;
bool supportsCache() const override;
@ -186,7 +186,7 @@ public:
String getStructure() const { return "DiskObjectStorage(" + object_storage->getName() + ")"; }
/// Get names of all cache layers. Name is how cache is defined in configuration file.
const std::unordered_set<String> & getCacheLayersNames() const override { return cache_layers; }
NameSet getCacheLayersNames() const override;
private:
@ -212,8 +212,6 @@ private:
size_t threadpool_size;
std::unique_ptr<DiskObjectStorageRemoteMetadataRestoreHelper> metadata_helper;
std::unordered_set<String> cache_layers;
};
using DiskObjectStoragePtr = std::shared_ptr<DiskObjectStorage>;

33
tests/config/config.d/storage_conf.xml
View File

@ -30,6 +30,13 @@
<access_key_id>clickhouse</access_key_id>
<secret_access_key>clickhouse</secret_access_key>
</s3_disk_4>
<s3_disk_5>
<type>s3</type>
<path>s3_disk_5/</path>
<endpoint>http://localhost:11111/test/00170_test/</endpoint>
<access_key_id>clickhouse</access_key_id>
<secret_access_key>clickhouse</secret_access_key>
</s3_disk_5>
<!-- cache for s3 disks -->
<s3_cache>
<type>cache</type>
@ -61,6 +68,12 @@
<cache_on_write_operations>1</cache_on_write_operations>
<enable_filesystem_query_cache_limit>1</enable_filesystem_query_cache_limit>
</s3_cache_4>
<s3_cache_5>
<type>cache</type>
<disk>s3_disk_5</disk>
<path>s3_cache_5/</path>
<max_size>22548578304</max_size>
</s3_cache_5>
<!-- local disks -->
<local_disk>
<type>local</type>
@ -96,6 +109,19 @@
<cache_on_write_operations>1</cache_on_write_operations>
<enable_cache_hits_threshold>1</enable_cache_hits_threshold>
</local_cache_3>
<!-- multi layer cache -->
<s3_cache_multi>
<type>cache</type>
<disk>s3_cache_5</disk>
<path>s3_cache_multi/</path>
<max_size>22548578304</max_size>
</s3_cache_multi>
<s3_cache_multi_2>
<type>cache</type>
<disk>s3_cache_multi</disk>
<path>s3_cache_multi_2/</path>
<max_size>22548578304</max_size>
</s3_cache_multi_2>
</disks>
<policies>
<s3_cache>
@ -126,6 +152,13 @@
</main>
</volumes>
</s3_cache_4>
<s3_cache_multi>
<volumes>
<main>
<disk>s3_cache_multi_2</disk>
</main>
</volumes>
</s3_cache_multi>
<local_cache>
<volumes>
<main>

2
tests/queries/0_stateless/02313_filesystem_cache_seeks.reference
View File

@ -2,3 +2,5 @@ Using storage policy: s3_cache
Using storage policy: local_cache
Using storage policy: s3_cache_multi

2
tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
View File

@ -11,7 +11,7 @@ TMP_PATH=${CLICKHOUSE_TEST_UNIQUE_NAME}
QUERIES_FILE=02313_filesystem_cache_seeks.queries
TEST_FILE=$CUR_DIR/filesystem_cache_queries/$QUERIES_FILE
for storagePolicy in 's3_cache' 'local_cache'; do
for storagePolicy in 's3_cache' 'local_cache' 's3_cache_multi'; do
echo "Using storage policy: $storagePolicy"
cat $TEST_FILE | sed -e "s/_storagePolicy/${storagePolicy}/" > $TMP_PATH
${CLICKHOUSE_CLIENT} --queries-file $TMP_PATH