mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 08:40:50 +00:00
Test for multi-layer cache
This commit is contained in:
parent
31f0abe672
commit
9aa7e8aed8
@ -236,9 +236,9 @@ void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & c
|
|||||||
delegate->applyNewSettings(config, context, config_prefix, map);
|
delegate->applyNewSettings(config, context, config_prefix, map);
|
||||||
}
|
}
|
||||||
|
|
||||||
DiskObjectStoragePtr DiskDecorator::createDiskObjectStorage(const String & name)
|
DiskObjectStoragePtr DiskDecorator::createDiskObjectStorage()
|
||||||
{
|
{
|
||||||
return delegate->createDiskObjectStorage(name);
|
return delegate->createDiskObjectStorage();
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -88,8 +88,8 @@ public:
|
|||||||
StoredObjects getStorageObjects(const String & path) const override { return delegate->getStorageObjects(path); }
|
StoredObjects getStorageObjects(const String & path) const override { return delegate->getStorageObjects(path); }
|
||||||
void getRemotePathsRecursive(const String & path, std::vector<LocalPathWithObjectStoragePaths> & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); }
|
void getRemotePathsRecursive(const String & path, std::vector<LocalPathWithObjectStoragePaths> & paths_map) override { return delegate->getRemotePathsRecursive(path, paths_map); }
|
||||||
|
|
||||||
DiskObjectStoragePtr createDiskObjectStorage(const String &) override;
|
DiskObjectStoragePtr createDiskObjectStorage() override;
|
||||||
const std::unordered_set<String> & getCacheLayersNames() const override { return delegate->getCacheLayersNames(); }
|
NameSet getCacheLayersNames() const override { return delegate->getCacheLayersNames(); }
|
||||||
|
|
||||||
MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); }
|
MetadataStoragePtr getMetadataStorage() override { return delegate->getMetadataStorage(); }
|
||||||
|
|
||||||
@ -97,8 +97,6 @@ public:
|
|||||||
|
|
||||||
UInt32 getRefCount(const String & path) const override { return delegate->getRefCount(path); }
|
UInt32 getRefCount(const String & path) const override { return delegate->getRefCount(path); }
|
||||||
|
|
||||||
DiskPtr getWrappedDisk() const override { return delegate; }
|
|
||||||
|
|
||||||
void syncRevision(UInt64 revision) override { delegate->syncRevision(revision); }
|
void syncRevision(UInt64 revision) override { delegate->syncRevision(revision); }
|
||||||
|
|
||||||
UInt64 getRevision() const override { return delegate->getRevision(); }
|
UInt64 getRevision() const override { return delegate->getRevision(); }
|
||||||
|
@ -596,7 +596,7 @@ catch (...)
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
DiskObjectStoragePtr DiskLocal::createDiskObjectStorage(const String & name_)
|
DiskObjectStoragePtr DiskLocal::createDiskObjectStorage()
|
||||||
{
|
{
|
||||||
auto object_storage = std::make_shared<LocalObjectStorage>();
|
auto object_storage = std::make_shared<LocalObjectStorage>();
|
||||||
auto metadata_storage = std::make_shared<FakeMetadataStorageFromDisk>(
|
auto metadata_storage = std::make_shared<FakeMetadataStorageFromDisk>(
|
||||||
@ -605,9 +605,9 @@ DiskObjectStoragePtr DiskLocal::createDiskObjectStorage(const String & name_)
|
|||||||
/* object_storage_root_path */getPath());
|
/* object_storage_root_path */getPath());
|
||||||
|
|
||||||
return std::make_shared<DiskObjectStorage>(
|
return std::make_shared<DiskObjectStorage>(
|
||||||
name_,
|
getName(),
|
||||||
disk_path,
|
disk_path,
|
||||||
"DiskLocalObjectStorage",
|
"Local",
|
||||||
metadata_storage,
|
metadata_storage,
|
||||||
object_storage,
|
object_storage,
|
||||||
DiskType::Local,
|
DiskType::Local,
|
||||||
|
@ -122,7 +122,7 @@ public:
|
|||||||
bool canRead() const noexcept;
|
bool canRead() const noexcept;
|
||||||
bool canWrite() const noexcept;
|
bool canWrite() const noexcept;
|
||||||
|
|
||||||
DiskObjectStoragePtr createDiskObjectStorage(const String & name_) override;
|
DiskObjectStoragePtr createDiskObjectStorage() override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::optional<UInt64> tryReserve(UInt64 bytes);
|
std::optional<UInt64> tryReserve(UInt64 bytes);
|
||||||
|
@ -222,16 +222,11 @@ public:
|
|||||||
|
|
||||||
virtual bool supportsCache() const { return false; }
|
virtual bool supportsCache() const { return false; }
|
||||||
|
|
||||||
virtual const std::unordered_set<String> & getCacheLayersNames() const
|
virtual NameSet getCacheLayersNames() const
|
||||||
{
|
{
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getCacheLayersNames()` is not implemented for disk: {}", getType());
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getCacheLayersNames()` is not implemented for disk: {}", getType());
|
||||||
}
|
}
|
||||||
|
|
||||||
virtual DiskPtr getWrappedDisk() const
|
|
||||||
{
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method `getWrappedDisk()` is not implemented for disk: {}", getType());
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns a list of storage objects (contains path, size, ...).
|
/// Returns a list of storage objects (contains path, size, ...).
|
||||||
/// (A list is returned because for Log family engines there might
|
/// (A list is returned because for Log family engines there might
|
||||||
/// be multiple files in remote fs for single clickhouse file.
|
/// be multiple files in remote fs for single clickhouse file.
|
||||||
@ -351,7 +346,7 @@ public:
|
|||||||
/// Return current disk revision.
|
/// Return current disk revision.
|
||||||
virtual UInt64 getRevision() const { return 0; }
|
virtual UInt64 getRevision() const { return 0; }
|
||||||
|
|
||||||
virtual DiskObjectStoragePtr createDiskObjectStorage(const String &)
|
virtual DiskObjectStoragePtr createDiskObjectStorage()
|
||||||
{
|
{
|
||||||
throw Exception(
|
throw Exception(
|
||||||
ErrorCodes::NOT_IMPLEMENTED,
|
ErrorCodes::NOT_IMPLEMENTED,
|
||||||
|
@ -21,10 +21,14 @@ namespace ErrorCodes
|
|||||||
extern const int CANNOT_STAT;
|
extern const int CANNOT_STAT;
|
||||||
}
|
}
|
||||||
|
|
||||||
CachedObjectStorage:: CachedObjectStorage(ObjectStoragePtr object_storage_, FileCachePtr cache_)
|
CachedObjectStorage:: CachedObjectStorage(
|
||||||
|
ObjectStoragePtr object_storage_,
|
||||||
|
FileCachePtr cache_,
|
||||||
|
const std::string & cache_config_name_)
|
||||||
: object_storage(object_storage_)
|
: object_storage(object_storage_)
|
||||||
, cache(cache_)
|
, cache(cache_)
|
||||||
, log(&Poco::Logger::get(getName() + "(" + object_storage_->getName() +")"))
|
, cache_config_name(cache_config_name_)
|
||||||
|
, log(&Poco::Logger::get(getName()))
|
||||||
{
|
{
|
||||||
cache->initialize();
|
cache->initialize();
|
||||||
}
|
}
|
||||||
|
@ -14,7 +14,7 @@ namespace DB
|
|||||||
class CachedObjectStorage : public IObjectStorage
|
class CachedObjectStorage : public IObjectStorage
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
CachedObjectStorage(ObjectStoragePtr object_storage_, FileCachePtr cache_);
|
CachedObjectStorage(ObjectStoragePtr object_storage_, FileCachePtr cache_, const String & cache_config_name_);
|
||||||
|
|
||||||
std::string getName() const override { return "CachedObjectStorage(" + object_storage->getName() + ")"; }
|
std::string getName() const override { return "CachedObjectStorage(" + object_storage->getName() + ")"; }
|
||||||
|
|
||||||
@ -95,6 +95,10 @@ public:
|
|||||||
|
|
||||||
bool isReadOnly() const override { return object_storage->isReadOnly(); }
|
bool isReadOnly() const override { return object_storage->isReadOnly(); }
|
||||||
|
|
||||||
|
const std::string & getCacheConfigName() const { return cache_config_name; }
|
||||||
|
|
||||||
|
ObjectStoragePtr getWrappedObjectStorage() { return object_storage; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
IFileCache::Key getCacheKey(const std::string & path) const;
|
IFileCache::Key getCacheKey(const std::string & path) const;
|
||||||
String getCachePath(const std::string & path) const;
|
String getCachePath(const std::string & path) const;
|
||||||
@ -102,6 +106,7 @@ private:
|
|||||||
|
|
||||||
ObjectStoragePtr object_storage;
|
ObjectStoragePtr object_storage;
|
||||||
FileCachePtr cache;
|
FileCachePtr cache;
|
||||||
|
std::string cache_config_name;
|
||||||
Poco::Logger * log;
|
Poco::Logger * log;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -35,6 +35,7 @@ void registerDiskCache(DiskFactory & factory)
|
|||||||
"There is not disk with name `{}`, disk name should be initialized before cache disk",
|
"There is not disk with name `{}`, disk name should be initialized before cache disk",
|
||||||
disk_name);
|
disk_name);
|
||||||
}
|
}
|
||||||
|
/// TODO: Add a check that there underlying storage is unique among cached storages.
|
||||||
|
|
||||||
FileCacheSettings file_cache_settings;
|
FileCacheSettings file_cache_settings;
|
||||||
file_cache_settings.loadFromConfig(config, config_prefix);
|
file_cache_settings.loadFromConfig(config, config_prefix);
|
||||||
@ -44,9 +45,10 @@ void registerDiskCache(DiskFactory & factory)
|
|||||||
fs::create_directories(cache_base_path);
|
fs::create_directories(cache_base_path);
|
||||||
|
|
||||||
auto disk = disk_it->second;
|
auto disk = disk_it->second;
|
||||||
auto disk_object_storage = disk->createDiskObjectStorage(disk_name);
|
|
||||||
|
|
||||||
auto cache = FileCacheFactory::instance().getOrCreate(cache_base_path, file_cache_settings, name);
|
auto cache = FileCacheFactory::instance().getOrCreate(cache_base_path, file_cache_settings, name);
|
||||||
|
auto disk_object_storage = disk->createDiskObjectStorage();
|
||||||
|
|
||||||
disk_object_storage->wrapWithCache(cache, name);
|
disk_object_storage->wrapWithCache(cache, name);
|
||||||
|
|
||||||
LOG_TEST(
|
LOG_TEST(
|
||||||
|
@ -445,12 +445,12 @@ bool DiskObjectStorage::isReadOnly() const
|
|||||||
return object_storage->isReadOnly();
|
return object_storage->isReadOnly();
|
||||||
}
|
}
|
||||||
|
|
||||||
DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage(const String & name_)
|
DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage()
|
||||||
{
|
{
|
||||||
return std::make_shared<DiskObjectStorage>(
|
return std::make_shared<DiskObjectStorage>(
|
||||||
name_,
|
getName(),
|
||||||
object_storage_root_path,
|
object_storage_root_path,
|
||||||
name,
|
getName(),
|
||||||
metadata_storage,
|
metadata_storage,
|
||||||
object_storage,
|
object_storage,
|
||||||
disk_type,
|
disk_type,
|
||||||
@ -460,8 +460,21 @@ DiskObjectStoragePtr DiskObjectStorage::createDiskObjectStorage(const String & n
|
|||||||
|
|
||||||
void DiskObjectStorage::wrapWithCache(FileCachePtr cache, const String & layer_name)
|
void DiskObjectStorage::wrapWithCache(FileCachePtr cache, const String & layer_name)
|
||||||
{
|
{
|
||||||
object_storage = std::make_shared<CachedObjectStorage>(object_storage, cache);
|
object_storage = std::make_shared<CachedObjectStorage>(object_storage, cache, layer_name);
|
||||||
cache_layers.insert(layer_name);
|
}
|
||||||
|
|
||||||
|
NameSet DiskObjectStorage::getCacheLayersNames() const
|
||||||
|
{
|
||||||
|
NameSet cache_layers;
|
||||||
|
auto current_object_storage = object_storage;
|
||||||
|
while (current_object_storage->supportsCache())
|
||||||
|
{
|
||||||
|
auto * cached_object_storage = assert_cast<CachedObjectStorage *>(current_object_storage.get());
|
||||||
|
cache_layers.insert(cached_object_storage->getCacheConfigName());
|
||||||
|
std::cerr << "\n\n" << "having cache layer: " << cached_object_storage->getCacheConfigName() << " for " << object_storage->getName() << "\n\n";
|
||||||
|
current_object_storage = cached_object_storage->getWrappedObjectStorage();
|
||||||
|
}
|
||||||
|
return cache_layers;
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
|
@ -164,7 +164,7 @@ public:
|
|||||||
|
|
||||||
UInt64 getRevision() const override;
|
UInt64 getRevision() const override;
|
||||||
|
|
||||||
DiskObjectStoragePtr createDiskObjectStorage(const String & name_) override;
|
DiskObjectStoragePtr createDiskObjectStorage() override;
|
||||||
|
|
||||||
bool supportsCache() const override;
|
bool supportsCache() const override;
|
||||||
|
|
||||||
@ -186,7 +186,7 @@ public:
|
|||||||
String getStructure() const { return "DiskObjectStorage(" + object_storage->getName() + ")"; }
|
String getStructure() const { return "DiskObjectStorage(" + object_storage->getName() + ")"; }
|
||||||
|
|
||||||
/// Get names of all cache layers. Name is how cache is defined in configuration file.
|
/// Get names of all cache layers. Name is how cache is defined in configuration file.
|
||||||
const std::unordered_set<String> & getCacheLayersNames() const override { return cache_layers; }
|
NameSet getCacheLayersNames() const override;
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
|
||||||
@ -212,8 +212,6 @@ private:
|
|||||||
size_t threadpool_size;
|
size_t threadpool_size;
|
||||||
|
|
||||||
std::unique_ptr<DiskObjectStorageRemoteMetadataRestoreHelper> metadata_helper;
|
std::unique_ptr<DiskObjectStorageRemoteMetadataRestoreHelper> metadata_helper;
|
||||||
|
|
||||||
std::unordered_set<String> cache_layers;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
using DiskObjectStoragePtr = std::shared_ptr<DiskObjectStorage>;
|
using DiskObjectStoragePtr = std::shared_ptr<DiskObjectStorage>;
|
||||||
|
@ -30,6 +30,13 @@
|
|||||||
<access_key_id>clickhouse</access_key_id>
|
<access_key_id>clickhouse</access_key_id>
|
||||||
<secret_access_key>clickhouse</secret_access_key>
|
<secret_access_key>clickhouse</secret_access_key>
|
||||||
</s3_disk_4>
|
</s3_disk_4>
|
||||||
|
<s3_disk_5>
|
||||||
|
<type>s3</type>
|
||||||
|
<path>s3_disk_5/</path>
|
||||||
|
<endpoint>http://localhost:11111/test/00170_test/</endpoint>
|
||||||
|
<access_key_id>clickhouse</access_key_id>
|
||||||
|
<secret_access_key>clickhouse</secret_access_key>
|
||||||
|
</s3_disk_5>
|
||||||
<!-- cache for s3 disks -->
|
<!-- cache for s3 disks -->
|
||||||
<s3_cache>
|
<s3_cache>
|
||||||
<type>cache</type>
|
<type>cache</type>
|
||||||
@ -61,6 +68,12 @@
|
|||||||
<cache_on_write_operations>1</cache_on_write_operations>
|
<cache_on_write_operations>1</cache_on_write_operations>
|
||||||
<enable_filesystem_query_cache_limit>1</enable_filesystem_query_cache_limit>
|
<enable_filesystem_query_cache_limit>1</enable_filesystem_query_cache_limit>
|
||||||
</s3_cache_4>
|
</s3_cache_4>
|
||||||
|
<s3_cache_5>
|
||||||
|
<type>cache</type>
|
||||||
|
<disk>s3_disk_5</disk>
|
||||||
|
<path>s3_cache_5/</path>
|
||||||
|
<max_size>22548578304</max_size>
|
||||||
|
</s3_cache_5>
|
||||||
<!-- local disks -->
|
<!-- local disks -->
|
||||||
<local_disk>
|
<local_disk>
|
||||||
<type>local</type>
|
<type>local</type>
|
||||||
@ -96,6 +109,19 @@
|
|||||||
<cache_on_write_operations>1</cache_on_write_operations>
|
<cache_on_write_operations>1</cache_on_write_operations>
|
||||||
<enable_cache_hits_threshold>1</enable_cache_hits_threshold>
|
<enable_cache_hits_threshold>1</enable_cache_hits_threshold>
|
||||||
</local_cache_3>
|
</local_cache_3>
|
||||||
|
<!-- multi layer cache -->
|
||||||
|
<s3_cache_multi>
|
||||||
|
<type>cache</type>
|
||||||
|
<disk>s3_cache_5</disk>
|
||||||
|
<path>s3_cache_multi/</path>
|
||||||
|
<max_size>22548578304</max_size>
|
||||||
|
</s3_cache_multi>
|
||||||
|
<s3_cache_multi_2>
|
||||||
|
<type>cache</type>
|
||||||
|
<disk>s3_cache_multi</disk>
|
||||||
|
<path>s3_cache_multi_2/</path>
|
||||||
|
<max_size>22548578304</max_size>
|
||||||
|
</s3_cache_multi_2>
|
||||||
</disks>
|
</disks>
|
||||||
<policies>
|
<policies>
|
||||||
<s3_cache>
|
<s3_cache>
|
||||||
@ -126,6 +152,13 @@
|
|||||||
</main>
|
</main>
|
||||||
</volumes>
|
</volumes>
|
||||||
</s3_cache_4>
|
</s3_cache_4>
|
||||||
|
<s3_cache_multi>
|
||||||
|
<volumes>
|
||||||
|
<main>
|
||||||
|
<disk>s3_cache_multi_2</disk>
|
||||||
|
</main>
|
||||||
|
</volumes>
|
||||||
|
</s3_cache_multi>
|
||||||
<local_cache>
|
<local_cache>
|
||||||
<volumes>
|
<volumes>
|
||||||
<main>
|
<main>
|
||||||
|
@ -2,3 +2,5 @@ Using storage policy: s3_cache
|
|||||||
|
|
||||||
Using storage policy: local_cache
|
Using storage policy: local_cache
|
||||||
|
|
||||||
|
Using storage policy: s3_cache_multi
|
||||||
|
|
||||||
|
@ -11,7 +11,7 @@ TMP_PATH=${CLICKHOUSE_TEST_UNIQUE_NAME}
|
|||||||
QUERIES_FILE=02313_filesystem_cache_seeks.queries
|
QUERIES_FILE=02313_filesystem_cache_seeks.queries
|
||||||
TEST_FILE=$CUR_DIR/filesystem_cache_queries/$QUERIES_FILE
|
TEST_FILE=$CUR_DIR/filesystem_cache_queries/$QUERIES_FILE
|
||||||
|
|
||||||
for storagePolicy in 's3_cache' 'local_cache'; do
|
for storagePolicy in 's3_cache' 'local_cache' 's3_cache_multi'; do
|
||||||
echo "Using storage policy: $storagePolicy"
|
echo "Using storage policy: $storagePolicy"
|
||||||
cat $TEST_FILE | sed -e "s/_storagePolicy/${storagePolicy}/" > $TMP_PATH
|
cat $TEST_FILE | sed -e "s/_storagePolicy/${storagePolicy}/" > $TMP_PATH
|
||||||
${CLICKHOUSE_CLIENT} --queries-file $TMP_PATH
|
${CLICKHOUSE_CLIENT} --queries-file $TMP_PATH
|
||||||
|
Loading…
Reference in New Issue
Block a user