thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 08:40:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Shard OpenedFileCache to avoid lock contention (#51341)

Browse Source 
* shard OpenedFileCache to avoid lock contention

* Update OpenedFileCache.h

* fix build

---------

Co-authored-by: Alexey Milovidov <milovidov@clickhouse.com>
This commit is contained in:
Nikita Taranov 2023-07-24 15:58:21 +02:00 committed by GitHub
parent 40652cf2de
commit c6e6fd7613
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 68 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/Common/ProfileEvents.cpp
View File

@ -45,6 +45,7 @@
M(MMappedFileCacheMisses, "Number of times a file has not been found in the MMap cache (for the 'mmap' read_method), so we had to mmap it again.") \
M(OpenedFileCacheHits, "Number of times a file has been found in the opened file cache, so we didn't have to open it again.") \
M(OpenedFileCacheMisses, "Number of times a file has been found in the opened file cache, so we had to open it again.") \
M(OpenedFileCacheMicroseconds, "Amount of time spent executing OpenedFileCache methods.") \
M(AIOWrite, "Number of writes with Linux or FreeBSD AIO interface") \
M(AIOWriteBytes, "Number of bytes written with Linux or FreeBSD AIO interface") \
M(AIORead, "Number of reads with Linux or FreeBSD AIO interface") \

33
src/IO/OpenedFileCache.h
View File

@ -4,14 +4,18 @@
#include <mutex>
#include <Core/Types.h>
#include <Common/ProfileEvents.h>
#include <IO/OpenedFile.h>
#include <Common/ElapsedTimeProfileEventIncrement.h>
#include <Common/ProfileEvents.h>
#include <city.h>
namespace ProfileEvents
{
extern const Event OpenedFileCacheHits;
extern const Event OpenedFileCacheMisses;
extern const Event OpenedFileCacheMicroseconds;
}
namespace DB
@ -26,7 +30,8 @@ namespace DB
*/
class OpenedFileCache
{
private:
class OpenedFileMap
{
using Key = std::pair<std::string /* path */, int /* flags */>;
using OpenedFileWeakPtr = std::weak_ptr<OpenedFile>;
@ -35,7 +40,7 @@ private:
Files files;
std::mutex mutex;
public:
public:
using OpenedFilePtr = std::shared_ptr<OpenedFile>;
OpenedFilePtr get(const std::string & path, int flags)
@ -78,6 +83,27 @@ public:
std::lock_guard lock(mutex);
files.erase(key);
}
};
static constexpr size_t buckets = 1024;
std::vector<OpenedFileMap> impls{buckets};
public:
using OpenedFilePtr = OpenedFileMap::OpenedFilePtr;
OpenedFilePtr get(const std::string & path, int flags)
{
ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::OpenedFileCacheMicroseconds);
const auto bucket = CityHash_v1_0_2::CityHash64(path.data(), path.length()) % buckets;
return impls[bucket].get(path, flags);
}
void remove(const std::string & path, int flags)
{
ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::OpenedFileCacheMicroseconds);
const auto bucket = CityHash_v1_0_2::CityHash64(path.data(), path.length()) % buckets;
impls[bucket].remove(path, flags);
}
static OpenedFileCache & instance()
{
@ -87,5 +113,4 @@ public:
};
using OpenedFileCachePtr = std::shared_ptr<OpenedFileCache>;
}