thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

build cache_log table

Browse Source
This commit is contained in:
KinderRiven 2022-04-24 19:46:49 +08:00
parent 463b42b4c3
commit 9c9870f021
5 changed files with 37 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/Common/FileCache.cpp
View File

@ -48,8 +48,10 @@ void IFileCache::addQueryRef(String & query_id)
{
std::lock_guard cache_lock(logs_mutex);
auto iter = cache_logs.find(query_id);
if (iter == cache_logs.end())
if (iter == cache_logs.end()) {
iter = cache_logs.insert({query_id, std::make_shared<CacheLogRecorder>()}).first;
iter->second->logs = std::make_shared<CacheFileTrace>();
}
iter->second->ref++;
}
}
@ -68,11 +70,16 @@ void IFileCache::DecQueryRef(String & query_id)
{
if (auto cache_log = Context::getGlobalContextInstance()->getCacheLog())
{
CacheLogElement cache_elem;
cache_elem.query_id = query_id;
cache_elem.hit_count = iter->second->cache_hit_count;
cache_elem.miss_count = iter->second->cache_miss_count;
cache_log->add(cache_elem);
/// [remote_file_name, [hit_count, miss_count]]
for (const auto & access : *(iter->second->logs))
{
CacheLogElement cache_elem;
cache_elem.query_id = iter->first;
cache_elem.remote_file_path = access.first;
cache_elem.hit_count = access.second.first;
cache_elem.miss_count = access.second.second;
cache_log->add(cache_elem);
}
}
cache_logs.erase(iter);
}
@ -80,7 +87,7 @@ void IFileCache::DecQueryRef(String & query_id)
}
}
void IFileCache::updateQueryCacheLog(String & query_id, size_t hit_count, size_t miss_count)
void IFileCache::updateQueryCacheLog(String & query_id, String &remote_fs_path, size_t hit_count, size_t miss_count)
{
/// must be a query log
if (query_id.size())
@ -89,8 +96,9 @@ void IFileCache::updateQueryCacheLog(String & query_id, size_t hit_count, size_t
auto iter = cache_logs.find(query_id);
if (iter != cache_logs.end())
{
iter->second->cache_hit_count += hit_count;
iter->second->cache_miss_count += miss_count;
/// [query_id, [remote_fs_path, [hit_count, miss_count]]]
(*iter->second->logs)[remote_fs_path].first += hit_count;
(*iter->second->logs)[remote_fs_path].second += miss_count;
}
}
}

2
src/Common/FileCache.h
View File

@ -96,7 +96,7 @@ public:
void DecQueryRef(String & query_id);
void updateQueryCacheLog(String &query_id, size_t hit_count, size_t miss_count);
void updateQueryCacheLog(String &query_id, String &remote_fs_path, size_t hit_count, size_t miss_count);
protected:
String cache_base_path;

2
src/Disks/IO/CachedReadBufferFromRemoteFS.cpp
View File

@ -48,7 +48,7 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS(
CachedReadBufferFromRemoteFS::~CachedReadBufferFromRemoteFS()
{
cache->updateQueryCacheLog(settings.current_query_id, cache_hit_count, cache_miss_count);
cache->updateQueryCacheLog(settings.current_query_id, remote_fs_object_path, cache_hit_count, cache_miss_count);
cache->DecQueryRef(settings.current_query_id);
}

19
src/Interpreters/CacheLog.cpp
View File

@ -14,9 +14,10 @@ NamesAndTypesList CacheLogElement::getNamesAndTypes()
{"event_date", std::make_shared<DataTypeDate>()},
{"event_time", std::make_shared<DataTypeDateTime>()},
{"query_id", std::make_shared<DataTypeString>()},
{"remote_file_path", std::make_shared<DataTypeString>()},
{"hit_count", std::make_shared<DataTypeUInt64>()},
{"miss_count", std::make_shared<DataTypeUInt64>()},
{"hit_ratio", std::make_shared<DataTypeFloat64>()},
/// {"hit_ratio", std::make_shared<DataTypeFloat64>()},
};
}
@ -40,22 +41,22 @@ void CacheLogElement::appendToBlock(MutableColumns & columns) const
size_t i = 0;
const auto current_time = std::chrono::system_clock::now();
auto event_time = std::chrono::system_clock::to_time_t(current_time);
/// auto event_time_microseconds = time_in_microseconds(current_time);
columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType());
columns[i++]->insert(event_time);
columns[i++]->insert(query_id);
columns[i++]->insert(remote_file_path);
columns[i++]->insert(hit_count);
columns[i++]->insert(miss_count);
if (hit_count + miss_count)
{
double hit_ratio = 1.0 * hit_count / (hit_count + miss_count);
columns[i++]->insert(hit_ratio);
}
else
columns[i++]->insert(0);
/// if (hit_count + miss_count)
/// {
/// double hit_ratio = 1.0 * hit_count / (hit_count + miss_count);
/// columns[i++]->insert(hit_ratio);
/// }
/// else
/// columns[i++]->insert(0);
}
};

19
src/Interpreters/CacheLog.h
View File

@ -3,37 +3,34 @@
#include <Core/NamesAndAliases.h>
#include <Core/NamesAndTypes.h>
#include <Core/Settings.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeTuple.h>
#include <Interpreters/SystemLog.h>
#include <Interpreters/TransactionVersionMetadata.h>
#include <base/logger_useful.h>
namespace DB
{
/// [remote_file_path, [hit_count, miss_count]]
using CacheFileTrace = std::unordered_map<String, std::pair<size_t, size_t>>;
struct CacheLogRecorder
{
public:
size_t ref = 0;
size_t cache_hit_count = 0;
size_t cache_miss_count = 0;
std::shared_ptr<CacheFileTrace> logs;
};
struct CacheLogElement
{
/// time_t event_time{};
/// Decimal64 event_time_microseconds{};
String query_id;
String remote_file_path;
UInt64 hit_count;
UInt64 miss_count;
/// Float64 hit_ratio;
/// Float64 miss_ratio;
/// TODO for next version
/// std::unordered_map<String, size_t> hit_records;
/// std::unordered_map<String, size_t> miss_records;
static std::string name() { return "CacheLog"; }
static NamesAndTypesList getNamesAndTypes();