build cache_log table

src/Common/FileCache.cpp

@@ -48,8 +48,10 @@ void IFileCache::addQueryRef(String & query_id)
     {
         std::lock_guard cache_lock(logs_mutex);
         auto iter = cache_logs.find(query_id);
-        if (iter == cache_logs.end())
+        if (iter == cache_logs.end()) {
             iter = cache_logs.insert({query_id, std::make_shared<CacheLogRecorder>()}).first;
+            iter->second->logs = std::make_shared<CacheFileTrace>();
+        }
         iter->second->ref++;
     }
 }
@@ -68,11 +70,16 @@ void IFileCache::DecQueryRef(String & query_id)
             {
                 if (auto cache_log = Context::getGlobalContextInstance()->getCacheLog())
                 {
-                    CacheLogElement cache_elem;
+                    /// [remote_file_name, [hit_count, miss_count]]
-                    cache_elem.query_id = query_id;
+                    for (const auto & access : *(iter->second->logs))
-                    cache_elem.hit_count = iter->second->cache_hit_count;
+                    {
-                    cache_elem.miss_count = iter->second->cache_miss_count;
+                        CacheLogElement cache_elem;
-                    cache_log->add(cache_elem);
+                        cache_elem.query_id = iter->first;
+                        cache_elem.remote_file_path = access.first;
+                        cache_elem.hit_count = access.second.first;
+                        cache_elem.miss_count = access.second.second;
+                        cache_log->add(cache_elem);
+                    }
                 }
                 cache_logs.erase(iter);
             }
@@ -80,7 +87,7 @@ void IFileCache::DecQueryRef(String & query_id)
     }
 }
 
-void IFileCache::updateQueryCacheLog(String & query_id, size_t hit_count, size_t miss_count)
+void IFileCache::updateQueryCacheLog(String & query_id, String &remote_fs_path, size_t hit_count, size_t miss_count)
 {
     /// must be a query log
     if (query_id.size())
@@ -89,8 +96,9 @@ void IFileCache::updateQueryCacheLog(String & query_id, size_t hit_count, size_t
         auto iter = cache_logs.find(query_id);
         if (iter != cache_logs.end()) 
         {
-            iter->second->cache_hit_count += hit_count;
+            /// [query_id, [remote_fs_path, [hit_count, miss_count]]]
-            iter->second->cache_miss_count += miss_count;
+            (*iter->second->logs)[remote_fs_path].first += hit_count;
+            (*iter->second->logs)[remote_fs_path].second += miss_count;
         }
     }
 }

src/Common/FileCache.h

@@ -96,7 +96,7 @@ public:
 
     void DecQueryRef(String & query_id);
 
-    void updateQueryCacheLog(String &query_id, size_t hit_count, size_t miss_count);
+    void updateQueryCacheLog(String &query_id, String &remote_fs_path, size_t hit_count, size_t miss_count);
 
 protected:
     String cache_base_path;

src/Disks/IO/CachedReadBufferFromRemoteFS.cpp

@@ -48,7 +48,7 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS(
 
 CachedReadBufferFromRemoteFS::~CachedReadBufferFromRemoteFS()
 {
-    cache->updateQueryCacheLog(settings.current_query_id, cache_hit_count, cache_miss_count);
+    cache->updateQueryCacheLog(settings.current_query_id, remote_fs_object_path, cache_hit_count, cache_miss_count);
     cache->DecQueryRef(settings.current_query_id);
 }
 

src/Interpreters/CacheLog.cpp

@@ -14,9 +14,10 @@ NamesAndTypesList CacheLogElement::getNamesAndTypes()
         {"event_date", std::make_shared<DataTypeDate>()},
         {"event_time", std::make_shared<DataTypeDateTime>()},
         {"query_id", std::make_shared<DataTypeString>()},
+        {"remote_file_path", std::make_shared<DataTypeString>()},
         {"hit_count", std::make_shared<DataTypeUInt64>()},
         {"miss_count", std::make_shared<DataTypeUInt64>()},
-        {"hit_ratio", std::make_shared<DataTypeFloat64>()},
+        /// {"hit_ratio", std::make_shared<DataTypeFloat64>()},
     };
 }
 
@@ -40,22 +41,22 @@ void CacheLogElement::appendToBlock(MutableColumns & columns) const
     size_t i = 0;
     const auto current_time = std::chrono::system_clock::now();
     auto event_time = std::chrono::system_clock::to_time_t(current_time);
-    /// auto event_time_microseconds = time_in_microseconds(current_time);
 
     columns[i++]->insert(DateLUT::instance().toDayNum(event_time).toUnderType());
     columns[i++]->insert(event_time);
 
     columns[i++]->insert(query_id);
+    columns[i++]->insert(remote_file_path);
     columns[i++]->insert(hit_count);
     columns[i++]->insert(miss_count);
 
-    if (hit_count + miss_count)
+    /// if (hit_count + miss_count)
-    {
+    /// {
-        double hit_ratio = 1.0 * hit_count / (hit_count + miss_count);
+    ///     double hit_ratio = 1.0 * hit_count / (hit_count + miss_count);
-        columns[i++]->insert(hit_ratio);
+    ///     columns[i++]->insert(hit_ratio);
-    }
+    /// }
-    else
+    /// else
-        columns[i++]->insert(0);
+    ///     columns[i++]->insert(0);
 }
 
 };

src/Interpreters/CacheLog.h

@@ -3,37 +3,34 @@
 #include <Core/NamesAndAliases.h>
 #include <Core/NamesAndTypes.h>
 #include <Core/Settings.h>
+#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeTuple.h>
 #include <Interpreters/SystemLog.h>
 #include <Interpreters/TransactionVersionMetadata.h>
+#include <base/logger_useful.h>
 
 namespace DB
 {
 
+/// [remote_file_path, [hit_count, miss_count]]
+using CacheFileTrace = std::unordered_map<String, std::pair<size_t, size_t>>;
+
 struct CacheLogRecorder
 {
 public:
     size_t ref = 0;
-    size_t cache_hit_count = 0;
+    std::shared_ptr<CacheFileTrace> logs;
-    size_t cache_miss_count = 0;
 };
 
 struct CacheLogElement
 {
     /// time_t event_time{};
-    /// Decimal64 event_time_microseconds{};
-
     String query_id;
+    String remote_file_path;
 
     UInt64 hit_count;
     UInt64 miss_count;
 
-    /// Float64 hit_ratio;
-    /// Float64 miss_ratio;
-
-    /// TODO for next version
-    /// std::unordered_map<String, size_t> hit_records;
-    /// std::unordered_map<String, size_t> miss_records;
-
     static std::string name() { return "CacheLog"; }
 
     static NamesAndTypesList getNamesAndTypes();