ClickHouse/src/Functions/FrequencyHolder.h

#pragma once

#include <Common/TLDListsHolder.h>
#include <Common/StringUtils/StringUtils.h>
#include <IO/ReadBufferFromFile.h>
#include <string_view>
#include <string>
#include <fstream>
#include <algorithm>
#include <cstring>
#include <limits>
#include <unordered_map>


namespace DB
{

class FrequencyHolder
{
public:
    using Map = std::unordered_map<UInt16, double>;
    using Container = std::unordered_map<std::string, Map>;


    static FrequencyHolder & getInstance()
    {
        static FrequencyHolder instance;
        return instance;
    }


    void parseDictionaries(const std::string& pt)
    {
        is_true = pt;
        loadEmotionalDict("/home/sergey/ClickHouse/src/Functions/ClassificationDictionaries/emotional_dictionary_rus.txt");
        loadEncodingsFrequency("/home/sergey/ClickHouse/src/Functions/ClassificationDictionaries/charset_freq.txt");
    }


    void loadEncodingsFrequency(const std::string path_to_charset_freq)
    {
        char charset_name_buf [40];
        std::string charset_name;
        ReadBufferFromFile in(path_to_charset_freq);
        while (!in.eof())
        {
            char * newline = find_first_symbols<'\n'>(in.position(), in.buffer().end());
        
            if (newline >= in.buffer().end())
                break;

            std::string_view line(in.position(), newline - in.position());
            in.position() = newline + 1;

            if (line.empty())
                continue;
            // Start load new charset
            if (line.size() > 2 && line[0] == '/' && line[1] == '/')
            {
                const char * st = line.data();
                sscanf(st + 2, "%39s", charset_name_buf);
                std::string s(charset_name_buf);
                charset_name = s;
            } else
            {
                const char * st = line.data();
                UInt16 bigram;
                double frequency;
                sscanf(st, "%hd %lg", &bigram, &frequency);
                encodings_freq[charset_name][bigram] = frequency;

            }
        }
    }


    void loadEmotionalDict(const std::string path_to_emotional_dict)
    {
        std::ifstream file(path_to_emotional_dict);
        std::string term, tag;
        double val;
        while (file >> term >> tag >> val) {
            std::vector<double> cur = {val};
            emotional_dict[term] = cur;
        }
        file.close();
    }


    const std::string & get_path()
    {
        return is_true;
    }

    const std::unordered_map<std::string, std::vector<double>> getEmotionalDict()
    {
        return emotional_dict;
    }

    const Container getEncodingsFrequency()
    {
        return encodings_freq;
    }


protected:

    std::string is_true;
    std::unordered_map<std::string, std::vector<double>> emotional_dict;
    Container encodings_freq;
};
}