ClickHouse/src/Storages/StorageTinyLog.cpp

#include <sys/stat.h>
#include <sys/types.h>
#include <errno.h>

#include <map>
#include <cassert>

#include <Poco/Util/XMLConfiguration.h>

#include <Common/escapeForFileName.h>
#include <Common/Exception.h>
#include <Common/typeid_cast.h>

#include <IO/ReadBufferFromFileBase.h>
#include <IO/WriteBufferFromFileBase.h>
#include <IO/LimitReadBuffer.h>
#include <Compression/CompressionFactory.h>
#include <Compression/CompressedReadBuffer.h>
#include <Compression/CompressedWriteBuffer.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>

#include <DataTypes/NestedUtils.h>

#include <DataStreams/IBlockInputStream.h>
#include <DataStreams/IBlockOutputStream.h>

#include <Columns/ColumnArray.h>

#include <Interpreters/Context.h>

#include <Interpreters/evaluateConstantExpression.h>
#include <Parsers/ASTLiteral.h>
#include <Storages/CheckResults.h>
#include <Storages/StorageFactory.h>
#include <Storages/StorageTinyLog.h>
#include "StorageLogSettings.h"

#include <Processors/Sources/SourceWithProgress.h>
#include <Processors/Pipe.h>

#define DBMS_STORAGE_LOG_DATA_FILE_EXTENSION ".bin"


namespace DB
{

namespace ErrorCodes
{
    extern const int TIMEOUT_EXCEEDED;
    extern const int DUPLICATE_COLUMN;
    extern const int INCORRECT_FILE_NAME;
    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}


class TinyLogSource final : public SourceWithProgress
{
public:
    static Block getHeader(const NamesAndTypesList & columns)
    {
        Block res;

        for (const auto & name_type : columns)
            res.insert({ name_type.type->createColumn(), name_type.type, name_type.name });

        return res;
    }

    TinyLogSource(
        size_t block_size_,
        const NamesAndTypesList & columns_,
        StorageTinyLog & storage_,
        size_t max_read_buffer_size_,
        FileChecker::Map file_sizes_)
        : SourceWithProgress(getHeader(columns_))
        , block_size(block_size_), columns(columns_), storage(storage_)
        , max_read_buffer_size(max_read_buffer_size_), file_sizes(std::move(file_sizes_))
    {
    }

    String getName() const override { return "TinyLog"; }

protected:
    Chunk generate() override;

private:
    size_t block_size;
    NamesAndTypesList columns;
    StorageTinyLog & storage;
    bool is_finished = false;
    size_t max_read_buffer_size;
    FileChecker::Map file_sizes;

    struct Stream
    {
        Stream(const DiskPtr & disk, const String & data_path, size_t max_read_buffer_size_, size_t file_size)
            : plain(file_size ? disk->readFile(data_path, std::min(max_read_buffer_size_, file_size)) : std::make_unique<ReadBuffer>(nullptr, 0)),
            limited(std::make_unique<LimitReadBuffer>(*plain, file_size, false)),
            compressed(*limited)
        {
        }

        std::unique_ptr<ReadBuffer> plain;
        std::unique_ptr<ReadBuffer> limited;
        CompressedReadBuffer compressed;
    };

    using FileStreams = std::map<String, std::unique_ptr<Stream>>;
    FileStreams streams;

    using DeserializeState = ISerialization::DeserializeBinaryBulkStatePtr;
    using DeserializeStates = std::map<String, DeserializeState>;
    DeserializeStates deserialize_states;

    void readData(const NameAndTypePair & name_and_type, ColumnPtr & column, UInt64 limit, ISerialization::SubstreamsCache & cache);
};


Chunk TinyLogSource::generate()
{
    Block res;

    if (is_finished || file_sizes.empty() || (!streams.empty() && streams.begin()->second->compressed.eof()))
    {
        /** Close the files (before destroying the object).
          * When many sources are created, but simultaneously reading only a few of them,
          * buffers don't waste memory.
          */
        is_finished = true;
        streams.clear();
        return {};
    }

    std::unordered_map<String, ISerialization::SubstreamsCache> caches;
    for (const auto & name_type : columns)
    {
        ColumnPtr column;
        try
        {
            column = name_type.type->createColumn();
            readData(name_type, column, block_size, caches[name_type.getNameInStorage()]);
        }
        catch (Exception & e)
        {
            e.addMessage("while reading column " + name_type.name + " at " + fullPath(storage.disk, storage.table_path));
            throw;
        }

        if (!column->empty())
            res.insert(ColumnWithTypeAndName(std::move(column), name_type.type, name_type.name));
    }

    if (!res || streams.begin()->second->compressed.eof())
    {
        is_finished = true;
        streams.clear();
    }

    return Chunk(res.getColumns(), res.rows());
}


void TinyLogSource::readData(const NameAndTypePair & name_and_type,
    ColumnPtr & column, UInt64 limit, ISerialization::SubstreamsCache & cache)
{
    ISerialization::DeserializeBinaryBulkSettings settings; /// TODO Use avg_value_size_hint.
    const auto & [name, type] = name_and_type;
    auto serialization = IDataType::getSerialization(name_and_type);

    settings.getter = [&] (const ISerialization::SubstreamPath & path) -> ReadBuffer *
    {
        if (cache.count(ISerialization::getSubcolumnNameForStream(path)))
            return nullptr;

        String stream_name = ISerialization::getFileNameForStream(name_and_type, path);
        auto & stream = streams[stream_name];
        if (!stream)
        {
            String file_path = storage.files[stream_name].data_file_path;
            stream = std::make_unique<Stream>(
                storage.disk, file_path, max_read_buffer_size, file_sizes[fileName(file_path)]);
        }

        return &stream->compressed;
    };

    if (deserialize_states.count(name) == 0)
         serialization->deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]);

    serialization->deserializeBinaryBulkWithMultipleStreams(column, limit, settings, deserialize_states[name], &cache);
}


class TinyLogBlockOutputStream final : public IBlockOutputStream
{
public:
    explicit TinyLogBlockOutputStream(
        StorageTinyLog & storage_,
        const StorageMetadataPtr & metadata_snapshot_,
        std::unique_lock<std::shared_timed_mutex> && lock_)
        : storage(storage_), metadata_snapshot(metadata_snapshot_), lock(std::move(lock_))
    {
        if (!lock)
            throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED);

        /// If there were no files, add info to rollback in case of error.
        if (storage.file_checker.empty())
        {
            for (const auto & file : storage.files)
                storage.file_checker.setEmpty(file.second.data_file_path);
            storage.file_checker.save();
        }
    }

    ~TinyLogBlockOutputStream() override
    {
        try
        {
            if (!done)
            {
                /// Rollback partial writes.
                LOG_WARNING(storage.log, "Rollback partial writes");
                streams.clear();
                storage.file_checker.repair();
            }
        }
        catch (...)
        {
            tryLogCurrentException(__PRETTY_FUNCTION__);
        }
    }

    Block getHeader() const override { return metadata_snapshot->getSampleBlock(); }

    void write(const Block & block) override;
    void writeSuffix() override;

private:
    StorageTinyLog & storage;
    StorageMetadataPtr metadata_snapshot;
    std::unique_lock<std::shared_timed_mutex> lock;
    bool done = false;

    struct Stream
    {
        Stream(const DiskPtr & disk, const String & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) :
            plain(disk->writeFile(data_path, max_compress_block_size, WriteMode::Append)),
            compressed(*plain, std::move(codec), max_compress_block_size)
        {
        }

        std::unique_ptr<WriteBuffer> plain;
        CompressedWriteBuffer compressed;

        void finalize()
        {
            compressed.next();
            plain->finalize();
        }
    };

    using FileStreams = std::map<String, std::unique_ptr<Stream>>;
    FileStreams streams;

    using SerializeState = ISerialization::SerializeBinaryBulkStatePtr;
    using SerializeStates = std::map<String, SerializeState>;
    SerializeStates serialize_states;

    using WrittenStreams = std::set<String>;

    ISerialization::OutputStreamGetter createStreamGetter(const NameAndTypePair & column, WrittenStreams & written_streams);
    void writeData(const NameAndTypePair & name_and_type, const IColumn & column, WrittenStreams & written_streams);
};


ISerialization::OutputStreamGetter TinyLogBlockOutputStream::createStreamGetter(
    const NameAndTypePair & column,
    WrittenStreams & written_streams)
{
    return [&] (const ISerialization::SubstreamPath & path) -> WriteBuffer *
    {
        String stream_name = ISerialization::getFileNameForStream(column, path);

        if (!written_streams.insert(stream_name).second)
            return nullptr;

        const auto & columns = metadata_snapshot->getColumns();
        if (!streams.count(stream_name))
            streams[stream_name] = std::make_unique<Stream>(
                storage.disk,
                storage.files[stream_name].data_file_path,
                columns.getCodecOrDefault(column.name),
                storage.max_compress_block_size);

        return &streams[stream_name]->compressed;
    };
}


void TinyLogBlockOutputStream::writeData(const NameAndTypePair & name_and_type, const IColumn & column, WrittenStreams & written_streams)
{
    ISerialization::SerializeBinaryBulkSettings settings;
    const auto & [name, type] = name_and_type;
    auto serialization = type->getDefaultSerialization();

    if (serialize_states.count(name) == 0)
    {
        /// Some stream getters may be called form `serializeBinaryBulkStatePrefix`.
        /// Use different WrittenStreams set, or we get nullptr for them in `serializeBinaryBulkWithMultipleStreams`
        WrittenStreams prefix_written_streams;
        settings.getter = createStreamGetter(name_and_type, prefix_written_streams);
        serialization->serializeBinaryBulkStatePrefix(settings, serialize_states[name]);
    }

    settings.getter = createStreamGetter(name_and_type, written_streams);
    serialization->serializeBinaryBulkWithMultipleStreams(column, 0, 0, settings, serialize_states[name]);
}


void TinyLogBlockOutputStream::writeSuffix()
{
    if (done)
        return;

    /// If nothing was written - leave the table in initial state.
    if (streams.empty())
    {
        done = true;
        return;
    }

    WrittenStreams written_streams;
    ISerialization::SerializeBinaryBulkSettings settings;
    for (const auto & column : getHeader())
    {
        auto it = serialize_states.find(column.name);
        if (it != serialize_states.end())
        {
            settings.getter = createStreamGetter(NameAndTypePair(column.name, column.type), written_streams);
            auto serialization = column.type->getDefaultSerialization();
            serialization->serializeBinaryBulkStateSuffix(settings, it->second);
        }
    }

    /// Finish write.
    for (auto & stream : streams)
        stream.second->finalize();

    Strings column_files;
    for (auto & pair : streams)
        column_files.push_back(storage.files[pair.first].data_file_path);

    streams.clear();
    done = true;

    for (const auto & file : column_files)
        storage.file_checker.update(file);
    storage.file_checker.save();

    /// unlock should be done from the same thread as lock, and dtor may be
    /// called from different thread, so it should be done here (at least in
    /// case of no exceptions occurred)
    lock.unlock();
}


void TinyLogBlockOutputStream::write(const Block & block)
{
    metadata_snapshot->check(block, true);

    /// The set of written offset columns so that you do not write shared columns for nested structures multiple times
    WrittenStreams written_streams;

    for (size_t i = 0; i < block.columns(); ++i)
    {
        const ColumnWithTypeAndName & column = block.safeGetByPosition(i);
        writeData(NameAndTypePair(column.name, column.type), *column.column, written_streams);
    }
}


StorageTinyLog::StorageTinyLog(
    DiskPtr disk_,
    const String & relative_path_,
    const StorageID & table_id_,
    const ColumnsDescription & columns_,
    const ConstraintsDescription & constraints_,
    bool attach,
    size_t max_compress_block_size_)
    : IStorage(table_id_)
    , disk(std::move(disk_))
    , table_path(relative_path_)
    , max_compress_block_size(max_compress_block_size_)
    , file_checker(disk, table_path + "sizes.json")
    , log(&Poco::Logger::get("StorageTinyLog"))
{
    StorageInMemoryMetadata storage_metadata;
    storage_metadata.setColumns(columns_);
    storage_metadata.setConstraints(constraints_);
    setInMemoryMetadata(storage_metadata);

    if (relative_path_.empty())
        throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME);

    if (!attach)
    {
        /// create directories if they do not exist
        disk->createDirectories(table_path);
    }
    else
    {
        try
        {
            file_checker.repair();
        }
        catch (...)
        {
            tryLogCurrentException(__PRETTY_FUNCTION__);
        }
    }

    for (const auto & col : storage_metadata.getColumns().getAllPhysical())
        addFiles(col);
}


void StorageTinyLog::addFiles(const NameAndTypePair & column)
{
    const auto & [name, type] = column;
    if (files.end() != files.find(name))
        throw Exception("Duplicate column with name " + name + " in constructor of StorageTinyLog.",
            ErrorCodes::DUPLICATE_COLUMN);

    ISerialization::StreamCallback stream_callback = [&] (const ISerialization::SubstreamPath & substream_path)
    {
        String stream_name = ISerialization::getFileNameForStream(column, substream_path);
        if (!files.count(stream_name))
        {
            ColumnData column_data;
            files.insert(std::make_pair(stream_name, column_data));
            files[stream_name].data_file_path = table_path + stream_name + DBMS_STORAGE_LOG_DATA_FILE_EXTENSION;
        }
    };

    ISerialization::SubstreamPath substream_path;
    auto serialization = type->getDefaultSerialization();
    serialization->enumerateStreams(stream_callback, substream_path);
}


void StorageTinyLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id)
{
    assert(table_path != new_path_to_table_data);
    {
        disk->moveDirectory(table_path, new_path_to_table_data);

        table_path = new_path_to_table_data;
        file_checker.setPath(table_path + "sizes.json");

        for (auto & file : files)
            file.second.data_file_path = table_path + fileName(file.second.data_file_path);
    }
    renameInMemory(new_table_id);
}


static std::chrono::seconds getLockTimeout(ContextPtr context)
{
    const Settings & settings = context->getSettingsRef();
    Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds();
    if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout)
        lock_timeout = settings.max_execution_time.totalSeconds();
    return std::chrono::seconds{lock_timeout};
}


Pipe StorageTinyLog::read(
    const Names & column_names,
    const StorageMetadataPtr & metadata_snapshot,
    SelectQueryInfo & /*query_info*/,
    ContextPtr context,
    QueryProcessingStage::Enum /*processed_stage*/,
    const size_t max_block_size,
    const unsigned /*num_streams*/)
{
    metadata_snapshot->check(column_names, getVirtuals(), getStorageID());

    auto all_columns = metadata_snapshot->getColumns().getAllWithSubcolumns().addTypes(column_names);

    // When reading, we lock the entire storage, because we only have one file
    // per column and can't modify it concurrently.
    const Settings & settings = context->getSettingsRef();

    std::shared_lock lock{rwlock, getLockTimeout(context)};
    if (!lock)
        throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED);

    /// No need to hold lock while reading because we read fixed range of data that does not change while appending more data.
    return Pipe(std::make_shared<TinyLogSource>(
        max_block_size,
        Nested::convertToSubcolumns(all_columns),
        *this,
        settings.max_read_buffer_size,
        file_checker.getFileSizes()));
}


BlockOutputStreamPtr StorageTinyLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr context)
{
    return std::make_shared<TinyLogBlockOutputStream>(*this, metadata_snapshot, std::unique_lock{rwlock, getLockTimeout(context)});
}


CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, ContextPtr context)
{
    std::shared_lock lock(rwlock, getLockTimeout(context));
    if (!lock)
        throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED);

    return file_checker.check();
}

void StorageTinyLog::truncate(
    const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr, TableExclusiveLockHolder &)
{
    disk->clearDirectory(table_path);

    files.clear();
    file_checker = FileChecker{disk, table_path + "sizes.json"};

    for (const auto & column : metadata_snapshot->getColumns().getAllPhysical())
        addFiles(column);
}


void registerStorageTinyLog(StorageFactory & factory)
{
    StorageFactory::StorageFeatures features{
        .supports_settings = true
    };

    factory.registerStorage("TinyLog", [](const StorageFactory::Arguments & args)
    {
        if (!args.engine_args.empty())
            throw Exception(
                "Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)",
                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);

        String disk_name = getDiskName(*args.storage_def);
        DiskPtr disk = args.getContext()->getDisk(disk_name);

        return StorageTinyLog::create(
            disk, args.relative_data_path, args.table_id, args.columns, args.constraints,
            args.attach, args.getContext()->getSettings().max_compress_block_size);
    }, features);
}

}