2017-04-01 09:19:00 +00:00
|
|
|
#include <Storages/StorageLog.h>
|
2017-12-30 00:36:06 +00:00
|
|
|
#include <Storages/StorageFactory.h>
|
2011-11-05 23:31:19 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Common/Exception.h>
|
2018-01-15 19:07:47 +00:00
|
|
|
#include <Common/StringUtils/StringUtils.h>
|
2019-12-23 16:57:16 +00:00
|
|
|
#include <Common/typeid_cast.h>
|
2010-03-18 19:32:14 +00:00
|
|
|
|
2020-02-14 14:28:33 +00:00
|
|
|
#include <Interpreters/evaluateConstantExpression.h>
|
|
|
|
|
|
|
|
#include <IO/ReadBufferFromFileBase.h>
|
2020-02-20 16:39:32 +00:00
|
|
|
#include <IO/WriteBufferFromFileBase.h>
|
2020-10-26 17:24:15 +00:00
|
|
|
#include <Compression/CompressedReadBuffer.h>
|
2018-12-28 18:15:26 +00:00
|
|
|
#include <Compression/CompressedWriteBuffer.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/ReadHelpers.h>
|
|
|
|
#include <IO/WriteHelpers.h>
|
2012-01-09 19:20:48 +00:00
|
|
|
|
2017-12-25 18:58:39 +00:00
|
|
|
#include <DataTypes/NestedUtils.h>
|
2012-08-29 20:07:24 +00:00
|
|
|
|
2019-01-23 14:48:50 +00:00
|
|
|
#include <DataStreams/IBlockInputStream.h>
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <DataStreams/IBlockOutputStream.h>
|
2015-01-18 08:25:56 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnArray.h>
|
2012-08-29 20:07:24 +00:00
|
|
|
|
2017-05-24 21:06:29 +00:00
|
|
|
#include <Interpreters/Context.h>
|
2020-02-14 14:28:33 +00:00
|
|
|
#include <Parsers/ASTLiteral.h>
|
2020-02-18 14:41:30 +00:00
|
|
|
#include "StorageLogSettings.h"
|
2020-01-31 15:10:10 +00:00
|
|
|
#include <Processors/Sources/SourceWithProgress.h>
|
|
|
|
#include <Processors/Pipe.h>
|
2017-01-21 04:24:28 +00:00
|
|
|
|
2020-09-18 19:25:56 +00:00
|
|
|
#include <cassert>
|
|
|
|
|
2010-03-18 19:32:14 +00:00
|
|
|
|
2017-12-30 00:36:06 +00:00
|
|
|
#define DBMS_STORAGE_LOG_DATA_FILE_EXTENSION ".bin"
|
|
|
|
#define DBMS_STORAGE_LOG_MARKS_FILE_NAME "__marks.mrk"
|
2012-01-09 19:20:48 +00:00
|
|
|
|
|
|
|
|
2010-03-18 19:32:14 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
|
|
|
|
2016-01-11 21:46:36 +00:00
|
|
|
namespace ErrorCodes
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
extern const int LOGICAL_ERROR;
|
|
|
|
extern const int DUPLICATE_COLUMN;
|
|
|
|
extern const int SIZES_OF_MARKS_FILES_ARE_INCONSISTENT;
|
2017-12-30 00:36:06 +00:00
|
|
|
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
|
2017-11-03 19:53:10 +00:00
|
|
|
extern const int INCORRECT_FILE_NAME;
|
2016-01-11 21:46:36 +00:00
|
|
|
}
|
|
|
|
|
2020-01-31 15:10:10 +00:00
|
|
|
class LogSource final : public SourceWithProgress
|
2010-03-18 19:32:14 +00:00
|
|
|
{
|
2015-01-18 08:25:56 +00:00
|
|
|
public:
|
2020-01-31 15:10:10 +00:00
|
|
|
|
|
|
|
static Block getHeader(const NamesAndTypesList & columns)
|
|
|
|
{
|
|
|
|
Block res;
|
|
|
|
|
|
|
|
for (const auto & name_type : columns)
|
|
|
|
res.insert({ name_type.type->createColumn(), name_type.type, name_type.name });
|
|
|
|
|
2020-11-10 17:32:00 +00:00
|
|
|
return res;
|
2020-01-31 15:10:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
LogSource(
|
2018-01-02 06:13:22 +00:00
|
|
|
size_t block_size_, const NamesAndTypesList & columns_, StorageLog & storage_,
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t mark_number_, size_t rows_limit_, size_t max_read_buffer_size_)
|
2020-01-31 15:10:10 +00:00
|
|
|
: SourceWithProgress(getHeader(columns_)),
|
|
|
|
block_size(block_size_),
|
2018-01-02 06:13:22 +00:00
|
|
|
columns(columns_),
|
2017-04-01 07:20:54 +00:00
|
|
|
storage(storage_),
|
|
|
|
mark_number(mark_number_),
|
|
|
|
rows_limit(rows_limit_),
|
|
|
|
max_read_buffer_size(max_read_buffer_size_)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
String getName() const override { return "Log"; }
|
|
|
|
|
2015-01-18 08:25:56 +00:00
|
|
|
protected:
|
2020-01-31 15:10:10 +00:00
|
|
|
Chunk generate() override;
|
2016-08-24 00:39:38 +00:00
|
|
|
|
2015-01-18 08:25:56 +00:00
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t block_size;
|
2018-01-02 06:13:22 +00:00
|
|
|
NamesAndTypesList columns;
|
2017-04-01 07:20:54 +00:00
|
|
|
StorageLog & storage;
|
|
|
|
size_t mark_number; /// from what mark to read data
|
|
|
|
size_t rows_limit; /// The maximum number of rows that can be read
|
|
|
|
size_t rows_read = 0;
|
|
|
|
size_t max_read_buffer_size;
|
|
|
|
|
|
|
|
struct Stream
|
|
|
|
{
|
2019-12-23 16:57:16 +00:00
|
|
|
Stream(const DiskPtr & disk, const String & data_path, size_t offset, size_t max_read_buffer_size_)
|
2020-10-26 17:24:15 +00:00
|
|
|
: plain(disk->readFile(data_path, std::min(max_read_buffer_size_, disk->getFileSize(data_path))))
|
|
|
|
, compressed(*plain)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
if (offset)
|
2020-10-29 14:14:23 +00:00
|
|
|
plain->seek(offset, SEEK_SET);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-10-26 17:24:15 +00:00
|
|
|
std::unique_ptr<ReadBufferFromFileBase> plain;
|
|
|
|
CompressedReadBuffer compressed;
|
2017-04-01 07:20:54 +00:00
|
|
|
};
|
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
using FileStreams = std::map<String, Stream>;
|
2017-04-01 07:20:54 +00:00
|
|
|
FileStreams streams;
|
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
using DeserializeState = IDataType::DeserializeBinaryBulkStatePtr;
|
|
|
|
using DeserializeStates = std::map<String, DeserializeState>;
|
|
|
|
DeserializeStates deserialize_states;
|
|
|
|
|
2020-11-10 17:32:00 +00:00
|
|
|
void readData(const NameAndTypePair & name_and_type, ColumnPtr & column, size_t max_rows_to_read, IDataType::SubstreamsCache & cache);
|
2015-01-18 08:25:56 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
2017-09-08 04:58:57 +00:00
|
|
|
class LogBlockOutputStream final : public IBlockOutputStream
|
2014-03-19 10:45:13 +00:00
|
|
|
{
|
2015-01-18 08:25:56 +00:00
|
|
|
public:
|
2020-06-16 15:51:29 +00:00
|
|
|
explicit LogBlockOutputStream(StorageLog & storage_, const StorageMetadataPtr & metadata_snapshot_)
|
|
|
|
: storage(storage_)
|
|
|
|
, metadata_snapshot(metadata_snapshot_)
|
|
|
|
, lock(storage.rwlock)
|
|
|
|
, marks_stream(
|
|
|
|
storage.disk->writeFile(storage.marks_file_path, 4096, WriteMode::Rewrite))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
~LogBlockOutputStream() override
|
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
2020-07-12 02:31:58 +00:00
|
|
|
if (!done)
|
|
|
|
{
|
|
|
|
/// Rollback partial writes.
|
|
|
|
streams.clear();
|
|
|
|
storage.file_checker.repair();
|
|
|
|
}
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
catch (...)
|
|
|
|
{
|
|
|
|
tryLogCurrentException(__PRETTY_FUNCTION__);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-06-16 15:51:29 +00:00
|
|
|
Block getHeader() const override { return metadata_snapshot->getSampleBlock(); }
|
2017-04-01 07:20:54 +00:00
|
|
|
void write(const Block & block) override;
|
|
|
|
void writeSuffix() override;
|
2015-04-02 23:58:26 +00:00
|
|
|
|
2015-01-18 08:25:56 +00:00
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
StorageLog & storage;
|
2020-06-16 15:51:29 +00:00
|
|
|
StorageMetadataPtr metadata_snapshot;
|
2017-07-28 17:34:02 +00:00
|
|
|
std::unique_lock<std::shared_mutex> lock;
|
2017-04-01 07:20:54 +00:00
|
|
|
bool done = false;
|
|
|
|
|
|
|
|
struct Stream
|
|
|
|
{
|
2019-12-23 16:57:16 +00:00
|
|
|
Stream(const DiskPtr & disk, const String & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) :
|
2020-01-22 16:17:25 +00:00
|
|
|
plain(disk->writeFile(data_path, max_compress_block_size, WriteMode::Append)),
|
|
|
|
compressed(*plain, std::move(codec), max_compress_block_size),
|
2019-12-23 16:57:16 +00:00
|
|
|
plain_offset(disk->getFileSize(data_path))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
|
2020-01-22 16:17:25 +00:00
|
|
|
std::unique_ptr<WriteBuffer> plain;
|
2017-04-01 07:20:54 +00:00
|
|
|
CompressedWriteBuffer compressed;
|
|
|
|
|
|
|
|
size_t plain_offset; /// How many bytes were in the file at the time the LogBlockOutputStream was created.
|
|
|
|
|
|
|
|
void finalize()
|
|
|
|
{
|
|
|
|
compressed.next();
|
2020-01-22 16:17:25 +00:00
|
|
|
plain->next();
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2017-11-26 19:22:33 +00:00
|
|
|
using Mark = StorageLog::Mark;
|
2017-08-30 18:13:32 +00:00
|
|
|
using MarksForColumns = std::vector<std::pair<size_t, Mark>>;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
using FileStreams = std::map<String, Stream>;
|
2017-04-01 07:20:54 +00:00
|
|
|
FileStreams streams;
|
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
using WrittenStreams = std::set<String>;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-01-22 16:17:25 +00:00
|
|
|
std::unique_ptr<WriteBuffer> marks_stream; /// Declared below `lock` to make the file open when rwlock is captured.
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
using SerializeState = IDataType::SerializeBinaryBulkStatePtr;
|
|
|
|
using SerializeStates = std::map<String, SerializeState>;
|
|
|
|
SerializeStates serialize_states;
|
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
IDataType::OutputStreamGetter createStreamGetter(const NameAndTypePair & name_and_type, WrittenStreams & written_streams);
|
2018-06-07 18:14:37 +00:00
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
void writeData(const NameAndTypePair & name_and_type, const IColumn & column,
|
2017-08-07 07:31:16 +00:00
|
|
|
MarksForColumns & out_marks,
|
|
|
|
WrittenStreams & written_streams);
|
|
|
|
|
2017-11-27 21:21:09 +00:00
|
|
|
void writeMarks(MarksForColumns && marks);
|
2015-01-18 08:25:56 +00:00
|
|
|
};
|
2014-03-19 10:45:13 +00:00
|
|
|
|
|
|
|
|
2020-01-31 15:10:10 +00:00
|
|
|
Chunk LogSource::generate()
|
2010-03-18 19:32:14 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
Block res;
|
|
|
|
|
|
|
|
if (rows_read == rows_limit)
|
2020-01-31 15:10:10 +00:00
|
|
|
return {};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
if (storage.disk->isDirectoryEmpty(storage.table_path))
|
2020-01-31 15:10:10 +00:00
|
|
|
return {};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
|
|
|
/// How many rows to read for the next block.
|
|
|
|
size_t max_rows_to_read = std::min(block_size, rows_limit - rows_read);
|
2020-11-10 17:32:00 +00:00
|
|
|
std::unordered_map<String, IDataType::SubstreamsCache> caches;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-01-02 06:13:22 +00:00
|
|
|
for (const auto & name_type : columns)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-11-10 17:32:00 +00:00
|
|
|
ColumnPtr column;
|
|
|
|
try
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-11-10 17:32:00 +00:00
|
|
|
column = name_type.type->createColumn();
|
2020-12-22 15:03:48 +00:00
|
|
|
readData(name_type, column, max_rows_to_read, caches[name_type.getNameInStorage()]);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2020-11-10 17:32:00 +00:00
|
|
|
catch (Exception & e)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-11-10 17:32:00 +00:00
|
|
|
e.addMessage("while reading column " + name_type.name + " at " + fullPath(storage.disk, storage.table_path));
|
|
|
|
throw;
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-01-31 15:10:10 +00:00
|
|
|
if (!column->empty())
|
2018-01-02 06:13:22 +00:00
|
|
|
res.insert(ColumnWithTypeAndName(std::move(column), name_type.type, name_type.name));
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
if (res)
|
|
|
|
rows_read += res.rows();
|
|
|
|
|
|
|
|
if (!res || rows_read == rows_limit)
|
|
|
|
{
|
|
|
|
/** Close the files (before destroying the object).
|
|
|
|
* When many sources are created, but simultaneously reading only a few of them,
|
|
|
|
* buffers don't waste memory.
|
|
|
|
*/
|
|
|
|
streams.clear();
|
|
|
|
}
|
|
|
|
|
2020-01-31 15:10:10 +00:00
|
|
|
UInt64 num_rows = res.rows();
|
|
|
|
return Chunk(res.getColumns(), num_rows);
|
2010-03-18 19:32:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-11-10 17:32:00 +00:00
|
|
|
void LogSource::readData(const NameAndTypePair & name_and_type, ColumnPtr & column,
|
|
|
|
size_t max_rows_to_read, IDataType::SubstreamsCache & cache)
|
2012-08-29 20:07:24 +00:00
|
|
|
{
|
2018-06-07 18:14:37 +00:00
|
|
|
IDataType::DeserializeBinaryBulkSettings settings; /// TODO Use avg_value_size_hint.
|
2020-09-14 11:22:17 +00:00
|
|
|
const auto & [name, type] = name_and_type;
|
2018-06-07 18:14:37 +00:00
|
|
|
|
2020-10-21 23:02:20 +00:00
|
|
|
auto create_stream_getter = [&](bool stream_for_prefix)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2018-09-02 02:27:56 +00:00
|
|
|
return [&, stream_for_prefix] (const IDataType::SubstreamPath & path) -> ReadBuffer *
|
2018-06-07 18:14:37 +00:00
|
|
|
{
|
2020-11-10 17:32:00 +00:00
|
|
|
if (cache.count(IDataType::getSubcolumnNameForStream(path)))
|
|
|
|
return nullptr;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-11-10 17:32:00 +00:00
|
|
|
String stream_name = IDataType::getFileNameForStream(name_and_type, path);
|
2018-06-07 18:14:37 +00:00
|
|
|
const auto & file_it = storage.files.find(stream_name);
|
|
|
|
if (storage.files.end() == file_it)
|
|
|
|
throw Exception("Logical error: no information about file " + stream_name + " in StorageLog", ErrorCodes::LOGICAL_ERROR);
|
2017-11-28 02:13:46 +00:00
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
UInt64 offset = 0;
|
|
|
|
if (!stream_for_prefix && mark_number)
|
|
|
|
offset = file_it->second.marks[mark_number].offset;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-25 08:24:13 +00:00
|
|
|
auto & data_file_path = file_it->second.data_file_path;
|
2019-12-23 16:57:16 +00:00
|
|
|
auto it = streams.try_emplace(stream_name, storage.disk, data_file_path, offset, max_read_buffer_size).first;
|
2020-10-21 23:02:20 +00:00
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
return &it->second.compressed;
|
|
|
|
};
|
2017-08-07 07:31:16 +00:00
|
|
|
};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
if (deserialize_states.count(name) == 0)
|
|
|
|
{
|
2020-10-21 23:02:20 +00:00
|
|
|
settings.getter = create_stream_getter(true);
|
2020-09-14 11:22:17 +00:00
|
|
|
type->deserializeBinaryBulkStatePrefix(settings, deserialize_states[name]);
|
2018-06-07 18:14:37 +00:00
|
|
|
}
|
|
|
|
|
2020-10-21 23:02:20 +00:00
|
|
|
settings.getter = create_stream_getter(false);
|
2020-11-10 17:32:00 +00:00
|
|
|
type->deserializeBinaryBulkWithMultipleStreams(column, max_rows_to_read, settings, deserialize_states[name], &cache);
|
2012-08-29 20:07:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2010-03-18 19:32:14 +00:00
|
|
|
void LogBlockOutputStream::write(const Block & block)
|
|
|
|
{
|
2020-06-17 14:32:25 +00:00
|
|
|
metadata_snapshot->check(block, true);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-08-07 07:31:16 +00:00
|
|
|
/// The set of written offset columns so that you do not write shared offsets of columns for nested structures multiple times
|
|
|
|
WrittenStreams written_streams;
|
2010-03-18 19:32:14 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
MarksForColumns marks;
|
|
|
|
marks.reserve(storage.file_count);
|
2016-07-13 10:35:00 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
for (size_t i = 0; i < block.columns(); ++i)
|
|
|
|
{
|
|
|
|
const ColumnWithTypeAndName & column = block.safeGetByPosition(i);
|
2020-09-14 11:22:17 +00:00
|
|
|
writeData(NameAndTypePair(column.name, column.type), *column.column, marks, written_streams);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2016-08-10 19:12:29 +00:00
|
|
|
|
2017-11-27 21:21:09 +00:00
|
|
|
writeMarks(std::move(marks));
|
2012-08-29 20:07:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2013-09-15 01:40:29 +00:00
|
|
|
void LogBlockOutputStream::writeSuffix()
|
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
if (done)
|
|
|
|
return;
|
2015-04-02 23:58:26 +00:00
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
WrittenStreams written_streams;
|
|
|
|
IDataType::SerializeBinaryBulkSettings settings;
|
|
|
|
for (const auto & column : getHeader())
|
|
|
|
{
|
|
|
|
auto it = serialize_states.find(column.name);
|
|
|
|
if (it != serialize_states.end())
|
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
settings.getter = createStreamGetter(NameAndTypePair(column.name, column.type), written_streams);
|
2018-06-07 18:14:37 +00:00
|
|
|
column.type->serializeBinaryBulkStateSuffix(settings, it->second);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Finish write.
|
2020-01-22 16:17:25 +00:00
|
|
|
marks_stream->next();
|
2020-07-30 13:42:05 +00:00
|
|
|
marks_stream->finalize();
|
2013-09-15 01:40:29 +00:00
|
|
|
|
2017-11-26 19:22:33 +00:00
|
|
|
for (auto & name_stream : streams)
|
|
|
|
name_stream.second.finalize();
|
2013-09-15 01:40:29 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
Strings column_files;
|
2017-11-26 19:22:33 +00:00
|
|
|
for (const auto & name_stream : streams)
|
2019-12-25 08:24:13 +00:00
|
|
|
column_files.push_back(storage.files[name_stream.first].data_file_path);
|
|
|
|
column_files.push_back(storage.marks_file_path);
|
2014-08-04 06:36:24 +00:00
|
|
|
|
2020-07-12 02:31:58 +00:00
|
|
|
for (const auto & file : column_files)
|
|
|
|
storage.file_checker.update(file);
|
|
|
|
storage.file_checker.save();
|
2014-08-04 06:36:24 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
streams.clear();
|
2020-07-12 02:31:58 +00:00
|
|
|
done = true;
|
2013-09-15 01:40:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
IDataType::OutputStreamGetter LogBlockOutputStream::createStreamGetter(const NameAndTypePair & name_and_type,
|
2018-06-07 18:14:37 +00:00
|
|
|
WrittenStreams & written_streams)
|
|
|
|
{
|
|
|
|
return [&] (const IDataType::SubstreamPath & path) -> WriteBuffer *
|
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
String stream_name = IDataType::getFileNameForStream(name_and_type, path);
|
2018-06-07 18:14:37 +00:00
|
|
|
if (written_streams.count(stream_name))
|
|
|
|
return nullptr;
|
|
|
|
|
|
|
|
auto it = streams.find(stream_name);
|
|
|
|
if (streams.end() == it)
|
|
|
|
throw Exception("Logical error: stream was not created when writing data in LogBlockOutputStream",
|
|
|
|
ErrorCodes::LOGICAL_ERROR);
|
|
|
|
return &it->second.compressed;
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
void LogBlockOutputStream::writeData(const NameAndTypePair & name_and_type, const IColumn & column,
|
2019-12-23 16:57:16 +00:00
|
|
|
MarksForColumns & out_marks, WrittenStreams & written_streams)
|
2012-08-29 20:07:24 +00:00
|
|
|
{
|
2018-06-07 18:14:37 +00:00
|
|
|
IDataType::SerializeBinaryBulkSettings settings;
|
2020-09-14 11:22:17 +00:00
|
|
|
const auto & [name, type] = name_and_type;
|
2018-06-07 18:14:37 +00:00
|
|
|
|
2020-09-30 16:34:29 +00:00
|
|
|
type->enumerateStreams([&] (const IDataType::SubstreamPath & path, const IDataType & /* substream_type */)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
String stream_name = IDataType::getFileNameForStream(name_and_type, path);
|
2017-11-28 02:13:46 +00:00
|
|
|
if (written_streams.count(stream_name))
|
|
|
|
return;
|
2017-09-05 02:36:35 +00:00
|
|
|
|
2020-06-17 16:39:58 +00:00
|
|
|
const auto & columns = metadata_snapshot->getColumns();
|
2019-01-21 14:00:06 +00:00
|
|
|
streams.try_emplace(
|
|
|
|
stream_name,
|
2019-12-23 16:57:16 +00:00
|
|
|
storage.disk,
|
2019-12-25 08:24:13 +00:00
|
|
|
storage.files[stream_name].data_file_path,
|
2020-09-14 11:22:17 +00:00
|
|
|
columns.getCodecOrDefault(name_and_type.name),
|
2019-01-21 14:00:06 +00:00
|
|
|
storage.max_compress_block_size);
|
2018-06-07 18:14:37 +00:00
|
|
|
}, settings.path);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
settings.getter = createStreamGetter(name_and_type, written_streams);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
if (serialize_states.count(name) == 0)
|
2020-09-14 11:22:17 +00:00
|
|
|
type->serializeBinaryBulkStatePrefix(settings, serialize_states[name]);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-09-30 16:34:29 +00:00
|
|
|
type->enumerateStreams([&] (const IDataType::SubstreamPath & path, const IDataType & /* substream_type */)
|
2017-11-26 19:22:33 +00:00
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
String stream_name = IDataType::getFileNameForStream(name_and_type, path);
|
2017-11-28 02:13:46 +00:00
|
|
|
if (written_streams.count(stream_name))
|
2018-06-07 18:14:37 +00:00
|
|
|
return;
|
2017-11-26 19:22:33 +00:00
|
|
|
|
2018-06-07 18:14:37 +00:00
|
|
|
const auto & file = storage.files[stream_name];
|
|
|
|
const auto stream_it = streams.find(stream_name);
|
|
|
|
|
|
|
|
Mark mark;
|
|
|
|
mark.rows = (file.marks.empty() ? 0 : file.marks.back().rows) + column.size();
|
2020-01-22 16:17:25 +00:00
|
|
|
mark.offset = stream_it->second.plain_offset + stream_it->second.plain->count();
|
2018-06-07 18:14:37 +00:00
|
|
|
|
|
|
|
out_marks.emplace_back(file.column_index, mark);
|
|
|
|
}, settings.path);
|
2017-11-26 19:22:33 +00:00
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
type->serializeBinaryBulkWithMultipleStreams(column, 0, 0, settings, serialize_states[name]);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-09-30 16:34:29 +00:00
|
|
|
type->enumerateStreams([&] (const IDataType::SubstreamPath & path, const IDataType & /* substream_type */)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
String stream_name = IDataType::getFileNameForStream(name_and_type, path);
|
2017-11-28 02:13:46 +00:00
|
|
|
if (!written_streams.emplace(stream_name).second)
|
|
|
|
return;
|
|
|
|
|
2017-11-26 19:22:33 +00:00
|
|
|
auto it = streams.find(stream_name);
|
|
|
|
if (streams.end() == it)
|
|
|
|
throw Exception("Logical error: stream was not created when writing data in LogBlockOutputStream", ErrorCodes::LOGICAL_ERROR);
|
|
|
|
it->second.compressed.next();
|
2018-06-07 18:14:37 +00:00
|
|
|
}, settings.path);
|
2010-03-18 19:32:14 +00:00
|
|
|
}
|
|
|
|
|
2013-02-26 13:06:01 +00:00
|
|
|
|
2017-11-27 21:21:09 +00:00
|
|
|
void LogBlockOutputStream::writeMarks(MarksForColumns && marks)
|
2016-07-12 18:08:16 +00:00
|
|
|
{
|
2017-08-07 07:31:16 +00:00
|
|
|
if (marks.size() != storage.file_count)
|
2017-04-01 07:20:54 +00:00
|
|
|
throw Exception("Wrong number of marks generated from block. Makes no sense.", ErrorCodes::LOGICAL_ERROR);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-08-07 07:31:16 +00:00
|
|
|
std::sort(marks.begin(), marks.end(), [](const auto & a, const auto & b) { return a.first < b.first; });
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-11-26 19:22:33 +00:00
|
|
|
for (const auto & mark : marks)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-01-22 16:17:25 +00:00
|
|
|
writeIntBinary(mark.second.rows, *marks_stream);
|
|
|
|
writeIntBinary(mark.second.offset, *marks_stream);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-11-26 19:22:33 +00:00
|
|
|
size_t column_index = mark.first;
|
2019-01-04 12:10:00 +00:00
|
|
|
storage.files[storage.column_names_by_idx[column_index]].marks.push_back(mark.second);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2013-02-26 13:06:01 +00:00
|
|
|
}
|
|
|
|
|
2014-09-30 03:08:47 +00:00
|
|
|
StorageLog::StorageLog(
|
2019-12-12 08:57:25 +00:00
|
|
|
DiskPtr disk_,
|
2019-12-26 14:03:32 +00:00
|
|
|
const String & relative_path_,
|
2019-12-04 16:06:55 +00:00
|
|
|
const StorageID & table_id_,
|
2018-03-06 20:18:34 +00:00
|
|
|
const ColumnsDescription & columns_,
|
2019-08-24 21:20:20 +00:00
|
|
|
const ConstraintsDescription & constraints_,
|
2020-07-12 02:31:58 +00:00
|
|
|
bool attach,
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t max_compress_block_size_)
|
2019-12-04 16:06:55 +00:00
|
|
|
: IStorage(table_id_)
|
2020-01-13 11:41:42 +00:00
|
|
|
, disk(std::move(disk_))
|
|
|
|
, table_path(relative_path_)
|
2019-12-04 16:06:55 +00:00
|
|
|
, max_compress_block_size(max_compress_block_size_)
|
2020-01-13 11:41:42 +00:00
|
|
|
, file_checker(disk, table_path + "sizes.json")
|
2010-03-18 19:32:14 +00:00
|
|
|
{
|
2020-06-19 15:39:41 +00:00
|
|
|
StorageInMemoryMetadata storage_metadata;
|
|
|
|
storage_metadata.setColumns(columns_);
|
|
|
|
storage_metadata.setConstraints(constraints_);
|
|
|
|
setInMemoryMetadata(storage_metadata);
|
2019-08-24 21:20:20 +00:00
|
|
|
|
2019-10-25 19:07:47 +00:00
|
|
|
if (relative_path_.empty())
|
2017-11-03 19:53:10 +00:00
|
|
|
throw Exception("Storage " + getName() + " requires data path", ErrorCodes::INCORRECT_FILE_NAME);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2020-07-12 02:31:58 +00:00
|
|
|
if (!attach)
|
|
|
|
{
|
|
|
|
/// create directories if they do not exist
|
|
|
|
disk->createDirectories(table_path);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
|
|
|
file_checker.repair();
|
|
|
|
}
|
|
|
|
catch (...)
|
|
|
|
{
|
|
|
|
tryLogCurrentException(__PRETTY_FUNCTION__);
|
|
|
|
}
|
|
|
|
}
|
2012-01-10 22:11:51 +00:00
|
|
|
|
2020-06-19 15:39:41 +00:00
|
|
|
for (const auto & column : storage_metadata.getColumns().getAllPhysical())
|
2020-09-14 11:22:17 +00:00
|
|
|
addFiles(column);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2019-12-25 08:24:13 +00:00
|
|
|
marks_file_path = table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME;
|
2020-07-12 02:31:58 +00:00
|
|
|
|
|
|
|
if (!attach)
|
|
|
|
for (const auto & file : files)
|
|
|
|
file_checker.setEmpty(file.second.data_file_path);
|
2012-08-29 20:07:24 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2020-09-14 11:22:17 +00:00
|
|
|
void StorageLog::addFiles(const NameAndTypePair & column)
|
2012-08-29 20:07:24 +00:00
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
if (files.end() != files.find(column.name))
|
|
|
|
throw Exception("Duplicate column with name " + column.name + " in constructor of StorageLog.",
|
2017-04-01 07:20:54 +00:00
|
|
|
ErrorCodes::DUPLICATE_COLUMN);
|
|
|
|
|
2020-09-18 11:37:58 +00:00
|
|
|
IDataType::StreamCallback stream_callback = [&] (const IDataType::SubstreamPath & substream_path, const IDataType & /* substream_type */)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-09-14 11:22:17 +00:00
|
|
|
String stream_name = IDataType::getFileNameForStream(column, substream_path);
|
2017-09-05 02:36:35 +00:00
|
|
|
|
2017-08-07 07:31:16 +00:00
|
|
|
if (!files.count(stream_name))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2017-08-07 07:31:16 +00:00
|
|
|
ColumnData & column_data = files[stream_name];
|
2017-09-05 02:36:35 +00:00
|
|
|
column_data.column_index = file_count;
|
2019-12-25 08:24:13 +00:00
|
|
|
column_data.data_file_path = table_path + stream_name + DBMS_STORAGE_LOG_DATA_FILE_EXTENSION;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
column_names_by_idx.push_back(stream_name);
|
2017-09-05 02:36:35 +00:00
|
|
|
++file_count;
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
2017-08-07 07:31:16 +00:00
|
|
|
};
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-01-04 12:10:00 +00:00
|
|
|
IDataType::SubstreamPath substream_path;
|
2020-09-14 11:22:17 +00:00
|
|
|
column.type->enumerateStreams(stream_callback, substream_path);
|
2012-06-21 16:33:00 +00:00
|
|
|
}
|
2012-01-10 22:11:51 +00:00
|
|
|
|
2012-06-21 16:33:00 +00:00
|
|
|
|
|
|
|
void StorageLog::loadMarks()
|
|
|
|
{
|
2017-07-28 17:34:02 +00:00
|
|
|
std::unique_lock<std::shared_mutex> lock(rwlock);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
if (loaded_marks)
|
|
|
|
return;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
using FilesByIndex = std::vector<Files::iterator>;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-08-07 07:31:16 +00:00
|
|
|
FilesByIndex files_by_index(file_count);
|
2019-12-12 08:57:25 +00:00
|
|
|
for (Files::iterator it = files.begin(); it != files.end(); ++it)
|
2017-04-01 07:20:54 +00:00
|
|
|
files_by_index[it->second.column_index] = it;
|
|
|
|
|
2019-12-25 08:24:13 +00:00
|
|
|
if (disk->exists(marks_file_path))
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2019-12-25 08:24:13 +00:00
|
|
|
size_t file_size = disk->getFileSize(marks_file_path);
|
2017-08-07 07:31:16 +00:00
|
|
|
if (file_size % (file_count * sizeof(Mark)) != 0)
|
2017-04-01 07:20:54 +00:00
|
|
|
throw Exception("Size of marks file is inconsistent", ErrorCodes::SIZES_OF_MARKS_FILES_ARE_INCONSISTENT);
|
|
|
|
|
2017-08-07 07:31:16 +00:00
|
|
|
size_t marks_count = file_size / (file_count * sizeof(Mark));
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-11-26 19:22:33 +00:00
|
|
|
for (auto & file : files_by_index)
|
|
|
|
file->second.marks.reserve(marks_count);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-26 14:28:22 +00:00
|
|
|
std::unique_ptr<ReadBuffer> marks_rb = disk->readFile(marks_file_path, 32768);
|
2019-12-12 08:57:25 +00:00
|
|
|
while (!marks_rb->eof())
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2020-03-09 01:22:33 +00:00
|
|
|
for (auto & file : files_by_index)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
|
|
|
Mark mark;
|
2019-12-12 08:57:25 +00:00
|
|
|
readIntBinary(mark.rows, *marks_rb);
|
|
|
|
readIntBinary(mark.offset, *marks_rb);
|
2020-03-09 01:22:33 +00:00
|
|
|
file->second.marks.push_back(mark);
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-08-07 07:31:16 +00:00
|
|
|
|
|
|
|
loaded_marks = true;
|
2010-03-18 19:32:14 +00:00
|
|
|
}
|
|
|
|
|
2013-02-07 13:03:19 +00:00
|
|
|
|
2020-04-07 14:05:51 +00:00
|
|
|
void StorageLog::rename(const String & new_path_to_table_data, const StorageID & new_table_id)
|
2012-06-18 06:19:13 +00:00
|
|
|
{
|
2020-09-18 19:25:56 +00:00
|
|
|
assert(table_path != new_path_to_table_data);
|
2020-09-17 19:50:43 +00:00
|
|
|
{
|
|
|
|
std::unique_lock<std::shared_mutex> lock(rwlock);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2020-09-17 19:50:43 +00:00
|
|
|
disk->moveDirectory(table_path, new_path_to_table_data);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2020-09-17 19:50:43 +00:00
|
|
|
table_path = new_path_to_table_data;
|
|
|
|
file_checker.setPath(table_path + "sizes.json");
|
2012-06-18 06:19:13 +00:00
|
|
|
|
2020-09-17 19:50:43 +00:00
|
|
|
for (auto & file : files)
|
|
|
|
file.second.data_file_path = table_path + fileName(file.second.data_file_path);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2020-09-17 19:50:43 +00:00
|
|
|
marks_file_path = table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME;
|
|
|
|
}
|
2020-04-07 14:05:51 +00:00
|
|
|
renameInMemory(new_table_id);
|
2012-06-18 06:19:13 +00:00
|
|
|
}
|
|
|
|
|
2020-06-18 16:10:47 +00:00
|
|
|
void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, const Context &, TableExclusiveLockHolder &)
|
2018-04-21 00:35:20 +00:00
|
|
|
{
|
|
|
|
std::shared_lock<std::shared_mutex> lock(rwlock);
|
|
|
|
|
2018-06-09 15:48:22 +00:00
|
|
|
files.clear();
|
|
|
|
file_count = 0;
|
|
|
|
loaded_marks = false;
|
2018-04-21 00:35:20 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
disk->clearDirectory(table_path);
|
2018-04-21 00:35:20 +00:00
|
|
|
|
2020-06-17 16:39:58 +00:00
|
|
|
for (const auto & column : metadata_snapshot->getColumns().getAllPhysical())
|
2020-09-14 11:22:17 +00:00
|
|
|
addFiles(column);
|
2018-04-21 00:35:20 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
file_checker = FileChecker{disk, table_path + "sizes.json"};
|
2019-12-25 08:24:13 +00:00
|
|
|
marks_file_path = table_path + DBMS_STORAGE_LOG_MARKS_FILE_NAME;
|
2018-04-21 00:35:20 +00:00
|
|
|
}
|
|
|
|
|
2012-06-18 06:19:13 +00:00
|
|
|
|
2020-06-17 16:39:58 +00:00
|
|
|
const StorageLog::Marks & StorageLog::getMarksWithRealRowCount(const StorageMetadataPtr & metadata_snapshot) const
|
2016-07-12 18:08:16 +00:00
|
|
|
{
|
2020-02-19 18:06:18 +00:00
|
|
|
/// There should be at least one physical column
|
2020-09-14 11:22:17 +00:00
|
|
|
auto column = *metadata_snapshot->getColumns().getAllPhysical().begin();
|
2017-04-01 07:20:54 +00:00
|
|
|
String filename;
|
|
|
|
|
|
|
|
/** We take marks from first column.
|
2017-12-02 22:12:27 +00:00
|
|
|
* If this is a data type with multiple stream, get the first stream, that we assume have real row count.
|
|
|
|
* (Example: for Array data type, first stream is array sizes; and number of array sizes is the number of arrays).
|
2017-04-01 07:20:54 +00:00
|
|
|
*/
|
2019-01-04 12:10:00 +00:00
|
|
|
IDataType::SubstreamPath substream_root_path;
|
2020-09-30 16:34:29 +00:00
|
|
|
column.type->enumerateStreams([&](const IDataType::SubstreamPath & substream_path, const IDataType & /* substream_type */)
|
2017-12-02 22:12:27 +00:00
|
|
|
{
|
|
|
|
if (filename.empty())
|
2020-09-14 11:22:17 +00:00
|
|
|
filename = IDataType::getFileNameForStream(column, substream_path);
|
2019-01-04 12:10:00 +00:00
|
|
|
}, substream_root_path);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2019-12-12 08:57:25 +00:00
|
|
|
Files::const_iterator it = files.find(filename);
|
2017-04-01 07:20:54 +00:00
|
|
|
if (files.end() == it)
|
|
|
|
throw Exception("Cannot find file " + filename, ErrorCodes::LOGICAL_ERROR);
|
|
|
|
|
|
|
|
return it->second.marks;
|
2013-12-12 22:55:47 +00:00
|
|
|
}
|
|
|
|
|
2020-08-03 13:54:14 +00:00
|
|
|
Pipe StorageLog::read(
|
2017-04-01 07:20:54 +00:00
|
|
|
const Names & column_names,
|
2020-06-17 14:32:25 +00:00
|
|
|
const StorageMetadataPtr & metadata_snapshot,
|
2020-09-20 17:52:17 +00:00
|
|
|
SelectQueryInfo & /*query_info*/,
|
2017-04-01 07:20:54 +00:00
|
|
|
const Context & context,
|
2018-09-08 11:29:23 +00:00
|
|
|
QueryProcessingStage::Enum /*processed_stage*/,
|
2019-02-18 23:38:44 +00:00
|
|
|
size_t max_block_size,
|
2017-06-02 15:54:39 +00:00
|
|
|
unsigned num_streams)
|
2010-03-18 19:32:14 +00:00
|
|
|
{
|
2020-06-19 17:17:13 +00:00
|
|
|
metadata_snapshot->check(column_names, getVirtuals(), getStorageID());
|
2017-04-01 07:20:54 +00:00
|
|
|
loadMarks();
|
|
|
|
|
2020-10-29 14:14:23 +00:00
|
|
|
auto all_columns = metadata_snapshot->getColumns().getAllWithSubcolumns().addTypes(column_names);
|
2020-12-07 19:02:26 +00:00
|
|
|
all_columns = Nested::convertToSubcolumns(all_columns);
|
2018-01-02 06:13:22 +00:00
|
|
|
|
2017-07-28 17:34:02 +00:00
|
|
|
std::shared_lock<std::shared_mutex> lock(rwlock);
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-01-31 15:10:10 +00:00
|
|
|
Pipes pipes;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2020-06-17 16:39:58 +00:00
|
|
|
const Marks & marks = getMarksWithRealRowCount(metadata_snapshot);
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t marks_size = marks.size();
|
|
|
|
|
2017-06-02 15:54:39 +00:00
|
|
|
if (num_streams > marks_size)
|
|
|
|
num_streams = marks_size;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-05-24 21:06:29 +00:00
|
|
|
size_t max_read_buffer_size = context.getSettingsRef().max_read_buffer_size;
|
2017-04-01 07:20:54 +00:00
|
|
|
|
2017-08-07 07:31:16 +00:00
|
|
|
for (size_t stream = 0; stream < num_streams; ++stream)
|
2017-04-01 07:20:54 +00:00
|
|
|
{
|
2017-11-27 21:21:09 +00:00
|
|
|
size_t mark_begin = stream * marks_size / num_streams;
|
|
|
|
size_t mark_end = (stream + 1) * marks_size / num_streams;
|
|
|
|
|
|
|
|
size_t rows_begin = mark_begin ? marks[mark_begin - 1].rows : 0;
|
|
|
|
size_t rows_end = mark_end ? marks[mark_end - 1].rows : 0;
|
|
|
|
|
2020-01-31 15:10:10 +00:00
|
|
|
pipes.emplace_back(std::make_shared<LogSource>(
|
2017-08-07 07:31:16 +00:00
|
|
|
max_block_size,
|
2018-03-06 20:18:34 +00:00
|
|
|
all_columns,
|
2017-08-07 07:31:16 +00:00
|
|
|
*this,
|
2017-11-27 21:21:09 +00:00
|
|
|
mark_begin,
|
|
|
|
rows_end - rows_begin,
|
2017-08-07 07:31:16 +00:00
|
|
|
max_read_buffer_size));
|
2017-04-01 07:20:54 +00:00
|
|
|
}
|
|
|
|
|
2020-08-06 12:24:05 +00:00
|
|
|
return Pipe::unitePipes(std::move(pipes));
|
2010-03-18 19:32:14 +00:00
|
|
|
}
|
|
|
|
|
2020-06-16 15:51:29 +00:00
|
|
|
BlockOutputStreamPtr StorageLog::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, const Context & /*context*/)
|
2010-03-18 19:32:14 +00:00
|
|
|
{
|
2017-04-01 07:20:54 +00:00
|
|
|
loadMarks();
|
2020-06-16 15:51:29 +00:00
|
|
|
return std::make_shared<LogBlockOutputStream>(*this, metadata_snapshot);
|
2010-03-18 19:32:14 +00:00
|
|
|
}
|
|
|
|
|
2019-07-03 13:17:19 +00:00
|
|
|
CheckResults StorageLog::checkData(const ASTPtr & /* query */, const Context & /* context */)
|
2014-08-04 06:36:24 +00:00
|
|
|
{
|
2017-07-28 17:34:02 +00:00
|
|
|
std::shared_lock<std::shared_mutex> lock(rwlock);
|
2017-04-01 07:20:54 +00:00
|
|
|
return file_checker.check();
|
2014-08-04 06:36:24 +00:00
|
|
|
}
|
2011-11-05 23:31:19 +00:00
|
|
|
|
2017-12-30 00:36:06 +00:00
|
|
|
|
|
|
|
void registerStorageLog(StorageFactory & factory)
|
|
|
|
{
|
2020-02-18 14:41:30 +00:00
|
|
|
StorageFactory::StorageFeatures features{
|
|
|
|
.supports_settings = true
|
|
|
|
};
|
|
|
|
|
2017-12-30 00:36:06 +00:00
|
|
|
factory.registerStorage("Log", [](const StorageFactory::Arguments & args)
|
|
|
|
{
|
|
|
|
if (!args.engine_args.empty())
|
|
|
|
throw Exception(
|
|
|
|
"Engine " + args.engine_name + " doesn't support any arguments (" + toString(args.engine_args.size()) + " given)",
|
|
|
|
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
|
|
|
|
|
2020-02-18 14:41:30 +00:00
|
|
|
String disk_name = getDiskName(*args.storage_def);
|
|
|
|
DiskPtr disk = args.context.getDisk(disk_name);
|
2020-02-14 14:28:33 +00:00
|
|
|
|
2017-12-30 00:36:06 +00:00
|
|
|
return StorageLog::create(
|
2020-02-14 14:28:33 +00:00
|
|
|
disk, args.relative_data_path, args.table_id, args.columns, args.constraints,
|
2020-07-12 02:31:58 +00:00
|
|
|
args.attach, args.context.getSettings().max_compress_block_size);
|
2020-02-18 14:41:30 +00:00
|
|
|
}, features);
|
2017-12-30 00:36:06 +00:00
|
|
|
}
|
|
|
|
|
2010-03-18 19:32:14 +00:00
|
|
|
}
|