ClickHouse/dbms/include/DB/Storages/MergeTree/MergedBlockOutputStream.h

395 lines
13 KiB
C
Raw Normal View History

2013-04-24 10:31:32 +00:00
#pragma once
2013-09-15 01:10:16 +00:00
#include <DB/IO/WriteBufferFromFile.h>
#include <DB/IO/CompressedWriteBuffer.h>
2014-03-27 17:30:04 +00:00
#include <DB/IO/HashingWriteBuffer.h>
2013-09-15 01:10:16 +00:00
2014-03-09 17:36:01 +00:00
#include <DB/Storages/MergeTree/MergeTreeData.h>
#include <DB/Common/escapeForFileName.h>
#include <DB/DataTypes/DataTypeNested.h>
#include <DB/DataTypes/DataTypeArray.h>
2013-04-24 10:31:32 +00:00
2013-09-15 01:10:16 +00:00
2013-04-24 10:31:32 +00:00
namespace DB
{
class IMergedBlockOutputStream : public IBlockOutputStream
2013-04-24 10:31:32 +00:00
{
public:
2014-04-08 15:29:12 +00:00
IMergedBlockOutputStream(MergeTreeData & storage_, size_t min_compress_block_size_, size_t max_compress_block_size_) : storage(storage_), index_offset(0), min_compress_block_size(min_compress_block_size_), max_compress_block_size(max_compress_block_size_)
2013-04-24 10:31:32 +00:00
{
}
2013-08-24 08:01:19 +00:00
protected:
typedef std::set<std::string> OffsetColumns;
2013-04-24 10:31:32 +00:00
struct ColumnStream
{
2014-04-08 15:29:12 +00:00
ColumnStream(const String & escaped_column_name_, const String & data_path, const std::string & marks_path, size_t max_compress_block_size = DEFAULT_MAX_COMPRESS_BLOCK_SIZE) :
2014-03-27 17:30:04 +00:00
escaped_column_name(escaped_column_name_),
2014-03-28 14:36:24 +00:00
plain_file(data_path, max_compress_block_size, O_TRUNC | O_CREAT | O_WRONLY),
2014-04-14 13:08:26 +00:00
plain_hashing(plain_file), compressed_buf(plain_hashing), compressed(compressed_buf),
marks_file(marks_path, 4096, O_TRUNC | O_CREAT | O_WRONLY), marks(marks_file) {}
2014-03-27 17:30:04 +00:00
String escaped_column_name;
2014-04-14 13:08:26 +00:00
/// compressed -> compressed_buf -> plain_hashing -> plain_file
2014-03-27 17:30:04 +00:00
WriteBufferFromFile plain_file;
2014-04-14 13:08:26 +00:00
HashingWriteBuffer plain_hashing;
2014-03-27 17:30:04 +00:00
CompressedWriteBuffer compressed_buf;
2014-04-14 13:13:20 +00:00
HashingWriteBuffer compressed;
2014-04-14 13:08:26 +00:00
/// marks -> marks_file
WriteBufferFromFile marks_file;
2014-03-27 17:30:04 +00:00
HashingWriteBuffer marks;
2013-09-15 01:10:16 +00:00
2013-09-26 19:16:43 +00:00
void finalize()
2013-09-15 01:10:16 +00:00
{
compressed.next();
2014-03-27 17:30:04 +00:00
plain_file.next();
2013-10-03 13:06:27 +00:00
marks.next();
2013-09-15 01:10:16 +00:00
}
2014-03-05 16:28:24 +00:00
void sync()
{
2014-03-27 17:30:04 +00:00
plain_file.sync();
marks_file.sync();
}
void addToChecksums(MergeTreeData::DataPart::Checksums & checksums, String name = "")
{
if (name == "")
name = escaped_column_name;
2014-04-14 13:08:26 +00:00
checksums.files[name + ".bin"].is_compressed = true;
checksums.files[name + ".bin"].uncompressed_size = compressed.count();
checksums.files[name + ".bin"].uncompressed_hash = compressed.getHash();
checksums.files[name + ".bin"].file_size = plain_hashing.count();
checksums.files[name + ".bin"].file_hash = plain_hashing.getHash();
checksums.files[name + ".mrk"].file_size = marks.count();
checksums.files[name + ".mrk"].file_hash = marks.getHash();
2014-03-05 16:28:24 +00:00
}
2013-04-24 10:31:32 +00:00
};
2014-04-22 22:43:55 +00:00
typedef std::map<String, std::unique_ptr<ColumnStream> > ColumnStreams;
void addStream(const String & path, const String & name, const IDataType & type, size_t level = 0, String filename = "")
2013-04-24 10:31:32 +00:00
{
String escaped_column_name;
if (filename.size())
escaped_column_name = escapeForFileName(filename);
else
escaped_column_name = escapeForFileName(name);
2013-04-24 10:31:32 +00:00
/// Для массивов используются отдельные потоки для размеров.
if (const DataTypeArray * type_arr = typeid_cast<const DataTypeArray *>(&type))
2013-04-24 10:31:32 +00:00
{
String size_name = DataTypeNested::extractNestedTableName(name)
+ ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level);
String escaped_size_name = escapeForFileName(DataTypeNested::extractNestedTableName(name))
+ ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level);
2014-04-22 22:43:55 +00:00
column_streams[size_name].reset(new ColumnStream(
2014-03-27 17:30:04 +00:00
escaped_size_name,
path + escaped_size_name + ".bin",
2014-03-28 14:36:24 +00:00
path + escaped_size_name + ".mrk",
2014-04-22 22:43:55 +00:00
max_compress_block_size));
addStream(path, name, *type_arr->getNestedType(), level + 1);
2013-04-24 10:31:32 +00:00
}
else
2014-04-22 22:43:55 +00:00
column_streams[name].reset(new ColumnStream(
2014-03-27 17:30:04 +00:00
escaped_column_name,
path + escaped_column_name + ".bin",
2014-03-28 14:36:24 +00:00
path + escaped_column_name + ".mrk",
2014-04-22 22:43:55 +00:00
max_compress_block_size));
2013-04-24 10:31:32 +00:00
}
2013-04-24 10:31:32 +00:00
/// Записать данные одного столбца.
void writeData(const String & name, const IDataType & type, const IColumn & column, OffsetColumns & offset_columns, size_t level = 0)
2013-04-24 10:31:32 +00:00
{
size_t size = column.size();
2013-04-24 10:31:32 +00:00
/// Для массивов требуется сначала сериализовать размеры, а потом значения.
if (const DataTypeArray * type_arr = typeid_cast<const DataTypeArray *>(&type))
2013-04-24 10:31:32 +00:00
{
String size_name = DataTypeNested::extractNestedTableName(name)
+ ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level);
if (offset_columns.count(size_name) == 0)
2013-04-24 10:31:32 +00:00
{
offset_columns.insert(size_name);
ColumnStream & stream = *column_streams[size_name];
size_t prev_mark = 0;
while (prev_mark < size)
2013-04-24 10:31:32 +00:00
{
size_t limit = 0;
/// Если есть index_offset, то первая засечка идёт не сразу, а после этого количества строк.
if (prev_mark == 0 && index_offset != 0)
{
limit = index_offset;
}
else
{
limit = storage.index_granularity;
/// Уже могло накопиться достаточно данных для сжатия в новый блок.
if (stream.compressed.offset() >= min_compress_block_size)
stream.compressed.next();
2014-04-14 13:08:26 +00:00
writeIntBinary(stream.plain_hashing.count(), stream.marks);
writeIntBinary(stream.compressed.offset(), stream.marks);
}
type_arr->serializeOffsets(column, stream.compressed, prev_mark, limit);
stream.compressed.nextIfAtEnd(); /// Чтобы вместо засечек, указывающих на конец сжатого блока, были засечки, указывающие на начало следующего.
prev_mark += limit;
2013-04-24 10:31:32 +00:00
}
}
}
2013-04-24 10:31:32 +00:00
{
ColumnStream & stream = *column_streams[name];
2013-04-24 10:31:32 +00:00
size_t prev_mark = 0;
while (prev_mark < size)
{
size_t limit = 0;
2013-04-24 10:31:32 +00:00
/// Если есть index_offset, то первая засечка идёт не сразу, а после этого количества строк.
if (prev_mark == 0 && index_offset != 0)
{
limit = index_offset;
}
else
{
limit = storage.index_granularity;
/// Уже могло накопиться достаточно данных для сжатия в новый блок.
if (stream.compressed.offset() >= min_compress_block_size)
stream.compressed.next();
2014-04-14 13:08:26 +00:00
writeIntBinary(stream.plain_hashing.count(), stream.marks);
2013-04-24 10:31:32 +00:00
writeIntBinary(stream.compressed.offset(), stream.marks);
}
2013-04-24 10:31:32 +00:00
type.serializeBinary(column, stream.compressed, prev_mark, limit);
stream.compressed.nextIfAtEnd(); /// Чтобы вместо засечек, указывающих на конец сжатого блока, были засечки, указывающие на начало следующего.
2013-04-24 10:31:32 +00:00
prev_mark += limit;
}
}
}
2014-03-09 17:36:01 +00:00
MergeTreeData & storage;
ColumnStreams column_streams;
/// Смещение до первой строчки блока, для которой надо записать индекс.
size_t index_offset;
2014-04-08 15:29:12 +00:00
size_t min_compress_block_size;
size_t max_compress_block_size;
};
/** Для записи одного куска. Данные уже отсортированы, относятся к одному месяцу, и пишутся в один кускок.
*/
class MergedBlockOutputStream : public IMergedBlockOutputStream
{
public:
MergedBlockOutputStream(MergeTreeData & storage_, String part_path_, const NamesAndTypesList & columns_list_)
2014-04-08 15:29:12 +00:00
: IMergedBlockOutputStream(storage_, storage_.context.getSettings().min_compress_block_size, storage_.context.getSettings().max_compress_block_size), columns_list(columns_list_), part_path(part_path_), marks_count(0)
{
Poco::File(part_path).createDirectories();
2014-03-27 17:30:04 +00:00
index_file_stream = new WriteBufferFromFile(part_path + "primary.idx", DBMS_DEFAULT_BUFFER_SIZE, O_TRUNC | O_CREAT | O_WRONLY);
index_stream = new HashingWriteBuffer(*index_file_stream);
2014-03-13 12:48:07 +00:00
for (const auto & it : columns_list)
addStream(part_path, it.name, *it.type);
}
2014-03-13 12:48:07 +00:00
void write(const Block & block)
{
size_t rows = block.rows();
/// Сначала пишем индекс. Индекс содержит значение Primary Key для каждой index_granularity строки.
typedef std::vector<const ColumnWithNameAndType *> PrimaryColumns;
PrimaryColumns primary_columns;
2014-03-14 17:03:52 +00:00
for (const auto & descr : storage.getSortDescription())
primary_columns.push_back(
2014-03-14 17:03:52 +00:00
!descr.column_name.empty()
? &block.getByName(descr.column_name)
: &block.getByPosition(descr.column_number));
for (size_t i = index_offset; i < rows; i += storage.index_granularity)
{
for (PrimaryColumns::const_iterator it = primary_columns.begin(); it != primary_columns.end(); ++it)
{
index_vec.push_back((*(*it)->column)[i]);
(*it)->type->serializeBinary(index_vec.back(), *index_stream);
}
++marks_count;
}
/// Множество записанных столбцов со смещениями, чтобы не писать общие для вложенных структур столбцы несколько раз
OffsetColumns offset_columns;
/// Теперь пишем данные.
2014-03-13 12:48:07 +00:00
for (const auto & it : columns_list)
{
const ColumnWithNameAndType & column = block.getByName(it.name);
writeData(column.name, *column.type, *column.column, offset_columns);
}
size_t written_for_last_mark = (storage.index_granularity - index_offset + rows) % storage.index_granularity;
index_offset = (storage.index_granularity - written_for_last_mark) % storage.index_granularity;
}
2014-03-27 17:30:04 +00:00
void writeSuffix() override
{
throw Exception("Method writeSuffix is not supported by MergedBlockOutputStream", ErrorCodes::NOT_IMPLEMENTED);
}
2014-03-27 17:30:04 +00:00
MergeTreeData::DataPart::Checksums writeSuffixAndGetChecksums()
{
2014-03-27 17:30:04 +00:00
/// Заканчиваем запись и достаем чексуммы.
MergeTreeData::DataPart::Checksums checksums;
index_stream->next();
2014-04-14 13:08:26 +00:00
checksums.files["primary.idx"].file_size = index_stream->count();
checksums.files["primary.idx"].file_hash = index_stream->getHash();
for (ColumnStreams::iterator it = column_streams.begin(); it != column_streams.end(); ++it)
2014-03-27 17:30:04 +00:00
{
it->second->finalize();
2014-03-27 17:30:04 +00:00
it->second->addToChecksums(checksums);
}
2014-04-08 07:31:51 +00:00
index_stream = nullptr;
column_streams.clear();
if (marks_count == 0)
{
/// Кусок пустой - все записи удалились.
Poco::File(part_path).remove(true);
2014-03-27 17:30:04 +00:00
checksums.files.clear();
2014-07-09 13:39:19 +00:00
return checksums;
2014-03-27 17:30:04 +00:00
}
2014-07-09 13:39:19 +00:00
{
/// Записываем файл с описанием столбцов.
WriteBufferFromFile out(part_path + "columns.txt", 4096);
columns_list.writeText(out);
}
2014-03-27 17:30:04 +00:00
{
/// Записываем файл с чексуммами.
2014-07-09 13:39:19 +00:00
WriteBufferFromFile out(part_path + "checksums.txt", 4096);
2014-03-27 17:30:04 +00:00
checksums.writeText(out);
}
2014-03-27 17:30:04 +00:00
return checksums;
}
MergeTreeData::DataPart::Index & getIndex()
{
return index_vec;
}
/// Сколько засечек уже записано.
size_t marksCount()
{
return marks_count;
}
private:
2014-03-13 12:48:07 +00:00
NamesAndTypesList columns_list;
String part_path;
2014-03-13 12:48:07 +00:00
size_t marks_count;
2014-03-27 17:30:04 +00:00
SharedPtr<WriteBufferFromFile> index_file_stream;
SharedPtr<HashingWriteBuffer> index_stream;
MergeTreeData::DataPart::Index index_vec;
2013-04-24 10:31:32 +00:00
};
typedef Poco::SharedPtr<MergedBlockOutputStream> MergedBlockOutputStreamPtr;
/// Записывает только те, столбцы, что лежат в block
class MergedColumnOnlyOutputStream : public IMergedBlockOutputStream
{
public:
2014-03-09 17:36:01 +00:00
MergedColumnOnlyOutputStream(MergeTreeData & storage_, String part_path_, bool sync_ = false) :
2014-04-08 15:29:12 +00:00
IMergedBlockOutputStream(storage_, storage_.context.getSettings().min_compress_block_size, storage_.context.getSettings().max_compress_block_size), part_path(part_path_), initialized(false), sync(sync_)
{
}
void write(const Block & block)
{
if (!initialized)
{
column_streams.clear();
for (size_t i = 0; i < block.columns(); ++i)
{
addStream(part_path, block.getByPosition(i).name,
2014-07-17 10:44:17 +00:00
*block.getByPosition(i).type, 0, block.getByPosition(i).name);
}
initialized = true;
}
size_t rows = block.rows();
OffsetColumns offset_columns;
for (size_t i = 0; i < block.columns(); ++i)
{
const ColumnWithNameAndType & column = block.getByPosition(i);
writeData(column.name, *column.type, *column.column, offset_columns);
}
size_t written_for_last_mark = (storage.index_granularity - index_offset + rows) % storage.index_granularity;
index_offset = (storage.index_granularity - written_for_last_mark) % storage.index_granularity;
}
2014-03-27 17:30:04 +00:00
void writeSuffix() override
{
2014-03-27 17:30:04 +00:00
throw Exception("Method writeSuffix is not supported by MergedColumnOnlyOutputStream", ErrorCodes::NOT_IMPLEMENTED);
}
MergeTreeData::DataPart::Checksums writeSuffixAndGetChecksums()
{
MergeTreeData::DataPart::Checksums checksums;
for (auto & column_stream : column_streams)
{
column_stream.second->finalize();
2014-03-05 16:28:24 +00:00
if (sync)
column_stream.second->sync();
2014-03-04 14:09:19 +00:00
std::string column = escapeForFileName(column_stream.first);
2014-03-27 17:30:04 +00:00
column_stream.second->addToChecksums(checksums, column);
}
2014-03-27 17:30:04 +00:00
column_streams.clear();
initialized = false;
2014-03-27 17:30:04 +00:00
return checksums;
}
private:
String part_path;
bool initialized;
2014-03-05 16:28:24 +00:00
bool sync;
};
2013-09-26 19:16:43 +00:00
}