2013-04-24 10:31:32 +00:00
|
|
|
#pragma once
|
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <IO/WriteBufferFromFile.h>
|
|
|
|
#include <IO/CompressedWriteBuffer.h>
|
|
|
|
#include <IO/HashingWriteBuffer.h>
|
|
|
|
#include <Storages/MergeTree/MergeTreeData.h>
|
|
|
|
#include <DataStreams/IBlockOutputStream.h>
|
2013-04-24 10:31:32 +00:00
|
|
|
|
2017-04-01 09:19:00 +00:00
|
|
|
#include <Columns/ColumnArray.h>
|
2016-11-03 12:00:44 +00:00
|
|
|
|
2013-09-15 01:10:16 +00:00
|
|
|
|
2013-04-24 10:31:32 +00:00
|
|
|
namespace DB
|
|
|
|
{
|
2015-08-14 02:45:40 +00:00
|
|
|
|
|
|
|
|
2014-03-04 11:30:50 +00:00
|
|
|
class IMergedBlockOutputStream : public IBlockOutputStream
|
2013-04-24 10:31:32 +00:00
|
|
|
{
|
|
|
|
public:
|
2017-04-01 07:20:54 +00:00
|
|
|
IMergedBlockOutputStream(
|
|
|
|
MergeTreeData & storage_,
|
|
|
|
size_t min_compress_block_size_,
|
|
|
|
size_t max_compress_block_size_,
|
|
|
|
CompressionMethod compression_method_,
|
|
|
|
size_t aio_threshold_);
|
2013-08-24 08:01:19 +00:00
|
|
|
|
2014-03-04 11:30:50 +00:00
|
|
|
protected:
|
2017-04-01 07:20:54 +00:00
|
|
|
using OffsetColumns = std::set<std::string>;
|
2015-03-14 02:36:39 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
struct ColumnStream
|
|
|
|
{
|
|
|
|
ColumnStream(
|
|
|
|
const String & escaped_column_name_,
|
|
|
|
const String & data_path,
|
|
|
|
const std::string & data_file_extension_,
|
|
|
|
const std::string & marks_path,
|
|
|
|
const std::string & marks_file_extension_,
|
|
|
|
size_t max_compress_block_size,
|
|
|
|
CompressionMethod compression_method,
|
|
|
|
size_t estimated_size,
|
|
|
|
size_t aio_threshold);
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
String escaped_column_name;
|
|
|
|
std::string data_file_extension;
|
|
|
|
std::string marks_file_extension;
|
2014-04-14 13:08:26 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// compressed -> compressed_buf -> plain_hashing -> plain_file
|
|
|
|
std::unique_ptr<WriteBufferFromFileBase> plain_file;
|
|
|
|
HashingWriteBuffer plain_hashing;
|
|
|
|
CompressedWriteBuffer compressed_buf;
|
|
|
|
HashingWriteBuffer compressed;
|
2014-04-14 13:08:26 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
/// marks -> marks_file
|
|
|
|
WriteBufferFromFile marks_file;
|
|
|
|
HashingWriteBuffer marks;
|
2013-09-15 01:10:16 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void finalize();
|
2016-07-21 16:22:24 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void sync();
|
2016-07-21 16:22:24 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void addToChecksums(MergeTreeData::DataPart::Checksums & checksums);
|
|
|
|
};
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
using ColumnStreams = std::map<String, std::unique_ptr<ColumnStream>>;
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void addStream(const String & path, const String & name, const IDataType & type, size_t estimated_size,
|
|
|
|
size_t level, const String & filename, bool skip_offsets);
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/// Write data of one column.
|
2017-04-01 07:20:54 +00:00
|
|
|
void writeData(const String & name, const IDataType & type, const IColumn & column, OffsetColumns & offset_columns,
|
|
|
|
size_t level, bool skip_offsets);
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
MergeTreeData & storage;
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
ColumnStreams column_streams;
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/// The offset to the first row of the block for which you want to write the index.
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t index_offset = 0;
|
2014-04-08 15:29:12 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t min_compress_block_size;
|
|
|
|
size_t max_compress_block_size;
|
2015-03-14 02:36:39 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t aio_threshold;
|
2015-04-08 16:48:47 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
CompressionMethod compression_method;
|
2016-10-20 10:13:07 +00:00
|
|
|
|
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
/// Internal version of writeData.
|
|
|
|
void writeDataImpl(const String & name, const IDataType & type, const IColumn & column,
|
|
|
|
OffsetColumns & offset_columns, size_t level, bool write_array_data, bool skip_offsets);
|
2014-03-04 11:30:50 +00:00
|
|
|
};
|
|
|
|
|
2015-08-14 02:45:40 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/** To write one part.
|
2017-08-14 18:16:11 +00:00
|
|
|
* The data refers to one partition, and is written in one part.
|
2014-03-04 11:30:50 +00:00
|
|
|
*/
|
|
|
|
class MergedBlockOutputStream : public IMergedBlockOutputStream
|
|
|
|
{
|
|
|
|
public:
|
2017-04-01 07:20:54 +00:00
|
|
|
MergedBlockOutputStream(
|
|
|
|
MergeTreeData & storage_,
|
|
|
|
String part_path_,
|
|
|
|
const NamesAndTypesList & columns_list_,
|
|
|
|
CompressionMethod compression_method);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
MergedBlockOutputStream(
|
|
|
|
MergeTreeData & storage_,
|
|
|
|
String part_path_,
|
|
|
|
const NamesAndTypesList & columns_list_,
|
|
|
|
CompressionMethod compression_method,
|
|
|
|
const MergeTreeData::DataPart::ColumnToSize & merged_column_to_size_,
|
|
|
|
size_t aio_threshold_);
|
2016-07-21 16:22:24 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
std::string getPartPath() const;
|
2016-01-28 16:06:57 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/// If the data is pre-sorted.
|
2017-04-01 07:20:54 +00:00
|
|
|
void write(const Block & block) override;
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/** If the data is not sorted, but we have previously calculated the permutation, after which they will be sorted.
|
|
|
|
* This method is used to save RAM, since you do not need to keep two blocks at once - the original one and the sorted one.
|
2017-04-01 07:20:54 +00:00
|
|
|
*/
|
|
|
|
void writeWithPermutation(const Block & block, const IColumn::Permutation * permutation);
|
2014-03-27 17:30:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void writeSuffix() override;
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-08-30 19:03:19 +00:00
|
|
|
void writeSuffixAndFinalizePart(
|
|
|
|
MergeTreeData::MutableDataPartPtr & new_part,
|
|
|
|
const NamesAndTypesList * total_columns_list = nullptr,
|
|
|
|
MergeTreeData::DataPart::Checksums * additional_column_checksums = nullptr);
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/// How many marks are already written.
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t marksCount();
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2015-04-10 15:31:51 +00:00
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
void init();
|
2015-04-10 15:31:51 +00:00
|
|
|
|
2017-05-09 19:07:35 +00:00
|
|
|
/** If `permutation` is given, it rearranges the values in the columns when writing.
|
2017-04-16 15:00:33 +00:00
|
|
|
* This is necessary to not keep the whole block in the RAM to sort it.
|
2017-04-01 07:20:54 +00:00
|
|
|
*/
|
|
|
|
void writeImpl(const Block & block, const IColumn::Permutation * permutation);
|
2015-08-14 02:45:40 +00:00
|
|
|
|
2014-03-04 11:30:50 +00:00
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
NamesAndTypesList columns_list;
|
|
|
|
String part_path;
|
2014-03-13 12:48:07 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
size_t marks_count = 0;
|
2014-03-27 12:32:37 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
std::unique_ptr<WriteBufferFromFile> index_file_stream;
|
|
|
|
std::unique_ptr<HashingWriteBuffer> index_stream;
|
|
|
|
MergeTreeData::DataPart::Index index_columns;
|
2013-04-24 10:31:32 +00:00
|
|
|
};
|
|
|
|
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-16 15:00:33 +00:00
|
|
|
/// Writes only those columns that are in `block`
|
2014-03-04 11:30:50 +00:00
|
|
|
class MergedColumnOnlyOutputStream : public IMergedBlockOutputStream
|
|
|
|
{
|
|
|
|
public:
|
2017-04-01 07:20:54 +00:00
|
|
|
MergedColumnOnlyOutputStream(
|
|
|
|
MergeTreeData & storage_, String part_path_, bool sync_, CompressionMethod compression_method, bool skip_offsets_);
|
2014-03-27 17:30:04 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
void write(const Block & block) override;
|
|
|
|
void writeSuffix() override;
|
|
|
|
MergeTreeData::DataPart::Checksums writeSuffixAndGetChecksums();
|
2014-03-04 11:30:50 +00:00
|
|
|
|
|
|
|
private:
|
2017-04-01 07:20:54 +00:00
|
|
|
String part_path;
|
2014-03-04 11:30:50 +00:00
|
|
|
|
2017-04-01 07:20:54 +00:00
|
|
|
bool initialized = false;
|
|
|
|
bool sync;
|
|
|
|
bool skip_offsets;
|
2014-03-04 11:30:50 +00:00
|
|
|
};
|
2014-06-26 00:58:14 +00:00
|
|
|
|
2013-09-26 19:16:43 +00:00
|
|
|
}
|