mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
LZ4 - initial commit
This commit is contained in:
parent
250f495456
commit
afad00b7d4
@ -10,6 +10,8 @@
|
|||||||
#include <IO/ZlibInflatingReadBuffer.h>
|
#include <IO/ZlibInflatingReadBuffer.h>
|
||||||
#include <IO/ZstdDeflatingWriteBuffer.h>
|
#include <IO/ZstdDeflatingWriteBuffer.h>
|
||||||
#include <IO/ZstdInflatingReadBuffer.h>
|
#include <IO/ZstdInflatingReadBuffer.h>
|
||||||
|
#include <IO/Lz4DeflatingWriteBuffer.h>
|
||||||
|
#include <IO/Lz4InflatingReadBuffer.h>
|
||||||
|
|
||||||
#if !defined(ARCADIA_BUILD)
|
#if !defined(ARCADIA_BUILD)
|
||||||
# include <Common/config.h>
|
# include <Common/config.h>
|
||||||
@ -40,6 +42,8 @@ std::string toContentEncodingName(CompressionMethod method)
|
|||||||
return "xz";
|
return "xz";
|
||||||
case CompressionMethod::Zstd:
|
case CompressionMethod::Zstd:
|
||||||
return "zstd";
|
return "zstd";
|
||||||
|
case CompressionMethod::Lz4:
|
||||||
|
return "lz4";
|
||||||
case CompressionMethod::None:
|
case CompressionMethod::None:
|
||||||
return "";
|
return "";
|
||||||
}
|
}
|
||||||
@ -69,11 +73,13 @@ CompressionMethod chooseCompressionMethod(const std::string & path, const std::s
|
|||||||
return CompressionMethod::Xz;
|
return CompressionMethod::Xz;
|
||||||
if (method_str == "zstd" || method_str == "zst")
|
if (method_str == "zstd" || method_str == "zst")
|
||||||
return CompressionMethod::Zstd;
|
return CompressionMethod::Zstd;
|
||||||
|
if (method_str == "lz4")
|
||||||
|
return CompressionMethod::Lz4;
|
||||||
if (hint.empty() || hint == "auto" || hint == "none")
|
if (hint.empty() || hint == "auto" || hint == "none")
|
||||||
return CompressionMethod::None;
|
return CompressionMethod::None;
|
||||||
|
|
||||||
throw Exception(
|
throw Exception(
|
||||||
"Unknown compression method " + hint + ". Only 'auto', 'none', 'gzip', 'deflate', 'br', 'xz', 'zstd' are supported as compression methods",
|
"Unknown compression method " + hint + ". Only 'auto', 'none', 'gzip', 'deflate', 'br', 'xz', 'zstd', 'lz4' are supported as compression methods",
|
||||||
ErrorCodes::NOT_IMPLEMENTED);
|
ErrorCodes::NOT_IMPLEMENTED);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -91,7 +97,8 @@ std::unique_ptr<ReadBuffer> wrapReadBufferWithCompressionMethod(
|
|||||||
return std::make_unique<LZMAInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
|
return std::make_unique<LZMAInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
|
||||||
if (method == CompressionMethod::Zstd)
|
if (method == CompressionMethod::Zstd)
|
||||||
return std::make_unique<ZstdInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
|
return std::make_unique<ZstdInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
|
||||||
|
if (method == CompressionMethod::Lz4)
|
||||||
|
return std::make_unique<Lz4InflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
|
||||||
if (method == CompressionMethod::None)
|
if (method == CompressionMethod::None)
|
||||||
return nested;
|
return nested;
|
||||||
|
|
||||||
@ -115,6 +122,9 @@ std::unique_ptr<WriteBuffer> wrapWriteBufferWithCompressionMethod(
|
|||||||
if (method == CompressionMethod::Zstd)
|
if (method == CompressionMethod::Zstd)
|
||||||
return std::make_unique<ZstdDeflatingWriteBuffer>(std::move(nested), level, buf_size, existing_memory, alignment);
|
return std::make_unique<ZstdDeflatingWriteBuffer>(std::move(nested), level, buf_size, existing_memory, alignment);
|
||||||
|
|
||||||
|
if (method == CompressionMethod::Lz4)
|
||||||
|
return std::make_unique<Lz4DeflatingWriteBuffer>(std::move(nested), level, buf_size, existing_memory, alignment);
|
||||||
|
|
||||||
if (method == CompressionMethod::None)
|
if (method == CompressionMethod::None)
|
||||||
return nested;
|
return nested;
|
||||||
|
|
||||||
|
@ -31,7 +31,8 @@ enum class CompressionMethod
|
|||||||
/// Zstd compressor
|
/// Zstd compressor
|
||||||
/// This option corresponds to HTTP Content-Encoding: zstd
|
/// This option corresponds to HTTP Content-Encoding: zstd
|
||||||
Zstd,
|
Zstd,
|
||||||
Brotli
|
Brotli,
|
||||||
|
Lz4
|
||||||
};
|
};
|
||||||
|
|
||||||
/// How the compression method is named in HTTP.
|
/// How the compression method is named in HTTP.
|
||||||
|
39
src/IO/Lz4DeflatingWriteBuffer.cpp
Normal file
39
src/IO/Lz4DeflatingWriteBuffer.cpp
Normal file
@ -0,0 +1,39 @@
|
|||||||
|
#include "Lz4DeflatingWriteBuffer.h"
|
||||||
|
#include <Common/Exception.h>
|
||||||
|
#include <Common/MemoryTracker.h>
|
||||||
|
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
namespace ErrorCodes
|
||||||
|
{
|
||||||
|
extern const int LZ4_ENCODER_FAILED;
|
||||||
|
}
|
||||||
|
|
||||||
|
Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer(
|
||||||
|
std::unique_ptr<WriteBuffer> out_, int /*compression_level*/, size_t buf_size, char * existing_memory, size_t alignment)
|
||||||
|
: BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_))
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
Lz4DeflatingWriteBuffer::~Lz4DeflatingWriteBuffer()
|
||||||
|
{
|
||||||
|
/// TODO: Implementation
|
||||||
|
}
|
||||||
|
|
||||||
|
void Lz4DeflatingWriteBuffer::nextImpl()
|
||||||
|
{
|
||||||
|
/// TODO: Implementation
|
||||||
|
}
|
||||||
|
|
||||||
|
void Lz4DeflatingWriteBuffer::finish()
|
||||||
|
{
|
||||||
|
/// TODO: Implementation
|
||||||
|
}
|
||||||
|
|
||||||
|
void Lz4DeflatingWriteBuffer::finishImpl()
|
||||||
|
{
|
||||||
|
/// TODO: Implementation
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
36
src/IO/Lz4DeflatingWriteBuffer.h
Normal file
36
src/IO/Lz4DeflatingWriteBuffer.h
Normal file
@ -0,0 +1,36 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <IO/BufferWithOwnMemory.h>
|
||||||
|
#include <IO/CompressionMethod.h>
|
||||||
|
#include <IO/WriteBuffer.h>
|
||||||
|
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
|
||||||
|
/// Performs compression using lz4 library and writes compressed data to out_ WriteBuffer.
|
||||||
|
class Lz4DeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
Lz4DeflatingWriteBuffer(
|
||||||
|
std::unique_ptr<WriteBuffer> out_,
|
||||||
|
int compression_level,
|
||||||
|
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||||
|
char * existing_memory = nullptr,
|
||||||
|
size_t alignment = 0);
|
||||||
|
|
||||||
|
void finalize() override { finish(); }
|
||||||
|
|
||||||
|
~Lz4DeflatingWriteBuffer() override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
void nextImpl() override;
|
||||||
|
|
||||||
|
void finish();
|
||||||
|
void finishImpl();
|
||||||
|
|
||||||
|
std::unique_ptr<WriteBuffer> out;
|
||||||
|
|
||||||
|
/// TODO: Complete Implementation
|
||||||
|
};
|
||||||
|
}
|
27
src/IO/Lz4InflatingReadBuffer.cpp
Normal file
27
src/IO/Lz4InflatingReadBuffer.cpp
Normal file
@ -0,0 +1,27 @@
|
|||||||
|
#include "Lz4InflatingReadBuffer.h"
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
namespace ErrorCodes
|
||||||
|
{
|
||||||
|
extern const int LZ4_DECODER_FAILED;
|
||||||
|
}
|
||||||
|
|
||||||
|
Lz4InflatingReadBuffer::Lz4InflatingReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
|
||||||
|
: BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_))
|
||||||
|
{
|
||||||
|
/// TODO: Implementation
|
||||||
|
}
|
||||||
|
|
||||||
|
Lz4InflatingReadBuffer::~Lz4InflatingReadBuffer()
|
||||||
|
{
|
||||||
|
/// TODO: Implementation
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Lz4InflatingReadBuffer::nextImpl()
|
||||||
|
{
|
||||||
|
/// TODO: Implementation
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
31
src/IO/Lz4InflatingReadBuffer.h
Normal file
31
src/IO/Lz4InflatingReadBuffer.h
Normal file
@ -0,0 +1,31 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <IO/BufferWithOwnMemory.h>
|
||||||
|
#include <IO/CompressionMethod.h>
|
||||||
|
#include <IO/ReadBuffer.h>
|
||||||
|
|
||||||
|
|
||||||
|
namespace DB
|
||||||
|
{
|
||||||
|
namespace ErrorCodes
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
class Lz4InflatingReadBuffer : public BufferWithOwnMemory<ReadBuffer>
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
Lz4InflatingReadBuffer(
|
||||||
|
std::unique_ptr<ReadBuffer> in_,
|
||||||
|
size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
|
||||||
|
char * existing_memory = nullptr,
|
||||||
|
size_t alignment = 0);
|
||||||
|
|
||||||
|
~Lz4InflatingReadBuffer() override;
|
||||||
|
|
||||||
|
private:
|
||||||
|
bool nextImpl() override;
|
||||||
|
|
||||||
|
std::unique_ptr<ReadBuffer> in;
|
||||||
|
};
|
||||||
|
|
||||||
|
}
|
@ -32,6 +32,8 @@ SRCS(
|
|||||||
LZMADeflatingWriteBuffer.cpp
|
LZMADeflatingWriteBuffer.cpp
|
||||||
LZMAInflatingReadBuffer.cpp
|
LZMAInflatingReadBuffer.cpp
|
||||||
LimitReadBuffer.cpp
|
LimitReadBuffer.cpp
|
||||||
|
Lz4DeflatingWriteBuffer.cpp
|
||||||
|
Lz4InflatingReadBuffer.cpp
|
||||||
MMapReadBufferFromFile.cpp
|
MMapReadBufferFromFile.cpp
|
||||||
MMapReadBufferFromFileDescriptor.cpp
|
MMapReadBufferFromFileDescriptor.cpp
|
||||||
MMapReadBufferFromFileWithCache.cpp
|
MMapReadBufferFromFileWithCache.cpp
|
||||||
|
Loading…
Reference in New Issue
Block a user