LZ4 - initial commit

src/IO/CompressionMethod.cpp

@@ -10,6 +10,8 @@
 #include <IO/ZlibInflatingReadBuffer.h>
 #include <IO/ZstdDeflatingWriteBuffer.h>
 #include <IO/ZstdInflatingReadBuffer.h>
+#include <IO/Lz4DeflatingWriteBuffer.h>
+#include <IO/Lz4InflatingReadBuffer.h>
 
 #if !defined(ARCADIA_BUILD)
 #    include <Common/config.h>
@@ -40,6 +42,8 @@ std::string toContentEncodingName(CompressionMethod method)
             return "xz";
         case CompressionMethod::Zstd:
             return "zstd";
+        case CompressionMethod::Lz4:
+            return "lz4";
         case CompressionMethod::None:
             return "";
     }
@@ -69,11 +73,13 @@ CompressionMethod chooseCompressionMethod(const std::string & path, const std::s
         return CompressionMethod::Xz;
     if (method_str == "zstd" || method_str == "zst")
         return CompressionMethod::Zstd;
+    if (method_str == "lz4")
+        return CompressionMethod::Lz4;
     if (hint.empty() || hint == "auto" || hint == "none")
         return CompressionMethod::None;
 
     throw Exception(
-        "Unknown compression method " + hint + ". Only 'auto', 'none', 'gzip', 'deflate', 'br', 'xz', 'zstd' are supported as compression methods",
+        "Unknown compression method " + hint + ". Only 'auto', 'none', 'gzip', 'deflate', 'br', 'xz', 'zstd', 'lz4' are supported as compression methods",
         ErrorCodes::NOT_IMPLEMENTED);
 }
 
@@ -91,7 +97,8 @@ std::unique_ptr<ReadBuffer> wrapReadBufferWithCompressionMethod(
         return std::make_unique<LZMAInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
     if (method == CompressionMethod::Zstd)
         return std::make_unique<ZstdInflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
-
+    if (method == CompressionMethod::Lz4)
+        return std::make_unique<Lz4InflatingReadBuffer>(std::move(nested), buf_size, existing_memory, alignment);
     if (method == CompressionMethod::None)
         return nested;
 
@@ -115,6 +122,9 @@ std::unique_ptr<WriteBuffer> wrapWriteBufferWithCompressionMethod(
     if (method == CompressionMethod::Zstd)
         return std::make_unique<ZstdDeflatingWriteBuffer>(std::move(nested), level, buf_size, existing_memory, alignment);
 
+    if (method == CompressionMethod::Lz4)
+        return std::make_unique<Lz4DeflatingWriteBuffer>(std::move(nested), level, buf_size, existing_memory, alignment);
+
     if (method == CompressionMethod::None)
         return nested;
 

src/IO/CompressionMethod.h

@@ -31,7 +31,8 @@ enum class CompressionMethod
     /// Zstd compressor
     ///  This option corresponds to HTTP Content-Encoding: zstd
     Zstd,
-    Brotli
+    Brotli,
+    Lz4
 };
 
 /// How the compression method is named in HTTP.

src/IO/Lz4DeflatingWriteBuffer.cpp

@@ -0,0 +1,39 @@
+#include "Lz4DeflatingWriteBuffer.h"
+#include <Common/Exception.h>
+#include <Common/MemoryTracker.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int LZ4_ENCODER_FAILED;
+}
+
+Lz4DeflatingWriteBuffer::Lz4DeflatingWriteBuffer(
+    std::unique_ptr<WriteBuffer> out_, int  /*compression_level*/, size_t buf_size, char * existing_memory, size_t alignment)
+    : BufferWithOwnMemory<WriteBuffer>(buf_size, existing_memory, alignment), out(std::move(out_))
+{
+}
+
+Lz4DeflatingWriteBuffer::~Lz4DeflatingWriteBuffer()
+{
+    /// TODO: Implementation
+}
+
+void Lz4DeflatingWriteBuffer::nextImpl()
+{
+    /// TODO: Implementation
+}
+
+void Lz4DeflatingWriteBuffer::finish()
+{
+    /// TODO: Implementation
+}
+
+void Lz4DeflatingWriteBuffer::finishImpl()
+{
+    /// TODO: Implementation
+}
+
+}

src/IO/Lz4DeflatingWriteBuffer.h

@@ -0,0 +1,36 @@
+#pragma once
+
+#include <IO/BufferWithOwnMemory.h>
+#include <IO/CompressionMethod.h>
+#include <IO/WriteBuffer.h>
+
+
+namespace DB
+{
+
+/// Performs compression using lz4 library and writes compressed data to out_ WriteBuffer.
+class Lz4DeflatingWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
+{
+public:
+    Lz4DeflatingWriteBuffer(
+        std::unique_ptr<WriteBuffer> out_,
+        int compression_level,
+        size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
+        char * existing_memory = nullptr,
+        size_t alignment = 0);
+
+    void finalize() override { finish(); }
+
+    ~Lz4DeflatingWriteBuffer() override;
+
+private:
+    void nextImpl() override;
+
+    void finish();
+    void finishImpl();
+
+    std::unique_ptr<WriteBuffer> out;
+
+    /// TODO: Complete Implementation
+};
+}

src/IO/Lz4InflatingReadBuffer.cpp

@@ -0,0 +1,27 @@
+#include "Lz4InflatingReadBuffer.h"
+
+namespace DB
+{
+namespace ErrorCodes
+{
+extern const int LZ4_DECODER_FAILED;
+}
+
+Lz4InflatingReadBuffer::Lz4InflatingReadBuffer(std::unique_ptr<ReadBuffer> in_, size_t buf_size, char * existing_memory, size_t alignment)
+    : BufferWithOwnMemory<ReadBuffer>(buf_size, existing_memory, alignment), in(std::move(in_))
+{
+    /// TODO: Implementation
+}
+
+Lz4InflatingReadBuffer::~Lz4InflatingReadBuffer()
+{
+   /// TODO: Implementation
+}
+
+bool Lz4InflatingReadBuffer::nextImpl()
+{
+    /// TODO: Implementation
+    return true;
+}
+
+}

src/IO/Lz4InflatingReadBuffer.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <IO/BufferWithOwnMemory.h>
+#include <IO/CompressionMethod.h>
+#include <IO/ReadBuffer.h>
+
+
+namespace DB
+{
+namespace ErrorCodes
+{
+}
+
+class Lz4InflatingReadBuffer : public BufferWithOwnMemory<ReadBuffer>
+{
+public:
+    Lz4InflatingReadBuffer(
+        std::unique_ptr<ReadBuffer> in_,
+        size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE,
+        char * existing_memory = nullptr,
+        size_t alignment = 0);
+
+    ~Lz4InflatingReadBuffer() override;
+
+private:
+    bool nextImpl() override;
+
+    std::unique_ptr<ReadBuffer> in;
+};
+
+}

src/IO/ya.make

@@ -32,6 +32,8 @@ SRCS(
     LZMADeflatingWriteBuffer.cpp
     LZMAInflatingReadBuffer.cpp
     LimitReadBuffer.cpp
+    Lz4DeflatingWriteBuffer.cpp
+    Lz4InflatingReadBuffer.cpp
     MMapReadBufferFromFile.cpp
     MMapReadBufferFromFileDescriptor.cpp
     MMapReadBufferFromFileWithCache.cpp