Fix cyclic lib depend (make clickhouse_compression lib)

dbms/CMakeLists.txt

@@ -64,7 +64,6 @@ set(dbms_sources)
 include(../cmake/dbms_glob_sources.cmake)
 
 add_headers_and_sources(clickhouse_common_io src/Common)
-add_headers_and_sources(clickhouse_common_io src/Compression)
 add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
 add_headers_and_sources(clickhouse_common_io src/IO)
 
@@ -159,7 +158,6 @@ target_link_libraries (clickhouse_common_io
         PUBLIC
     common
         PRIVATE
-    clickhouse_parsers
     string_utils
     widechar_width
     ${LINK_LIBRARIES_ONLY_ON_X86_64}
@@ -186,6 +184,8 @@ target_link_libraries (clickhouse_common_io
 )
 
 target_link_libraries (dbms
+        PUBLIC
+    clickhouse_compression
         PRIVATE
     clickhouse_parsers
     clickhouse_common_config

dbms/programs/compressor/CMakeLists.txt

@@ -1,5 +1,5 @@
 add_library (clickhouse-compressor-lib ${LINK_MODE} Compressor.cpp)
-target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries (clickhouse-compressor-lib PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
 
 if (CLICKHOUSE_SPLIT_BINARY)
     # Also in utils

dbms/src/Common/tests/CMakeLists.txt

@@ -20,10 +20,10 @@ add_executable (small_table small_table.cpp)
 target_link_libraries (small_table PRIVATE clickhouse_common_io)
 
 add_executable (parallel_aggregation parallel_aggregation.cpp)
-target_link_libraries (parallel_aggregation PRIVATE clickhouse_common_io)
+target_link_libraries (parallel_aggregation PRIVATE clickhouse_compression clickhouse_common_io)
 
 add_executable (parallel_aggregation2 parallel_aggregation2.cpp)
-target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_common_io)
+target_link_libraries (parallel_aggregation2 PRIVATE clickhouse_compression clickhouse_common_io)
 
 add_executable (int_hashes_perf int_hashes_perf.cpp AvalancheTest.cpp Random.cpp)
 target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io)
@@ -42,7 +42,7 @@ add_executable (shell_command_test shell_command_test.cpp)
 target_link_libraries (shell_command_test PRIVATE clickhouse_common_io)
 
 add_executable (arena_with_free_lists arena_with_free_lists.cpp)
-target_link_libraries (arena_with_free_lists PRIVATE clickhouse_common_io)
+target_link_libraries (arena_with_free_lists PRIVATE clickhouse_compression clickhouse_common_io)
 
 add_executable (pod_array pod_array.cpp)
 target_link_libraries (pod_array PRIVATE clickhouse_common_io)
@@ -61,7 +61,7 @@ target_link_libraries (space_saving PRIVATE clickhouse_common_io)
 
 add_executable (integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
 target_include_directories (integer_hash_tables_and_hashes SYSTEM BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR})
-target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_common_io)
+target_link_libraries (integer_hash_tables_and_hashes PRIVATE clickhouse_compression clickhouse_common_io)
 
 add_executable (allocator allocator.cpp)
 target_link_libraries (allocator PRIVATE clickhouse_common_io)

dbms/src/Compression/CMakeLists.txt

@@ -0,0 +1,9 @@
+include(${ClickHouse_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
+add_headers_and_sources(clickhouse_compression .)
+add_library(clickhouse_compression ${LINK_MODE} ${clickhouse_compression_headers} ${clickhouse_compression_sources})
+target_link_libraries(clickhouse_compression PRIVATE clickhouse_parsers clickhouse_common_io ${ZSTD_LIBRARY})
+target_include_directories(clickhouse_compression PUBLIC ${DBMS_INCLUDE_DIR})
+
+#if(ENABLE_TESTS)
+#    add_subdirectory(tests)
+#endif()

dbms/src/Compression/CachedCompressedReadBuffer.cpp

@@ -1,9 +1,10 @@
+#include "CachedCompressedReadBuffer.h"
+
 #include <IO/createReadBufferFromFileBase.h>
-#include <IO/CachedCompressedReadBuffer.h>
+#include "CachedCompressedReadBuffer.h"
 #include <IO/WriteHelpers.h>
 #include <Compression/CompressionInfo.h>
 #include <IO/LZ4_decompress_faster.h>
-#include "CachedCompressedReadBuffer.h"
 
 
 namespace DB

dbms/src/Compression/CachedCompressedReadBuffer.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <memory>
+#include <time.h>
+#include <IO/ReadBufferFromFileBase.h>
+#include "CompressedReadBufferBase.h"
+#include <IO/UncompressedCache.h>
+#include <port/clock.h>
+
+
+namespace DB
+{
+
+
+/** A buffer for reading from a compressed file using the cache of decompressed blocks.
+  * The external cache is passed as an argument to the constructor.
+  * Allows you to increase performance in cases where the same blocks are often read.
+  * Disadvantages:
+  * - in case you need to read a lot of data in a row, but of them only a part is cached, you have to do seek-and.
+  */
+class CachedCompressedReadBuffer : public CompressedReadBufferBase, public ReadBuffer
+{
+private:
+    const std::string path;
+    UncompressedCache * cache;
+    size_t buf_size;
+    size_t estimated_size;
+    size_t aio_threshold;
+
+    std::unique_ptr<ReadBufferFromFileBase> file_in;
+    size_t file_pos;
+
+    /// A piece of data from the cache, or a piece of read data that we put into the cache.
+    UncompressedCache::MappedPtr owned_cell;
+
+    void initInput();
+    bool nextImpl() override;
+
+    /// Passed into file_in.
+    ReadBufferFromFileBase::ProfileCallback profile_callback;
+    clockid_t clock_type {};
+
+public:
+    CachedCompressedReadBuffer(
+        const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_,
+        size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE);
+
+
+    void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
+
+    void setProfileCallback(const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE)
+    {
+        profile_callback = profile_callback_;
+        clock_type = clock_type_;
+    }
+};
+
+}

dbms/src/Compression/CompressedReadBuffer.cpp

@@ -1,4 +1,4 @@
-#include <IO/CompressedReadBuffer.h>
+#include "CompressedReadBuffer.h"
 #include <Compression/CompressionInfo.h>
 #include <IO/LZ4_decompress_faster.h>
 

dbms/src/Compression/CompressedReadBuffer.h

@@ -0,0 +1,33 @@
+#pragma once
+
+#include "CompressedReadBufferBase.h"
+#include <IO/BufferWithOwnMemory.h>
+#include <IO/ReadBuffer.h>
+
+
+namespace DB
+{
+
+class CompressedReadBuffer : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
+{
+private:
+    size_t size_compressed = 0;
+
+    bool nextImpl() override;
+
+public:
+    CompressedReadBuffer(ReadBuffer & in_)
+        : CompressedReadBufferBase(&in_), BufferWithOwnMemory<ReadBuffer>(0)
+    {
+    }
+
+    size_t readBig(char * to, size_t n) override;
+
+    /// The compressed size of the current block.
+    size_t getSizeCompressed() const
+    {
+        return size_compressed;
+    }
+};
+
+}

dbms/src/Compression/CompressedReadBufferBase.cpp

@@ -1,4 +1,4 @@
-#include <IO/CompressedReadBufferBase.h>
+#include "CompressedReadBufferBase.h"
 
 #include <vector>
 

dbms/src/Compression/CompressedReadBufferFromFile.cpp

@@ -1,4 +1,5 @@
-#include <IO/CompressedReadBufferFromFile.h>
+#include "CompressedReadBufferFromFile.h"
+
 #include <IO/createReadBufferFromFileBase.h>
 #include <IO/WriteHelpers.h>
 #include <Compression/CompressionInfo.h>

dbms/src/Compression/CompressedReadBufferFromFile.h

@@ -0,0 +1,45 @@
+#pragma once
+
+#include "CompressedReadBufferBase.h"
+#include <IO/ReadBufferFromFileBase.h>
+#include <time.h>
+#include <memory>
+#include <port/clock.h>
+
+
+namespace DB
+{
+
+
+/// Unlike CompressedReadBuffer, it can do seek.
+class CompressedReadBufferFromFile : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
+{
+private:
+      /** At any time, one of two things is true:
+      * a) size_compressed = 0
+      * b)
+      *  - `working_buffer` contains the entire block.
+      *  - `file_in` points to the end of this block.
+      *  - `size_compressed` contains the compressed size of this block.
+      */
+    std::unique_ptr<ReadBufferFromFileBase> p_file_in;
+    ReadBufferFromFileBase & file_in;
+    size_t size_compressed = 0;
+
+    bool nextImpl() override;
+
+public:
+    CompressedReadBufferFromFile(
+        const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
+
+    void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
+
+    size_t readBig(char * to, size_t n) override;
+
+    void setProfileCallback(const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE)
+    {
+        file_in.setProfileCallback(profile_callback_, clock_type_);
+    }
+};
+
+}

dbms/src/Compression/CompressedWriteBuffer.cpp

@@ -8,7 +8,7 @@
 #include <common/unaligned.h>
 #include <Core/Types.h>
 
-#include <IO/CompressedWriteBuffer.h>
+#include "CompressedWriteBuffer.h"
 #include <Compression/CompressionFactory.h>
 
 

dbms/src/Compression/CompressedWriteBuffer.h

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <memory>
+
+#include <Common/PODArray.h>
+
+#include <IO/WriteBuffer.h>
+#include <IO/BufferWithOwnMemory.h>
+#include <Compression/ICompressionCodec.h>
+#include <Compression/CompressionFactory.h>
+
+
+namespace DB
+{
+
+class CompressedWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
+{
+private:
+    WriteBuffer & out;
+    CompressionCodecPtr codec;
+
+    PODArray<char> compressed_buffer;
+
+    void nextImpl() override;
+
+public:
+    CompressedWriteBuffer(
+        WriteBuffer & out_,
+        CompressionCodecPtr codec_ = CompressionCodecFactory::instance().getDefaultCodec(),
+        size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
+
+    /// The amount of compressed data
+    size_t getCompressedBytes()
+    {
+        nextIfAtEnd();
+        return out.count();
+    }
+
+    /// How many uncompressed bytes were written to the buffer
+    size_t getUncompressedBytes()
+    {
+        return count();
+    }
+
+    /// How many bytes are in the buffer (not yet compressed)
+    size_t getRemainingBytes()
+    {
+        nextIfAtEnd();
+        return offset();
+    }
+
+    ~CompressedWriteBuffer() override;
+};
+
+}

dbms/src/IO/CachedCompressedReadBuffer.h

@@ -1,58 +1 @@
-#pragma once
-
-#include <memory>
-#include <time.h>
-#include <IO/ReadBufferFromFileBase.h>
-#include <IO/CompressedReadBufferBase.h>
-#include <IO/UncompressedCache.h>
-#include <port/clock.h>
-
-
-namespace DB
-{
-
-
-/** A buffer for reading from a compressed file using the cache of decompressed blocks.
-  * The external cache is passed as an argument to the constructor.
-  * Allows you to increase performance in cases where the same blocks are often read.
-  * Disadvantages:
-  * - in case you need to read a lot of data in a row, but of them only a part is cached, you have to do seek-and.
-  */
-class CachedCompressedReadBuffer : public CompressedReadBufferBase, public ReadBuffer
-{
-private:
-    const std::string path;
-    UncompressedCache * cache;
-    size_t buf_size;
-    size_t estimated_size;
-    size_t aio_threshold;
-
-    std::unique_ptr<ReadBufferFromFileBase> file_in;
-    size_t file_pos;
-
-    /// A piece of data from the cache, or a piece of read data that we put into the cache.
-    UncompressedCache::MappedPtr owned_cell;
-
-    void initInput();
-    bool nextImpl() override;
-
-    /// Passed into file_in.
-    ReadBufferFromFileBase::ProfileCallback profile_callback;
-    clockid_t clock_type {};
-
-public:
-    CachedCompressedReadBuffer(
-        const std::string & path_, UncompressedCache * cache_, size_t estimated_size_, size_t aio_threshold_,
-        size_t buf_size_ = DBMS_DEFAULT_BUFFER_SIZE);
-
-
-    void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
-
-    void setProfileCallback(const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE)
-    {
-        profile_callback = profile_callback_;
-        clock_type = clock_type_;
-    }
-};
-
-}
+#include <Compression/CachedCompressedReadBuffer.h>

dbms/src/IO/CompressedReadBuffer.h

@@ -1,33 +1 @@
-#pragma once
-
-#include <IO/CompressedReadBufferBase.h>
-#include <IO/BufferWithOwnMemory.h>
-#include <IO/ReadBuffer.h>
-
-
-namespace DB
-{
-
-class CompressedReadBuffer : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
-{
-private:
-    size_t size_compressed = 0;
-
-    bool nextImpl() override;
-
-public:
-    CompressedReadBuffer(ReadBuffer & in_)
-        : CompressedReadBufferBase(&in_), BufferWithOwnMemory<ReadBuffer>(0)
-    {
-    }
-
-    size_t readBig(char * to, size_t n) override;
-
-    /// The compressed size of the current block.
-    size_t getSizeCompressed() const
-    {
-        return size_compressed;
-    }
-};
-
-}
+#include <Compression/CompressedReadBuffer.h>

dbms/src/IO/CompressedReadBufferFromFile.h

@@ -1,45 +1 @@
-#pragma once
-
-#include <IO/CompressedReadBufferBase.h>
-#include <IO/ReadBufferFromFileBase.h>
-#include <time.h>
-#include <memory>
-#include <port/clock.h>
-
-
-namespace DB
-{
-
-
-/// Unlike CompressedReadBuffer, it can do seek.
-class CompressedReadBufferFromFile : public CompressedReadBufferBase, public BufferWithOwnMemory<ReadBuffer>
-{
-private:
-      /** At any time, one of two things is true:
-      * a) size_compressed = 0
-      * b)
-      *  - `working_buffer` contains the entire block.
-      *  - `file_in` points to the end of this block.
-      *  - `size_compressed` contains the compressed size of this block.
-      */
-    std::unique_ptr<ReadBufferFromFileBase> p_file_in;
-    ReadBufferFromFileBase & file_in;
-    size_t size_compressed = 0;
-
-    bool nextImpl() override;
-
-public:
-    CompressedReadBufferFromFile(
-        const std::string & path, size_t estimated_size, size_t aio_threshold, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
-
-    void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block);
-
-    size_t readBig(char * to, size_t n) override;
-
-    void setProfileCallback(const ReadBufferFromFileBase::ProfileCallback & profile_callback_, clockid_t clock_type_ = CLOCK_MONOTONIC_COARSE)
-    {
-        file_in.setProfileCallback(profile_callback_, clock_type_);
-    }
-};
-
-}
+#include <Compression/CompressedReadBufferFromFile.h>

dbms/src/IO/CompressedWriteBuffer.h

@@ -1,55 +1 @@
-#pragma once
-
-#include <memory>
-
-#include <Common/PODArray.h>
-
-#include <IO/WriteBuffer.h>
-#include <IO/BufferWithOwnMemory.h>
-#include <Compression/ICompressionCodec.h>
-#include <Compression/CompressionFactory.h>
-
-
-namespace DB
-{
-
-class CompressedWriteBuffer : public BufferWithOwnMemory<WriteBuffer>
-{
-private:
-    WriteBuffer & out;
-    CompressionCodecPtr codec;
-
-    PODArray<char> compressed_buffer;
-
-    void nextImpl() override;
-
-public:
-    CompressedWriteBuffer(
-        WriteBuffer & out_,
-        CompressionCodecPtr codec_ = CompressionCodecFactory::instance().getDefaultCodec(),
-        size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE);
-
-    /// The amount of compressed data
-    size_t getCompressedBytes()
-    {
-        nextIfAtEnd();
-        return out.count();
-    }
-
-    /// How many uncompressed bytes were written to the buffer
-    size_t getUncompressedBytes()
-    {
-        return count();
-    }
-
-    /// How many bytes are in the buffer (not yet compressed)
-    size_t getRemainingBytes()
-    {
-        nextIfAtEnd();
-        return offset();
-    }
-
-    ~CompressedWriteBuffer() override;
-};
-
-}
+#include <Compression/CompressedWriteBuffer.h>

dbms/src/IO/tests/CMakeLists.txt

@@ -19,8 +19,9 @@ target_link_libraries (valid_utf8_perf PRIVATE clickhouse_common_io)
 add_executable (valid_utf8 valid_utf8.cpp)
 target_link_libraries (valid_utf8 PRIVATE clickhouse_common_io)
 
+# TODO move to Compression
 add_executable (compressed_buffer compressed_buffer.cpp)
-target_link_libraries (compressed_buffer PRIVATE clickhouse_common_io)
+target_link_libraries (compressed_buffer PRIVATE clickhouse_compression clickhouse_common_io)
 
 add_executable (var_uint var_uint.cpp)
 target_link_libraries (var_uint PRIVATE clickhouse_common_io)
@@ -29,7 +30,7 @@ add_executable (read_escaped_string read_escaped_string.cpp)
 target_link_libraries (read_escaped_string PRIVATE clickhouse_common_io)
 
 add_executable (async_write async_write.cpp)
-target_link_libraries (async_write PRIVATE clickhouse_common_io)
+target_link_libraries (async_write PRIVATE clickhouse_compression clickhouse_common_io)
 
 add_executable (parse_int_perf parse_int_perf.cpp)
 target_link_libraries (parse_int_perf PRIVATE clickhouse_common_io)

utils/check-marks/CMakeLists.txt

@@ -1,2 +1,2 @@
 add_executable (check-marks main.cpp)
-target_link_libraries(check-marks PRIVATE clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})
+target_link_libraries(check-marks PRIVATE clickhouse_compression clickhouse_common_io ${Boost_PROGRAM_OPTIONS_LIBRARY})