thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Avro format support zstd codec

Browse Source
This commit is contained in:
flynn 2024-01-15 09:11:27 +00:00
parent 3f10ad7573
commit 20dbe652ae
3 changed files with 9 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
contrib/boost-cmake/CMakeLists.txt
View File

@ -44,12 +44,14 @@ set (SRCS_IOSTREAMS
"${LIBRARY_DIR}/libs/iostreams/src/gzip.cpp"
"${LIBRARY_DIR}/libs/iostreams/src/mapped_file.cpp"
"${LIBRARY_DIR}/libs/iostreams/src/zlib.cpp"
"${LIBRARY_DIR}/libs/iostreams/src/zstd.cpp"
)
add_library (_boost_iostreams ${SRCS_IOSTREAMS})
add_library (boost::iostreams ALIAS _boost_iostreams)
target_include_directories (_boost_iostreams PRIVATE ${LIBRARY_DIR})
target_link_libraries (_boost_iostreams PRIVATE ch_contrib::zlib)
target_link_libraries (_boost_iostreams PRIVATE ch_contrib::zstd)
# program_options

2
src/Core/Settings.h
View File

@ -1052,7 +1052,7 @@ class IColumn;
M(Bool, output_format_parquet_parallel_encoding, true, "Do Parquet encoding in multiple threads. Requires output_format_parquet_use_custom_encoder.", 0) \
M(UInt64, output_format_parquet_data_page_size, 1024 * 1024, "Target page size in bytes, before compression.", 0) \
M(UInt64, output_format_parquet_batch_size, 1024, "Check page size every this many rows. Consider decreasing if you have columns with average values size above a few KBs.", 0) \
M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy'.", 0) \
M(String, output_format_avro_codec, "", "Compression codec used for output. Possible values: 'null', 'deflate', 'snappy', 'zstandard'.", 0) \
M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \
M(String, output_format_avro_string_column_pattern, "", "For Avro format: regexp of String columns to select as AVRO string.", 0) \
M(UInt64, output_format_avro_rows_in_file, 1, "Max rows in a file (if permitted by storage)", 0) \

9
src/Processors/Formats/Impl/AvroRowOutputFormat.cpp
View File

@ -527,9 +527,12 @@ static avro::Codec getCodec(const std::string & codec_name)
if (codec_name == "null") return avro::Codec::NULL_CODEC;
if (codec_name == "deflate") return avro::Codec::DEFLATE_CODEC;
#ifdef SNAPPY_CODEC_AVAILABLE
if (codec_name == "snappy") return avro::Codec::SNAPPY_CODEC;
#endif
if (codec_name == "zstandard")
return avro::Codec::ZSTD_CODEC;
# ifdef SNAPPY_CODEC_AVAILABLE
if (codec_name == "snappy")
return avro::Codec::SNAPPY_CODEC;
# endif
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Avro codec {} is not available", codec_name);
}