From a3e999784b66f8c22e08cb45656ebdf236df2d46 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Sep 2020 14:24:10 +0300 Subject: [PATCH] Buildable code --- src/Compression/CompressionCodecDelta.cpp | 4 ++-- src/Compression/CompressionCodecDoubleDelta.cpp | 4 ++-- src/Compression/CompressionCodecGorilla.cpp | 4 ++-- src/Compression/CompressionCodecT64.cpp | 4 ++-- src/Compression/CompressionFactory.cpp | 13 +++++-------- src/Compression/CompressionFactory.h | 12 ++++++++++-- src/Compression/ICompressionCodec.h | 1 - src/Storages/ColumnsDescription.cpp | 2 +- src/Storages/ColumnsDescription.h | 2 +- .../MergeTree/MergeTreeDataPartWriterWide.cpp | 5 ++--- .../MergeTree/MergeTreeDataPartWriterWide.h | 1 - 11 files changed, 27 insertions(+), 25 deletions(-) diff --git a/src/Compression/CompressionCodecDelta.cpp b/src/Compression/CompressionCodecDelta.cpp index a10d2589576..f3d6953bd2a 100644 --- a/src/Compression/CompressionCodecDelta.cpp +++ b/src/Compression/CompressionCodecDelta.cpp @@ -136,7 +136,7 @@ void CompressionCodecDelta::doDecompressData(const char * source, UInt32 source_ namespace { -UInt8 getDeltaBytesSize(DataTypePtr column_type) +UInt8 getDeltaBytesSize(const IDataType * column_type) { if (!column_type->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Delta is not applicable for {} because the data type is not of fixed size", @@ -155,7 +155,7 @@ UInt8 getDeltaBytesSize(DataTypePtr column_type) void registerCodecDelta(CompressionCodecFactory & factory) { UInt8 method_code = UInt8(CompressionMethodByte::Delta); - factory.registerCompressionCodecWithType("Delta", method_code, [&](const ASTPtr & arguments, DataTypePtr column_type) -> CompressionCodecPtr + factory.registerCompressionCodecWithType("Delta", method_code, [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { UInt8 delta_bytes_size = 0; diff --git a/src/Compression/CompressionCodecDoubleDelta.cpp b/src/Compression/CompressionCodecDoubleDelta.cpp index 96fd29fe356..6895a80264d 100644 --- a/src/Compression/CompressionCodecDoubleDelta.cpp +++ b/src/Compression/CompressionCodecDoubleDelta.cpp @@ -307,7 +307,7 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest) } } -UInt8 getDataBytesSize(DataTypePtr column_type) +UInt8 getDataBytesSize(const IDataType * column_type) { if (!column_type->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec DoubleDelta is not applicable for {} because the data type is not of fixed size", @@ -413,7 +413,7 @@ void registerCodecDoubleDelta(CompressionCodecFactory & factory) { UInt8 method_code = UInt8(CompressionMethodByte::DoubleDelta); factory.registerCompressionCodecWithType("DoubleDelta", method_code, - [&](const ASTPtr & arguments, DataTypePtr column_type) -> CompressionCodecPtr + [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { if (arguments) throw Exception("Codec DoubleDelta does not accept any arguments", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Compression/CompressionCodecGorilla.cpp b/src/Compression/CompressionCodecGorilla.cpp index d739623a94b..582a9f41874 100644 --- a/src/Compression/CompressionCodecGorilla.cpp +++ b/src/Compression/CompressionCodecGorilla.cpp @@ -222,7 +222,7 @@ void decompressDataForType(const char * source, UInt32 source_size, char * dest) } } -UInt8 getDataBytesSize(DataTypePtr column_type) +UInt8 getDataBytesSize(const IDataType * column_type) { if (!column_type->isValueUnambiguouslyRepresentedInFixedSizeContiguousMemoryRegion()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Codec Gorilla is not applicable for {} because the data type is not of fixed size", @@ -329,7 +329,7 @@ void registerCodecGorilla(CompressionCodecFactory & factory) { UInt8 method_code = UInt8(CompressionMethodByte::Gorilla); factory.registerCompressionCodecWithType("Gorilla", method_code, - [&](const ASTPtr & arguments, DataTypePtr column_type) -> CompressionCodecPtr + [&](const ASTPtr & arguments, const IDataType * column_type) -> CompressionCodecPtr { if (arguments) throw Exception("Codec Gorilla does not accept any arguments", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Compression/CompressionCodecT64.cpp b/src/Compression/CompressionCodecT64.cpp index f081652f613..095610af615 100644 --- a/src/Compression/CompressionCodecT64.cpp +++ b/src/Compression/CompressionCodecT64.cpp @@ -136,7 +136,7 @@ TypeIndex baseType(TypeIndex type_idx) return TypeIndex::Nothing; } -TypeIndex typeIdx(const DataTypePtr & data_type) +TypeIndex typeIdx(const IDataType * data_type) { if (!data_type) return TypeIndex::Nothing; @@ -656,7 +656,7 @@ void CompressionCodecT64::updateHash(SipHash & hash) const void registerCodecT64(CompressionCodecFactory & factory) { - auto reg_func = [&](const ASTPtr & arguments, DataTypePtr type) -> CompressionCodecPtr + auto reg_func = [&](const ASTPtr & arguments, const IDataType * type) -> CompressionCodecPtr { Variant variant = Variant::Byte; diff --git a/src/Compression/CompressionFactory.cpp b/src/Compression/CompressionFactory.cpp index cebeee7c5a8..0f511aec96d 100644 --- a/src/Compression/CompressionFactory.cpp +++ b/src/Compression/CompressionFactory.cpp @@ -57,7 +57,7 @@ void CompressionCodecFactory::validateCodec(const String & family_name, std::opt } } -ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr & ast, DataTypePtr column_type, bool sanity_check) const +ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check) const { if (const auto * func = ast->as()) { @@ -140,16 +140,13 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(const ASTPtr " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS); } - std::shared_ptr result = std::make_shared(); - result->name = "CODEC"; - result->arguments = codecs_descriptions; - return result; + return ast; } throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC); } -CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, DataTypePtr column_type, CompressionCodecPtr current_default) const +CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default) const { if (current_default == nullptr) current_default = default_codec; @@ -203,7 +200,7 @@ CompressionCodecPtr CompressionCodecFactory::get(const uint8_t byte_code) const } -CompressionCodecPtr CompressionCodecFactory::getImpl(const String & family_name, const ASTPtr & arguments, DataTypePtr column_type) const +CompressionCodecPtr CompressionCodecFactory::getImpl(const String & family_name, const ASTPtr & arguments, const IDataType * column_type) const { if (family_name == "Multiple") throw Exception("Codec Multiple cannot be specified directly", ErrorCodes::UNKNOWN_CODEC); @@ -235,7 +232,7 @@ void CompressionCodecFactory::registerCompressionCodecWithType( void CompressionCodecFactory::registerCompressionCodec(const String & family_name, std::optional byte_code, Creator creator) { - registerCompressionCodecWithType(family_name, byte_code, [family_name, creator](const ASTPtr & ast, DataTypePtr /* data_type */) + registerCompressionCodecWithType(family_name, byte_code, [family_name, creator](const ASTPtr & ast, const IDataType * /* data_type */) { return creator(ast); }); diff --git a/src/Compression/CompressionFactory.h b/src/Compression/CompressionFactory.h index 8dae34389b0..784fb26ebbc 100644 --- a/src/Compression/CompressionFactory.h +++ b/src/Compression/CompressionFactory.h @@ -26,7 +26,7 @@ class CompressionCodecFactory final : private boost::noncopyable { protected: using Creator = std::function; - using CreatorWithType = std::function; + using CreatorWithType = std::function; using SimpleCreator = std::function; using CompressionCodecsDictionary = std::unordered_map; using CompressionCodecsCodeDictionary = std::unordered_map; @@ -39,6 +39,10 @@ public: /// Validate codecs AST specified by user and parses codecs description (substitute default parameters) ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const IDataType * column_type, bool sanity_check) const; + ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check) const + { + return validateCodecAndGetPreprocessedAST(ast, column_type.get(), sanity_check); + } /// Validate codecs AST specified by user void validateCodec(const String & family_name, std::optional level, bool sanity_check) const; @@ -49,6 +53,10 @@ public: /// codec, which can be alias to current default codec, which can be changed /// in runtime. CompressionCodecPtr get(const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default = nullptr) const; + CompressionCodecPtr get(const ASTPtr & ast, const DataTypePtr & column_type, CompressionCodecPtr current_default = nullptr) const + { + return get(ast, column_type.get(), current_default); + } /// Get codec by method byte (no params available) CompressionCodecPtr get(const uint8_t byte_code) const; @@ -65,7 +73,7 @@ public: void registerSimpleCompressionCodec(const String & family_name, std::optional byte_code, SimpleCreator creator); protected: - CompressionCodecPtr getImpl(const String & family_name, const ASTPtr & arguments, DataTypePtr column_type) const; + CompressionCodecPtr getImpl(const String & family_name, const ASTPtr & arguments, const IDataType * column_type) const; private: CompressionCodecsDictionary family_name_with_codec; diff --git a/src/Compression/ICompressionCodec.h b/src/Compression/ICompressionCodec.h index f3371d6d4b9..ff2197dbe2b 100644 --- a/src/Compression/ICompressionCodec.h +++ b/src/Compression/ICompressionCodec.h @@ -17,7 +17,6 @@ using CompressionCodecPtr = std::shared_ptr; using Codecs = std::vector; class IDataType; -using DataTypePtr = std::shared_ptr; /** * Represents interface for compression codecs like LZ4, ZSTD, etc. diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp index 84d1ffb4602..48cde6b6aa9 100644 --- a/src/Storages/ColumnsDescription.cpp +++ b/src/Storages/ColumnsDescription.cpp @@ -426,7 +426,7 @@ CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_ return getCodecOrDefault(column_name, CompressionCodecFactory::instance().getDefaultCodec()); } -ASTPtr ColumnsDescription::getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec) +ASTPtr ColumnsDescription::getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec) const { const auto it = columns.get<1>().find(column_name); diff --git a/src/Storages/ColumnsDescription.h b/src/Storages/ColumnsDescription.h index 6e22a2fdffc..8d1009b7263 100644 --- a/src/Storages/ColumnsDescription.h +++ b/src/Storages/ColumnsDescription.h @@ -115,7 +115,7 @@ public: bool hasCompressionCodec(const String & column_name) const; CompressionCodecPtr getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const; CompressionCodecPtr getCodecOrDefault(const String & column_name) const; - ASTPtr getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec); + ASTPtr getCodecDescOrDefault(const String & column_name, CompressionCodecPtr default_codec) const; String toString() const; static ColumnsDescription parse(const String & str); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index a7b8a45c3bc..bce00715c08 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -29,14 +29,13 @@ MergeTreeDataPartWriterWide::MergeTreeDataPartWriterWide( { const auto & columns = metadata_snapshot->getColumns(); for (const auto & it : columns_list) - addStreams(it.name, *it.type, columns.getCodecOrDefaultDesc(it.name, default_codec), default_codec, settings.estimated_size); + addStreams(it.name, *it.type, columns.getCodecDescOrDefault(it.name, default_codec), settings.estimated_size); } void MergeTreeDataPartWriterWide::addStreams( const String & name, const IDataType & type, const ASTPtr & effective_codec_desc, - const CompressionCodecPtr & default_codec, size_t estimated_size) { IDataType::StreamCallback callback = [&] (const IDataType::SubstreamPath & substream_path, const IDataType & substream_type) @@ -53,7 +52,7 @@ void MergeTreeDataPartWriterWide::addStreams( data_part->volume->getDisk(), part_path + stream_name, DATA_FILE_EXTENSION, part_path + stream_name, marks_file_extension, - effective_codec, + compression_codec, settings.max_compress_block_size, estimated_size, settings.aio_threshold); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 678f8acd3e6..6dd3104c5b4 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -67,7 +67,6 @@ private: const String & name, const IDataType & type, const ASTPtr & effective_codec_desc, - const CompressionCodecPtr & default_codec, size_t estimated_size); SerializationStates serialization_states;