mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-18 04:12:19 +00:00
Check for suspicious codecs #4966
This commit is contained in:
parent
884c2aa631
commit
99c18c5a09
@ -63,6 +63,8 @@ void checkAndWriteHeader(DB::ReadBuffer & in, DB::WriteBuffer & out)
|
|||||||
|
|
||||||
int mainEntryClickHouseCompressor(int argc, char ** argv)
|
int mainEntryClickHouseCompressor(int argc, char ** argv)
|
||||||
{
|
{
|
||||||
|
using namespace DB;
|
||||||
|
|
||||||
boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
|
boost::program_options::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
|
||||||
desc.add_options()
|
desc.add_options()
|
||||||
("help,h", "produce help message")
|
("help,h", "produce help message")
|
||||||
@ -99,10 +101,10 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
|
|||||||
codecs = options["codec"].as<std::vector<std::string>>();
|
codecs = options["codec"].as<std::vector<std::string>>();
|
||||||
|
|
||||||
if ((use_lz4hc || use_zstd || use_none) && !codecs.empty())
|
if ((use_lz4hc || use_zstd || use_none) && !codecs.empty())
|
||||||
throw DB::Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", DB::ErrorCodes::BAD_ARGUMENTS);
|
throw Exception("Wrong options, codec flags like --zstd and --codec options are mutually exclusive", ErrorCodes::BAD_ARGUMENTS);
|
||||||
|
|
||||||
if (!codecs.empty() && options.count("level"))
|
if (!codecs.empty() && options.count("level"))
|
||||||
throw DB::Exception("Wrong options, --level is not compatible with --codec list", DB::ErrorCodes::BAD_ARGUMENTS);
|
throw Exception("Wrong options, --level is not compatible with --codec list", ErrorCodes::BAD_ARGUMENTS);
|
||||||
|
|
||||||
std::string method_family = "LZ4";
|
std::string method_family = "LZ4";
|
||||||
|
|
||||||
@ -117,22 +119,21 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
|
|||||||
if (options.count("level"))
|
if (options.count("level"))
|
||||||
level = options["level"].as<int>();
|
level = options["level"].as<int>();
|
||||||
|
|
||||||
|
CompressionCodecPtr codec;
|
||||||
DB::CompressionCodecPtr codec;
|
|
||||||
if (!codecs.empty())
|
if (!codecs.empty())
|
||||||
{
|
{
|
||||||
DB::ParserCodec codec_parser;
|
ParserCodec codec_parser;
|
||||||
|
|
||||||
std::string codecs_line = boost::algorithm::join(codecs, ",");
|
std::string codecs_line = boost::algorithm::join(codecs, ",");
|
||||||
auto ast = DB::parseQuery(codec_parser, "(" + codecs_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
auto ast = parseQuery(codec_parser, "(" + codecs_line + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH);
|
||||||
codec = DB::CompressionCodecFactory::instance().get(ast, nullptr);
|
codec = CompressionCodecFactory::instance().get(ast, nullptr, false);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
codec = DB::CompressionCodecFactory::instance().get(method_family, level);
|
codec = CompressionCodecFactory::instance().get(method_family, level, false);
|
||||||
|
|
||||||
|
|
||||||
DB::ReadBufferFromFileDescriptor rb(STDIN_FILENO);
|
ReadBufferFromFileDescriptor rb(STDIN_FILENO);
|
||||||
DB::WriteBufferFromFileDescriptor wb(STDOUT_FILENO);
|
WriteBufferFromFileDescriptor wb(STDOUT_FILENO);
|
||||||
|
|
||||||
if (stat_mode)
|
if (stat_mode)
|
||||||
{
|
{
|
||||||
@ -142,20 +143,20 @@ int mainEntryClickHouseCompressor(int argc, char ** argv)
|
|||||||
else if (decompress)
|
else if (decompress)
|
||||||
{
|
{
|
||||||
/// Decompression
|
/// Decompression
|
||||||
DB::CompressedReadBuffer from(rb);
|
CompressedReadBuffer from(rb);
|
||||||
DB::copyData(from, wb);
|
copyData(from, wb);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
/// Compression
|
/// Compression
|
||||||
DB::CompressedWriteBuffer to(wb, codec, block_size);
|
CompressedWriteBuffer to(wb, codec, block_size);
|
||||||
DB::copyData(rb, to);
|
copyData(rb, to);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (...)
|
catch (...)
|
||||||
{
|
{
|
||||||
std::cerr << DB::getCurrentExceptionMessage(true);
|
std::cerr << getCurrentExceptionMessage(true);
|
||||||
return DB::getCurrentExceptionCode();
|
return getCurrentExceptionCode();
|
||||||
}
|
}
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -1066,14 +1066,16 @@ void TCPHandler::initBlockOutput(const Block & block)
|
|||||||
{
|
{
|
||||||
if (!state.maybe_compressed_out)
|
if (!state.maybe_compressed_out)
|
||||||
{
|
{
|
||||||
std::string method = Poco::toUpper(query_context->getSettingsRef().network_compression_method.toString());
|
const Settings & query_settings = query_context->getSettingsRef();
|
||||||
|
|
||||||
|
std::string method = Poco::toUpper(query_settings.network_compression_method.toString());
|
||||||
std::optional<int> level;
|
std::optional<int> level;
|
||||||
if (method == "ZSTD")
|
if (method == "ZSTD")
|
||||||
level = query_context->getSettingsRef().network_zstd_compression_level;
|
level = query_settings.network_zstd_compression_level;
|
||||||
|
|
||||||
if (state.compression == Protocol::Compression::Enable)
|
if (state.compression == Protocol::Compression::Enable)
|
||||||
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
|
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
|
||||||
*out, CompressionCodecFactory::instance().get(method, level));
|
*out, CompressionCodecFactory::instance().get(method, level, !query_settings.allow_suspicious_codecs));
|
||||||
else
|
else
|
||||||
state.maybe_compressed_out = out;
|
state.maybe_compressed_out = out;
|
||||||
}
|
}
|
||||||
|
@ -379,7 +379,7 @@ void Connection::sendQuery(
|
|||||||
if (method == "ZSTD")
|
if (method == "ZSTD")
|
||||||
level = settings->network_zstd_compression_level;
|
level = settings->network_zstd_compression_level;
|
||||||
|
|
||||||
compression_codec = CompressionCodecFactory::instance().get(method, level);
|
compression_codec = CompressionCodecFactory::instance().get(method, level, !settings->allow_suspicious_codecs);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
compression_codec = CompressionCodecFactory::instance().getDefaultCodec();
|
compression_codec = CompressionCodecFactory::instance().getDefaultCodec();
|
||||||
|
@ -23,6 +23,8 @@ protected:
|
|||||||
|
|
||||||
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override { return uncompressed_size + 2; }
|
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override { return uncompressed_size + 2; }
|
||||||
|
|
||||||
|
bool isCompression() const override { return false; }
|
||||||
|
bool isGenericCompression() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
UInt8 delta_bytes_size;
|
UInt8 delta_bytes_size;
|
||||||
|
@ -109,6 +109,9 @@ protected:
|
|||||||
|
|
||||||
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
|
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
|
||||||
|
|
||||||
|
bool isCompression() const override { return true; }
|
||||||
|
bool isGenericCompression() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
UInt8 data_bytes_size;
|
UInt8 data_bytes_size;
|
||||||
};
|
};
|
||||||
|
@ -106,6 +106,9 @@ protected:
|
|||||||
|
|
||||||
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
|
UInt32 getMaxCompressedDataSize(UInt32 uncompressed_size) const override;
|
||||||
|
|
||||||
|
bool isCompression() const override { return true; }
|
||||||
|
bool isGenericCompression() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
UInt8 data_bytes_size;
|
UInt8 data_bytes_size;
|
||||||
};
|
};
|
||||||
|
@ -21,6 +21,9 @@ public:
|
|||||||
protected:
|
protected:
|
||||||
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
UInt32 doCompressData(const char * source, UInt32 source_size, char * dest) const override;
|
||||||
|
|
||||||
|
bool isCompression() const override { return true; }
|
||||||
|
bool isGenericCompression() const override { return true; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||||
|
|
||||||
|
@ -17,12 +17,34 @@ namespace DB
|
|||||||
|
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
{
|
{
|
||||||
extern const int CORRUPTED_DATA;
|
extern const int CORRUPTED_DATA;
|
||||||
|
extern const int BAD_ARGUMENTS;
|
||||||
}
|
}
|
||||||
|
|
||||||
CompressionCodecMultiple::CompressionCodecMultiple(Codecs codecs_)
|
CompressionCodecMultiple::CompressionCodecMultiple(Codecs codecs_, bool sanity_check)
|
||||||
: codecs(codecs_)
|
: codecs(codecs_)
|
||||||
{
|
{
|
||||||
|
if (sanity_check)
|
||||||
|
{
|
||||||
|
/// It does not make sense to apply any transformations after generic compression algorithm
|
||||||
|
/// So, generic compression can be only one and only at the end.
|
||||||
|
bool has_generic_compression = false;
|
||||||
|
for (const auto & codec : codecs)
|
||||||
|
{
|
||||||
|
if (codec->isNone())
|
||||||
|
throw Exception("It does not make sense to have codec NONE along with other compression codecs: " + getCodecDescImpl()
|
||||||
|
+ " (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
|
||||||
|
ErrorCodes::BAD_ARGUMENTS);
|
||||||
|
|
||||||
|
if (has_generic_compression)
|
||||||
|
throw Exception("The combination of compression codecs " + getCodecDescImpl() + " is meaningless, "
|
||||||
|
" because it does not make sense to apply any transformations after generic compression algorithm."
|
||||||
|
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).", ErrorCodes::BAD_ARGUMENTS);
|
||||||
|
|
||||||
|
if (codec->isGenericCompression())
|
||||||
|
has_generic_compression = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
uint8_t CompressionCodecMultiple::getMethodByte() const
|
uint8_t CompressionCodecMultiple::getMethodByte() const
|
||||||
@ -31,6 +53,11 @@ uint8_t CompressionCodecMultiple::getMethodByte() const
|
|||||||
}
|
}
|
||||||
|
|
||||||
String CompressionCodecMultiple::getCodecDesc() const
|
String CompressionCodecMultiple::getCodecDesc() const
|
||||||
|
{
|
||||||
|
return getCodecDescImpl();
|
||||||
|
}
|
||||||
|
|
||||||
|
String CompressionCodecMultiple::getCodecDescImpl() const
|
||||||
{
|
{
|
||||||
WriteBufferFromOwnString out;
|
WriteBufferFromOwnString out;
|
||||||
for (size_t idx = 0; idx < codecs.size(); ++idx)
|
for (size_t idx = 0; idx < codecs.size(); ++idx)
|
||||||
@ -120,6 +147,14 @@ void CompressionCodecMultiple::doDecompressData(const char * source, UInt32 sour
|
|||||||
memcpy(dest, compressed_buf.data(), decompressed_size);
|
memcpy(dest, compressed_buf.data(), decompressed_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool CompressionCodecMultiple::isCompression() const
|
||||||
|
{
|
||||||
|
for (const auto & codec : codecs)
|
||||||
|
if (codec->isCompression())
|
||||||
|
return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
void registerCodecMultiple(CompressionCodecFactory & factory)
|
void registerCodecMultiple(CompressionCodecFactory & factory)
|
||||||
{
|
{
|
||||||
factory.registerSimpleCompressionCodec("Multiple", static_cast<UInt8>(CompressionMethodByte::Multiple), [&] ()
|
factory.registerSimpleCompressionCodec("Multiple", static_cast<UInt8>(CompressionMethodByte::Multiple), [&] ()
|
||||||
|
@ -9,7 +9,7 @@ class CompressionCodecMultiple final : public ICompressionCodec
|
|||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
CompressionCodecMultiple() = default;
|
CompressionCodecMultiple() = default;
|
||||||
explicit CompressionCodecMultiple(Codecs codecs_);
|
CompressionCodecMultiple(Codecs codecs_, bool sanity_check);
|
||||||
|
|
||||||
uint8_t getMethodByte() const override;
|
uint8_t getMethodByte() const override;
|
||||||
|
|
||||||
@ -24,9 +24,13 @@ protected:
|
|||||||
|
|
||||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 decompressed_size) const override;
|
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 decompressed_size) const override;
|
||||||
|
|
||||||
|
bool isCompression() const override;
|
||||||
|
bool isGenericCompression() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
Codecs codecs;
|
Codecs codecs;
|
||||||
|
|
||||||
|
String getCodecDescImpl() const;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
|
@ -20,6 +20,9 @@ protected:
|
|||||||
|
|
||||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||||
|
|
||||||
|
bool isCompression() const override { return false; }
|
||||||
|
bool isGenericCompression() const override { return false; }
|
||||||
|
bool isNone() const override { return true; }
|
||||||
};
|
};
|
||||||
|
|
||||||
class CompressionCodecFactory;
|
class CompressionCodecFactory;
|
||||||
|
@ -48,6 +48,9 @@ protected:
|
|||||||
return uncompressed_size + MAX_COMPRESSED_BLOCK_SIZE + HEADER_SIZE;
|
return uncompressed_size + MAX_COMPRESSED_BLOCK_SIZE + HEADER_SIZE;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool isCompression() const override { return true; }
|
||||||
|
bool isGenericCompression() const override { return false; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
TypeIndex type_idx;
|
TypeIndex type_idx;
|
||||||
Variant variant;
|
Variant variant;
|
||||||
|
@ -26,6 +26,9 @@ protected:
|
|||||||
|
|
||||||
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
void doDecompressData(const char * source, UInt32 source_size, char * dest, UInt32 uncompressed_size) const override;
|
||||||
|
|
||||||
|
bool isCompression() const override { return true; }
|
||||||
|
bool isGenericCompression() const override { return true; }
|
||||||
|
|
||||||
private:
|
private:
|
||||||
const int level;
|
const int level;
|
||||||
};
|
};
|
||||||
|
@ -29,22 +29,22 @@ CompressionCodecPtr CompressionCodecFactory::getDefaultCodec() const
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional<int> level) const
|
CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional<int> level, bool sanity_check) const
|
||||||
{
|
{
|
||||||
if (level)
|
if (level)
|
||||||
{
|
{
|
||||||
auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level));
|
auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level));
|
||||||
return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)));
|
return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), {}, sanity_check);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
auto identifier = std::make_shared<ASTIdentifier>(Poco::toUpper(family_name));
|
auto identifier = std::make_shared<ASTIdentifier>(Poco::toUpper(family_name));
|
||||||
return get(makeASTFunction("CODEC", identifier));
|
return get(makeASTFunction("CODEC", identifier), {}, sanity_check);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, DataTypePtr column_type) const
|
CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, DataTypePtr column_type, bool sanity_check) const
|
||||||
{
|
{
|
||||||
if (const auto * func = ast->as<ASTFunction>())
|
if (const auto * func = ast->as<ASTFunction>())
|
||||||
{
|
{
|
||||||
@ -60,10 +60,19 @@ CompressionCodecPtr CompressionCodecFactory::get(const ASTPtr & ast, DataTypePtr
|
|||||||
throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
|
throw Exception("Unexpected AST element for compression codec", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
CompressionCodecPtr res;
|
||||||
|
|
||||||
if (codecs.size() == 1)
|
if (codecs.size() == 1)
|
||||||
return codecs.back();
|
res = codecs.back();
|
||||||
else if (codecs.size() > 1)
|
else if (codecs.size() > 1)
|
||||||
return std::make_shared<CompressionCodecMultiple>(codecs);
|
res = std::make_shared<CompressionCodecMultiple>(codecs, sanity_check);
|
||||||
|
|
||||||
|
if (sanity_check && !res->isCompression())
|
||||||
|
throw Exception("The combination of compression codecs " + res->getCodecDesc() + " does not compress anything."
|
||||||
|
" (Note: you can enable setting 'allow_suspicious_codecs' to skip this check).",
|
||||||
|
ErrorCodes::BAD_ARGUMENTS);
|
||||||
|
|
||||||
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC);
|
throw Exception("Unknown codec family: " + queryToString(ast), ErrorCodes::UNKNOWN_CODEC);
|
||||||
|
@ -40,13 +40,13 @@ public:
|
|||||||
/// Get codec by AST and possible column_type
|
/// Get codec by AST and possible column_type
|
||||||
/// some codecs can use information about type to improve inner settings
|
/// some codecs can use information about type to improve inner settings
|
||||||
/// but every codec should be able to work without information about type
|
/// but every codec should be able to work without information about type
|
||||||
CompressionCodecPtr get(const ASTPtr & ast, DataTypePtr column_type = nullptr) const;
|
CompressionCodecPtr get(const ASTPtr & ast, DataTypePtr column_type, bool sanity_check) const;
|
||||||
|
|
||||||
/// Get codec by method byte (no params available)
|
/// Get codec by method byte (no params available)
|
||||||
CompressionCodecPtr get(const uint8_t byte_code) const;
|
CompressionCodecPtr get(const uint8_t byte_code) const;
|
||||||
|
|
||||||
/// For backward compatibility with config settings
|
/// For backward compatibility with config settings
|
||||||
CompressionCodecPtr get(const String & family_name, std::optional<int> level) const;
|
CompressionCodecPtr get(const String & family_name, std::optional<int> level, bool sanity_check) const;
|
||||||
|
|
||||||
/// Register codec with parameters and column type
|
/// Register codec with parameters and column type
|
||||||
void registerCompressionCodecWithType(const String & family_name, std::optional<uint8_t> byte_code, CreatorWithType creator);
|
void registerCompressionCodecWithType(const String & family_name, std::optional<uint8_t> byte_code, CreatorWithType creator);
|
||||||
|
@ -57,7 +57,16 @@ public:
|
|||||||
static uint8_t readMethod(const char * source);
|
static uint8_t readMethod(const char * source);
|
||||||
|
|
||||||
/// Some codecs may use information about column type which appears after codec creation
|
/// Some codecs may use information about column type which appears after codec creation
|
||||||
virtual void useInfoAboutType(DataTypePtr /* data_type */) { }
|
virtual void useInfoAboutType(DataTypePtr /* data_type */) {}
|
||||||
|
|
||||||
|
/// Return true if this codec actually compressing something. Otherwise it can be just transformation that helps compression (e.g. Delta).
|
||||||
|
virtual bool isCompression() const = 0;
|
||||||
|
|
||||||
|
/// Is it a generic compression algorithm like lz4, zstd. Usually it does not make sense to apply generic compression more than single time.
|
||||||
|
virtual bool isGenericCompression() const = 0;
|
||||||
|
|
||||||
|
/// If it does nothing.
|
||||||
|
virtual bool isNone() const { return false; }
|
||||||
|
|
||||||
protected:
|
protected:
|
||||||
|
|
||||||
|
@ -470,7 +470,7 @@ CompressionCodecPtr makeCodec(const std::string & codec_string, const DataTypePt
|
|||||||
|
|
||||||
parser.parse(token_iterator, codec_ast, expected);
|
parser.parse(token_iterator, codec_ast, expected);
|
||||||
|
|
||||||
return CompressionCodecFactory::instance().get(codec_ast, data_type);
|
return CompressionCodecFactory::instance().get(codec_ast, data_type, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
template <typename Timer>
|
template <typename Timer>
|
||||||
|
@ -265,6 +265,7 @@ struct Settings : public SettingsCollection<Settings>
|
|||||||
M(SettingBool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \
|
M(SettingBool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \
|
||||||
M(SettingBool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \
|
M(SettingBool, empty_result_for_aggregation_by_empty_set, false, "Return empty result when aggregating without keys on empty set.", 0) \
|
||||||
M(SettingBool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
|
M(SettingBool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
|
||||||
|
M(SettingBool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \
|
||||||
M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \
|
M(SettingUInt64, odbc_max_field_size, 1024, "Max size of filed can be read from ODBC dictionary. Long strings are truncated.", 0) \
|
||||||
M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
M(SettingUInt64, query_profiler_real_time_period_ns, 1000000000, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
||||||
M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
M(SettingUInt64, query_profiler_cpu_time_period_ns, 1000000000, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
||||||
|
@ -37,7 +37,7 @@ struct TemporaryFileStream
|
|||||||
std::atomic<bool> * is_cancelled, const std::string & codec)
|
std::atomic<bool> * is_cancelled, const std::string & codec)
|
||||||
{
|
{
|
||||||
WriteBufferFromFile file_buf(path);
|
WriteBufferFromFile file_buf(path);
|
||||||
CompressedWriteBuffer compressed_buf(file_buf, CompressionCodecFactory::instance().get(codec, {}));
|
CompressedWriteBuffer compressed_buf(file_buf, CompressionCodecFactory::instance().get(codec, {}, false));
|
||||||
NativeBlockOutputStream output(compressed_buf, 0, header);
|
NativeBlockOutputStream output(compressed_buf, 0, header);
|
||||||
copyData(input, output, is_cancelled);
|
copyData(input, output, is_cancelled);
|
||||||
}
|
}
|
||||||
|
@ -65,7 +65,7 @@ std::pair<String, StoragePtr> createTableFromAST(
|
|||||||
if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
|
if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns)
|
||||||
throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
|
throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED);
|
||||||
|
|
||||||
ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context);
|
ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context, false);
|
||||||
ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
|
ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
|
||||||
|
|
||||||
return
|
return
|
||||||
|
@ -56,7 +56,7 @@ BlockIO InterpreterAlterQuery::execute()
|
|||||||
LiveViewCommands live_view_commands;
|
LiveViewCommands live_view_commands;
|
||||||
for (ASTAlterCommand * command_ast : alter.command_list->commands)
|
for (ASTAlterCommand * command_ast : alter.command_list->commands)
|
||||||
{
|
{
|
||||||
if (auto alter_command = AlterCommand::parse(command_ast))
|
if (auto alter_command = AlterCommand::parse(command_ast, !context.getSettingsRef().allow_suspicious_codecs))
|
||||||
alter_commands.emplace_back(std::move(*alter_command));
|
alter_commands.emplace_back(std::move(*alter_command));
|
||||||
else if (auto partition_command = PartitionCommand::parse(command_ast))
|
else if (auto partition_command = PartitionCommand::parse(command_ast))
|
||||||
{
|
{
|
||||||
|
@ -267,7 +267,8 @@ ASTPtr InterpreterCreateQuery::formatConstraints(const ConstraintsDescription &
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpressionList & columns_ast, const Context & context)
|
ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
||||||
|
const ASTExpressionList & columns_ast, const Context & context, bool sanity_check_compression_codecs)
|
||||||
{
|
{
|
||||||
/// First, deduce implicit types.
|
/// First, deduce implicit types.
|
||||||
|
|
||||||
@ -355,7 +356,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(const ASTExpres
|
|||||||
column.comment = col_decl.comment->as<ASTLiteral &>().value.get<String>();
|
column.comment = col_decl.comment->as<ASTLiteral &>().value.get<String>();
|
||||||
|
|
||||||
if (col_decl.codec)
|
if (col_decl.codec)
|
||||||
column.codec = CompressionCodecFactory::instance().get(col_decl.codec, column.type);
|
column.codec = CompressionCodecFactory::instance().get(col_decl.codec, column.type, sanity_check_compression_codecs);
|
||||||
|
|
||||||
if (col_decl.ttl)
|
if (col_decl.ttl)
|
||||||
column.ttl = col_decl.ttl;
|
column.ttl = col_decl.ttl;
|
||||||
@ -390,7 +391,10 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::setProperties(AS
|
|||||||
if (create.columns_list)
|
if (create.columns_list)
|
||||||
{
|
{
|
||||||
if (create.columns_list->columns)
|
if (create.columns_list->columns)
|
||||||
properties.columns = getColumnsDescription(*create.columns_list->columns, context);
|
{
|
||||||
|
bool sanity_check_compression_codecs = !create.attach && !context.getSettingsRef().allow_suspicious_codecs;
|
||||||
|
properties.columns = getColumnsDescription(*create.columns_list->columns, context, sanity_check_compression_codecs);
|
||||||
|
}
|
||||||
|
|
||||||
if (create.columns_list->indices)
|
if (create.columns_list->indices)
|
||||||
for (const auto & index : create.columns_list->indices->children)
|
for (const auto & index : create.columns_list->indices->children)
|
||||||
|
@ -46,7 +46,7 @@ public:
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Obtain information about columns, their types, default values and column comments, for case when columns in CREATE query is specified explicitly.
|
/// Obtain information about columns, their types, default values and column comments, for case when columns in CREATE query is specified explicitly.
|
||||||
static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, const Context & context);
|
static ColumnsDescription getColumnsDescription(const ASTExpressionList & columns, const Context & context, bool sanity_check_compression_codecs);
|
||||||
static ConstraintsDescription getConstraintsDescription(const ASTExpressionList * constraints);
|
static ConstraintsDescription getConstraintsDescription(const ASTExpressionList * constraints);
|
||||||
|
|
||||||
private:
|
private:
|
||||||
|
@ -343,7 +343,7 @@ StoragePtr InterpreterSystemQuery::tryRestartReplica(const StorageID & replica,
|
|||||||
auto & create = create_ast->as<ASTCreateQuery &>();
|
auto & create = create_ast->as<ASTCreateQuery &>();
|
||||||
create.attach = true;
|
create.attach = true;
|
||||||
|
|
||||||
auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context);
|
auto columns = InterpreterCreateQuery::getColumnsDescription(*create.columns_list->columns, system_context, false);
|
||||||
auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints);
|
auto constraints = InterpreterCreateQuery::getConstraintsDescription(create.columns_list->constraints);
|
||||||
auto data_path = database->getTableDataPath(create);
|
auto data_path = database->getTableDataPath(create);
|
||||||
|
|
||||||
|
@ -44,7 +44,7 @@ namespace ErrorCodes
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_ast)
|
std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_ast, bool sanity_check_compression_codecs)
|
||||||
{
|
{
|
||||||
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
|
const DataTypeFactory & data_type_factory = DataTypeFactory::instance();
|
||||||
const CompressionCodecFactory & compression_codec_factory = CompressionCodecFactory::instance();
|
const CompressionCodecFactory & compression_codec_factory = CompressionCodecFactory::instance();
|
||||||
@ -75,7 +75,7 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (ast_col_decl.codec)
|
if (ast_col_decl.codec)
|
||||||
command.codec = compression_codec_factory.get(ast_col_decl.codec, command.data_type);
|
command.codec = compression_codec_factory.get(ast_col_decl.codec, command.data_type, sanity_check_compression_codecs);
|
||||||
|
|
||||||
if (command_ast->column)
|
if (command_ast->column)
|
||||||
command.after_column = getIdentifierName(command_ast->column);
|
command.after_column = getIdentifierName(command_ast->column);
|
||||||
@ -131,7 +131,7 @@ std::optional<AlterCommand> AlterCommand::parse(const ASTAlterCommand * command_
|
|||||||
command.ttl = ast_col_decl.ttl;
|
command.ttl = ast_col_decl.ttl;
|
||||||
|
|
||||||
if (ast_col_decl.codec)
|
if (ast_col_decl.codec)
|
||||||
command.codec = compression_codec_factory.get(ast_col_decl.codec, command.data_type);
|
command.codec = compression_codec_factory.get(ast_col_decl.codec, command.data_type, sanity_check_compression_codecs);
|
||||||
|
|
||||||
command.if_exists = command_ast->if_exists;
|
command.if_exists = command_ast->if_exists;
|
||||||
|
|
||||||
|
@ -100,7 +100,7 @@ struct AlterCommand
|
|||||||
/// Target column name
|
/// Target column name
|
||||||
String rename_to;
|
String rename_to;
|
||||||
|
|
||||||
static std::optional<AlterCommand> parse(const ASTAlterCommand * command);
|
static std::optional<AlterCommand> parse(const ASTAlterCommand * command, bool sanity_check_compression_codecs);
|
||||||
|
|
||||||
void apply(StorageInMemoryMetadata & metadata) const;
|
void apply(StorageInMemoryMetadata & metadata) const;
|
||||||
|
|
||||||
|
@ -120,7 +120,7 @@ void ColumnDescription::readText(ReadBuffer & buf)
|
|||||||
comment = col_ast->comment->as<ASTLiteral &>().value.get<String>();
|
comment = col_ast->comment->as<ASTLiteral &>().value.get<String>();
|
||||||
|
|
||||||
if (col_ast->codec)
|
if (col_ast->codec)
|
||||||
codec = CompressionCodecFactory::instance().get(col_ast->codec, type);
|
codec = CompressionCodecFactory::instance().get(col_ast->codec, type, false);
|
||||||
|
|
||||||
if (col_ast->ttl)
|
if (col_ast->ttl)
|
||||||
ttl = col_ast->ttl;
|
ttl = col_ast->ttl;
|
||||||
|
@ -91,7 +91,7 @@ public:
|
|||||||
|
|
||||||
for (const auto & element : elements)
|
for (const auto & element : elements)
|
||||||
if (element.check(part_size, part_size_ratio))
|
if (element.check(part_size, part_size_ratio))
|
||||||
res = factory.get(element.family_name, element.level);
|
res = factory.get(element.family_name, element.level, false);
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
@ -31,7 +31,7 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, con
|
|||||||
if (!columns_list)
|
if (!columns_list)
|
||||||
throw Exception("Could not cast AST to ASTExpressionList", ErrorCodes::LOGICAL_ERROR);
|
throw Exception("Could not cast AST to ASTExpressionList", ErrorCodes::LOGICAL_ERROR);
|
||||||
|
|
||||||
return InterpreterCreateQuery::getColumnsDescription(*columns_list, context);
|
return InterpreterCreateQuery::getColumnsDescription(*columns_list, context, !context.getSettingsRef().allow_suspicious_codecs);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
Loading…
Reference in New Issue
Block a user