enable_qpl_deflate_codec --> enable_deflate_qpl_codec

This commit is contained in:
Robert Schulze 2023-06-09 12:38:38 +00:00
parent 1f928f2d3d
commit 1aa158909e
No known key found for this signature in database
GPG Key ID: 26703B55FB13728A
16 changed files with 24 additions and 24 deletions

View File

@ -380,7 +380,7 @@ High compression levels are useful for asymmetric scenarios, like compress once,
`DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply: `DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_qpl_deflate_codec=1`. - DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_deflate_qpl_codec = 1`.
- DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details. - DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
- DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details. - DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
- DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled. - DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.

View File

@ -588,7 +588,7 @@ void Connection::sendQuery(
if (method == "ZSTD") if (method == "ZSTD")
level = settings->network_zstd_compression_level; level = settings->network_zstd_compression_level;
CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_qpl_deflate_codec); CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_deflate_qpl_codec);
compression_codec = CompressionCodecFactory::instance().get(method, level); compression_codec = CompressionCodecFactory::instance().get(method, level);
} }
else else

View File

@ -40,10 +40,10 @@ public:
CompressionCodecPtr getDefaultCodec() const; CompressionCodecPtr getDefaultCodec() const;
/// Validate codecs AST specified by user and parses codecs description (substitute default parameters) /// Validate codecs AST specified by user and parses codecs description (substitute default parameters)
ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const; ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const;
/// Validate codecs AST specified by user /// Validate codecs AST specified by user
void validateCodec(const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const; void validateCodec(const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const;
/// Get codec by AST and possible column_type. Some codecs can use /// Get codec by AST and possible column_type. Some codecs can use
/// information about type to improve inner settings, but every codec should /// information about type to improve inner settings, but every codec should

View File

@ -34,7 +34,7 @@ namespace ErrorCodes
void CompressionCodecFactory::validateCodec( void CompressionCodecFactory::validateCodec(
const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const
{ {
if (family_name.empty()) if (family_name.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty"); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty");
@ -43,13 +43,13 @@ void CompressionCodecFactory::validateCodec(
{ {
auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level)); auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level));
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)), validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)),
{}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec); {}, sanity_check, allow_experimental_codecs, enable_deflate_qpl_codec);
} }
else else
{ {
auto identifier = std::make_shared<ASTIdentifier>(Poco::toUpper(family_name)); auto identifier = std::make_shared<ASTIdentifier>(Poco::toUpper(family_name));
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier), validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier),
{}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec); {}, sanity_check, allow_experimental_codecs, enable_deflate_qpl_codec);
} }
} }
@ -77,7 +77,7 @@ bool innerDataTypeIsFloat(const DataTypePtr & type)
} }
ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST( ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const
{ {
if (const auto * func = ast->as<ASTFunction>()) if (const auto * func = ast->as<ASTFunction>())
{ {
@ -159,10 +159,10 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
" You can enable it with the 'allow_experimental_codecs' setting.", " You can enable it with the 'allow_experimental_codecs' setting.",
codec_family_name); codec_family_name);
if (!enable_qpl_deflate_codec && result_codec->isDeflateQplCompression()) if (!enable_deflate_qpl_codec && result_codec->isDeflateQplCompression())
throw Exception(ErrorCodes::BAD_ARGUMENTS, throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Codec {} is disabled by default." "Codec {} is disabled by default."
" You can enable it with the 'enable_qpl_deflate_codec' setting.", " You can enable it with the 'enable_deflate_qpl_codec' setting.",
codec_family_name); codec_family_name);
codecs_descriptions->children.emplace_back(result_codec->getCodecDesc()); codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());

View File

@ -319,7 +319,7 @@ class IColumn;
M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \ M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \ M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \
M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \ M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \
M(Bool, enable_qpl_deflate_codec, false, "If it is set to true, allow usage of the DEFLATE_QPL codec.", 0) \ M(Bool, enable_deflate_qpl_codec, false, "Enable/disable the DEFLATE_QPL codec.", 0) \
M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \ M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \ M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \

View File

@ -571,7 +571,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs; bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs;
bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs; bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs;
bool enable_qpl_deflate_codec = attach || context_->getSettingsRef().enable_qpl_deflate_codec; bool enable_deflate_qpl_codec = attach || context_->getSettingsRef().enable_deflate_qpl_codec;
ColumnsDescription res; ColumnsDescription res;
auto name_type_it = column_names_and_types.begin(); auto name_type_it = column_names_and_types.begin();
@ -632,7 +632,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
if (col_decl.default_specifier == "ALIAS") if (col_decl.default_specifier == "ALIAS")
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS");
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_qpl_deflate_codec); col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_deflate_qpl_codec);
} }
if (col_decl.ttl) if (col_decl.ttl)

View File

@ -1775,7 +1775,7 @@ void TCPHandler::initBlockOutput(const Block & block)
if (state.compression == Protocol::Compression::Enable) if (state.compression == Protocol::Compression::Enable)
{ {
CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_qpl_deflate_codec); CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_deflate_qpl_codec);
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>( state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
*out, CompressionCodecFactory::instance().get(method, level)); *out, CompressionCodecFactory::instance().get(method, level));

View File

@ -1067,7 +1067,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
"this column name is reserved for lightweight delete feature", backQuote(column_name)); "this column name is reserved for lightweight delete feature", backQuote(column_name));
if (command.codec) if (command.codec)
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec);
all_columns.add(ColumnDescription(column_name, command.data_type)); all_columns.add(ColumnDescription(column_name, command.data_type));
} }
@ -1093,7 +1093,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
{ {
if (all_columns.hasAlias(column_name)) if (all_columns.hasAlias(column_name))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS"); throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS");
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec);
} }
auto column_default = all_columns.getDefault(column_name); auto column_default = all_columns.getDefault(column_name);
if (column_default) if (column_default)

View File

@ -733,7 +733,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
if (compression_method == "ZSTD") if (compression_method == "ZSTD")
compression_level = settings.network_zstd_compression_level; compression_level = settings.network_zstd_compression_level;
CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_qpl_deflate_codec); CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enale_deflate_qpl_codec);
CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level); CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level);
/// tmp directory is used to ensure atomicity of transactions /// tmp directory is used to ensure atomicity of transactions

View File

@ -285,7 +285,7 @@ TTLDescription TTLDescription::getTTLFromAST(
{ {
result.recompression_codec = result.recompression_codec =
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST( CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec); ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec);
} }
} }

View File

@ -20,7 +20,7 @@ def get_options(i, upgrade_check):
'''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i) '''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i)
) )
client_options.append("allow_experimental_database_replicated=1") client_options.append("allow_experimental_database_replicated=1")
client_options.append("enable_qpl_deflate_codec=1") client_options.append("enable_deflate_qpl_codec=1")
# If database name is not specified, new database is created for each functional test. # If database name is not specified, new database is created for each functional test.
# Run some threads with one database for all tests. # Run some threads with one database for all tests.

View File

@ -1,7 +1,7 @@
<clickhouse> <clickhouse>
<profiles> <profiles>
<default> <default>
<enable_qpl_deflate_codec>1</enable_qpl_deflate_codec> <enable_deflate_qpl_codec>1</enable_deflate_qpl_codec>
</default> </default>
</profiles> </profiles>
</clickhouse> </clickhouse>

View File

@ -28,7 +28,7 @@ SELECT * FROM alter_compression_codec ORDER BY id;
OPTIMIZE TABLE alter_compression_codec FINAL; OPTIMIZE TABLE alter_compression_codec FINAL;
SELECT * FROM alter_compression_codec ORDER BY id; SELECT * FROM alter_compression_codec ORDER BY id;
SET enable_qpl_deflate_codec = 1; SET enable_deflate_qpl_codec = 1;
ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL); ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL);
SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column'; SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column';

View File

@ -1,6 +1,6 @@
SET send_logs_level = 'fatal'; SET send_logs_level = 'fatal';
SET allow_suspicious_codecs = 1; SET allow_suspicious_codecs = 1;
SET enable_qpl_deflate_codec = 1; SET enable_deflate_qpl_codec = 1;
DROP TABLE IF EXISTS compression_codec; DROP TABLE IF EXISTS compression_codec;

View File

@ -1,6 +1,6 @@
SET send_logs_level = 'fatal'; SET send_logs_level = 'fatal';
SET allow_suspicious_codecs = 1; SET allow_suspicious_codecs = 1;
SET enable_qpl_deflate_codec = 1; SET enable_deflate_qpl_codec = 1;
-- copy-paste for storage log -- copy-paste for storage log

View File

@ -1,5 +1,5 @@
SET send_logs_level = 'fatal'; SET send_logs_level = 'fatal';
SET enable_qpl_deflate_codec = 1; SET enable_deflate_qpl_codec = 1;
DROP TABLE IF EXISTS compression_codec; DROP TABLE IF EXISTS compression_codec;