mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-29 19:12:03 +00:00
enable_qpl_deflate_codec --> enable_deflate_qpl_codec
This commit is contained in:
parent
1f928f2d3d
commit
1aa158909e
@ -380,7 +380,7 @@ High compression levels are useful for asymmetric scenarios, like compress once,
|
|||||||
|
|
||||||
`DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
|
`DEFLATE_QPL` — [Deflate compression algorithm](https://github.com/intel/qpl) implemented by Intel® Query Processing Library. Some limitations apply:
|
||||||
|
|
||||||
- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_qpl_deflate_codec=1`.
|
- DEFLATE_QPL is disabled by default and can only be used after setting configuration parameter `enable_deflate_qpl_codec = 1`.
|
||||||
- DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
|
- DEFLATE_QPL requires a ClickHouse build compiled with SSE 4.2 instructions (by default, this is the case). Refer to [Build Clickhouse with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Build-Clickhouse-with-DEFLATE_QPL) for more details.
|
||||||
- DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
|
- DEFLATE_QPL works best if the system has a Intel® IAA (In-Memory Analytics Accelerator) offloading device. Refer to [Accelerator Configuration](https://intel.github.io/qpl/documentation/get_started_docs/installation.html#accelerator-configuration) and [Benchmark with DEFLATE_QPL](/docs/en/development/building_and_benchmarking_deflate_qpl.md/#Run-Benchmark-with-DEFLATE_QPL) for more details.
|
||||||
- DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.
|
- DEFLATE_QPL-compressed data can only be transferred between ClickHouse nodes compiled with SSE 4.2 enabled.
|
||||||
|
@ -588,7 +588,7 @@ void Connection::sendQuery(
|
|||||||
if (method == "ZSTD")
|
if (method == "ZSTD")
|
||||||
level = settings->network_zstd_compression_level;
|
level = settings->network_zstd_compression_level;
|
||||||
|
|
||||||
CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_qpl_deflate_codec);
|
CompressionCodecFactory::instance().validateCodec(method, level, !settings->allow_suspicious_codecs, settings->allow_experimental_codecs, settings->enable_deflate_qpl_codec);
|
||||||
compression_codec = CompressionCodecFactory::instance().get(method, level);
|
compression_codec = CompressionCodecFactory::instance().get(method, level);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -40,10 +40,10 @@ public:
|
|||||||
CompressionCodecPtr getDefaultCodec() const;
|
CompressionCodecPtr getDefaultCodec() const;
|
||||||
|
|
||||||
/// Validate codecs AST specified by user and parses codecs description (substitute default parameters)
|
/// Validate codecs AST specified by user and parses codecs description (substitute default parameters)
|
||||||
ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const;
|
ASTPtr validateCodecAndGetPreprocessedAST(const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const;
|
||||||
|
|
||||||
/// Validate codecs AST specified by user
|
/// Validate codecs AST specified by user
|
||||||
void validateCodec(const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const;
|
void validateCodec(const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const;
|
||||||
|
|
||||||
/// Get codec by AST and possible column_type. Some codecs can use
|
/// Get codec by AST and possible column_type. Some codecs can use
|
||||||
/// information about type to improve inner settings, but every codec should
|
/// information about type to improve inner settings, but every codec should
|
||||||
|
@ -34,7 +34,7 @@ namespace ErrorCodes
|
|||||||
|
|
||||||
|
|
||||||
void CompressionCodecFactory::validateCodec(
|
void CompressionCodecFactory::validateCodec(
|
||||||
const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const
|
const String & family_name, std::optional<int> level, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const
|
||||||
{
|
{
|
||||||
if (family_name.empty())
|
if (family_name.empty())
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty");
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Compression codec name cannot be empty");
|
||||||
@ -43,13 +43,13 @@ void CompressionCodecFactory::validateCodec(
|
|||||||
{
|
{
|
||||||
auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level));
|
auto literal = std::make_shared<ASTLiteral>(static_cast<UInt64>(*level));
|
||||||
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)),
|
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), literal)),
|
||||||
{}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec);
|
{}, sanity_check, allow_experimental_codecs, enable_deflate_qpl_codec);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
auto identifier = std::make_shared<ASTIdentifier>(Poco::toUpper(family_name));
|
auto identifier = std::make_shared<ASTIdentifier>(Poco::toUpper(family_name));
|
||||||
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier),
|
validateCodecAndGetPreprocessedAST(makeASTFunction("CODEC", identifier),
|
||||||
{}, sanity_check, allow_experimental_codecs, enable_qpl_deflate_codec);
|
{}, sanity_check, allow_experimental_codecs, enable_deflate_qpl_codec);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -77,7 +77,7 @@ bool innerDataTypeIsFloat(const DataTypePtr & type)
|
|||||||
}
|
}
|
||||||
|
|
||||||
ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
||||||
const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_qpl_deflate_codec) const
|
const ASTPtr & ast, const DataTypePtr & column_type, bool sanity_check, bool allow_experimental_codecs, bool enable_deflate_qpl_codec) const
|
||||||
{
|
{
|
||||||
if (const auto * func = ast->as<ASTFunction>())
|
if (const auto * func = ast->as<ASTFunction>())
|
||||||
{
|
{
|
||||||
@ -159,10 +159,10 @@ ASTPtr CompressionCodecFactory::validateCodecAndGetPreprocessedAST(
|
|||||||
" You can enable it with the 'allow_experimental_codecs' setting.",
|
" You can enable it with the 'allow_experimental_codecs' setting.",
|
||||||
codec_family_name);
|
codec_family_name);
|
||||||
|
|
||||||
if (!enable_qpl_deflate_codec && result_codec->isDeflateQplCompression())
|
if (!enable_deflate_qpl_codec && result_codec->isDeflateQplCompression())
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
throw Exception(ErrorCodes::BAD_ARGUMENTS,
|
||||||
"Codec {} is disabled by default."
|
"Codec {} is disabled by default."
|
||||||
" You can enable it with the 'enable_qpl_deflate_codec' setting.",
|
" You can enable it with the 'enable_deflate_qpl_codec' setting.",
|
||||||
codec_family_name);
|
codec_family_name);
|
||||||
|
|
||||||
codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());
|
codecs_descriptions->children.emplace_back(result_codec->getCodecDesc());
|
||||||
|
@ -319,7 +319,7 @@ class IColumn;
|
|||||||
M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
|
M(Bool, allow_distributed_ddl, true, "If it is set to true, then a user is allowed to executed distributed DDL queries.", 0) \
|
||||||
M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \
|
M(Bool, allow_suspicious_codecs, false, "If it is set to true, allow to specify meaningless compression codecs.", 0) \
|
||||||
M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \
|
M(Bool, allow_experimental_codecs, false, "If it is set to true, allow to specify experimental compression codecs (but we don't have those yet and this option does nothing).", 0) \
|
||||||
M(Bool, enable_qpl_deflate_codec, false, "If it is set to true, allow usage of the DEFLATE_QPL codec.", 0) \
|
M(Bool, enable_deflate_qpl_codec, false, "Enable/disable the DEFLATE_QPL codec.", 0) \
|
||||||
M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
M(UInt64, query_profiler_real_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for real clock timer of query profiler (in nanoseconds). Set 0 value to turn off the real clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
||||||
M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
M(UInt64, query_profiler_cpu_time_period_ns, QUERY_PROFILER_DEFAULT_SAMPLE_RATE_NS, "Period for CPU clock timer of query profiler (in nanoseconds). Set 0 value to turn off the CPU clock query profiler. Recommended value is at least 10000000 (100 times a second) for single queries or 1000000000 (once a second) for cluster-wide profiling.", 0) \
|
||||||
M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \
|
M(Bool, metrics_perf_events_enabled, false, "If enabled, some of the perf events will be measured throughout queries' execution.", 0) \
|
||||||
|
@ -571,7 +571,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
|||||||
|
|
||||||
bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs;
|
bool sanity_check_compression_codecs = !attach && !context_->getSettingsRef().allow_suspicious_codecs;
|
||||||
bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs;
|
bool allow_experimental_codecs = attach || context_->getSettingsRef().allow_experimental_codecs;
|
||||||
bool enable_qpl_deflate_codec = attach || context_->getSettingsRef().enable_qpl_deflate_codec;
|
bool enable_deflate_qpl_codec = attach || context_->getSettingsRef().enable_deflate_qpl_codec;
|
||||||
|
|
||||||
ColumnsDescription res;
|
ColumnsDescription res;
|
||||||
auto name_type_it = column_names_and_types.begin();
|
auto name_type_it = column_names_and_types.begin();
|
||||||
@ -632,7 +632,7 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
|
|||||||
if (col_decl.default_specifier == "ALIAS")
|
if (col_decl.default_specifier == "ALIAS")
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS");
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS");
|
||||||
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
|
column.codec = CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
|
||||||
col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_qpl_deflate_codec);
|
col_decl.codec, column.type, sanity_check_compression_codecs, allow_experimental_codecs, enable_deflate_qpl_codec);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (col_decl.ttl)
|
if (col_decl.ttl)
|
||||||
|
@ -1775,7 +1775,7 @@ void TCPHandler::initBlockOutput(const Block & block)
|
|||||||
|
|
||||||
if (state.compression == Protocol::Compression::Enable)
|
if (state.compression == Protocol::Compression::Enable)
|
||||||
{
|
{
|
||||||
CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_qpl_deflate_codec);
|
CompressionCodecFactory::instance().validateCodec(method, level, !query_settings.allow_suspicious_codecs, query_settings.allow_experimental_codecs, query_settings.enable_deflate_qpl_codec);
|
||||||
|
|
||||||
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
|
state.maybe_compressed_out = std::make_shared<CompressedWriteBuffer>(
|
||||||
*out, CompressionCodecFactory::instance().get(method, level));
|
*out, CompressionCodecFactory::instance().get(method, level));
|
||||||
|
@ -1067,7 +1067,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
|
|||||||
"this column name is reserved for lightweight delete feature", backQuote(column_name));
|
"this column name is reserved for lightweight delete feature", backQuote(column_name));
|
||||||
|
|
||||||
if (command.codec)
|
if (command.codec)
|
||||||
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec);
|
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec);
|
||||||
|
|
||||||
all_columns.add(ColumnDescription(column_name, command.data_type));
|
all_columns.add(ColumnDescription(column_name, command.data_type));
|
||||||
}
|
}
|
||||||
@ -1093,7 +1093,7 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
|
|||||||
{
|
{
|
||||||
if (all_columns.hasAlias(column_name))
|
if (all_columns.hasAlias(column_name))
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS");
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot specify codec for column type ALIAS");
|
||||||
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec);
|
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(command.codec, command.data_type, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec);
|
||||||
}
|
}
|
||||||
auto column_default = all_columns.getDefault(column_name);
|
auto column_default = all_columns.getDefault(column_name);
|
||||||
if (column_default)
|
if (column_default)
|
||||||
|
@ -733,7 +733,7 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const
|
|||||||
if (compression_method == "ZSTD")
|
if (compression_method == "ZSTD")
|
||||||
compression_level = settings.network_zstd_compression_level;
|
compression_level = settings.network_zstd_compression_level;
|
||||||
|
|
||||||
CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enable_qpl_deflate_codec);
|
CompressionCodecFactory::instance().validateCodec(compression_method, compression_level, !settings.allow_suspicious_codecs, settings.allow_experimental_codecs, settings.enale_deflate_qpl_codec);
|
||||||
CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level);
|
CompressionCodecPtr compression_codec = CompressionCodecFactory::instance().get(compression_method, compression_level);
|
||||||
|
|
||||||
/// tmp directory is used to ensure atomicity of transactions
|
/// tmp directory is used to ensure atomicity of transactions
|
||||||
|
@ -285,7 +285,7 @@ TTLDescription TTLDescription::getTTLFromAST(
|
|||||||
{
|
{
|
||||||
result.recompression_codec =
|
result.recompression_codec =
|
||||||
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
|
CompressionCodecFactory::instance().validateCodecAndGetPreprocessedAST(
|
||||||
ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_qpl_deflate_codec);
|
ttl_element->recompression_codec, {}, !context->getSettingsRef().allow_suspicious_codecs, context->getSettingsRef().allow_experimental_codecs, context->getSettingsRef().enable_deflate_qpl_codec);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -20,7 +20,7 @@ def get_options(i, upgrade_check):
|
|||||||
'''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i)
|
'''--db-engine="Replicated('/test/db/test_{}', 's1', 'r1')"'''.format(i)
|
||||||
)
|
)
|
||||||
client_options.append("allow_experimental_database_replicated=1")
|
client_options.append("allow_experimental_database_replicated=1")
|
||||||
client_options.append("enable_qpl_deflate_codec=1")
|
client_options.append("enable_deflate_qpl_codec=1")
|
||||||
|
|
||||||
# If database name is not specified, new database is created for each functional test.
|
# If database name is not specified, new database is created for each functional test.
|
||||||
# Run some threads with one database for all tests.
|
# Run some threads with one database for all tests.
|
||||||
|
@ -1,7 +1,7 @@
|
|||||||
<clickhouse>
|
<clickhouse>
|
||||||
<profiles>
|
<profiles>
|
||||||
<default>
|
<default>
|
||||||
<enable_qpl_deflate_codec>1</enable_qpl_deflate_codec>
|
<enable_deflate_qpl_codec>1</enable_deflate_qpl_codec>
|
||||||
</default>
|
</default>
|
||||||
</profiles>
|
</profiles>
|
||||||
</clickhouse>
|
</clickhouse>
|
||||||
|
@ -28,7 +28,7 @@ SELECT * FROM alter_compression_codec ORDER BY id;
|
|||||||
OPTIMIZE TABLE alter_compression_codec FINAL;
|
OPTIMIZE TABLE alter_compression_codec FINAL;
|
||||||
SELECT * FROM alter_compression_codec ORDER BY id;
|
SELECT * FROM alter_compression_codec ORDER BY id;
|
||||||
|
|
||||||
SET enable_qpl_deflate_codec = 1;
|
SET enable_deflate_qpl_codec = 1;
|
||||||
ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL);
|
ALTER TABLE alter_compression_codec MODIFY COLUMN alter_column CODEC(DEFLATE_QPL);
|
||||||
SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column';
|
SELECT compression_codec FROM system.columns WHERE database = currentDatabase() AND table = 'alter_compression_codec' AND name = 'alter_column';
|
||||||
|
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
SET send_logs_level = 'fatal';
|
SET send_logs_level = 'fatal';
|
||||||
SET allow_suspicious_codecs = 1;
|
SET allow_suspicious_codecs = 1;
|
||||||
SET enable_qpl_deflate_codec = 1;
|
SET enable_deflate_qpl_codec = 1;
|
||||||
|
|
||||||
DROP TABLE IF EXISTS compression_codec;
|
DROP TABLE IF EXISTS compression_codec;
|
||||||
|
|
||||||
|
@ -1,6 +1,6 @@
|
|||||||
SET send_logs_level = 'fatal';
|
SET send_logs_level = 'fatal';
|
||||||
SET allow_suspicious_codecs = 1;
|
SET allow_suspicious_codecs = 1;
|
||||||
SET enable_qpl_deflate_codec = 1;
|
SET enable_deflate_qpl_codec = 1;
|
||||||
|
|
||||||
-- copy-paste for storage log
|
-- copy-paste for storage log
|
||||||
|
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
SET send_logs_level = 'fatal';
|
SET send_logs_level = 'fatal';
|
||||||
SET enable_qpl_deflate_codec = 1;
|
SET enable_deflate_qpl_codec = 1;
|
||||||
|
|
||||||
DROP TABLE IF EXISTS compression_codec;
|
DROP TABLE IF EXISTS compression_codec;
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user