thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-19 16:20:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Parallel replicas is Beta

Browse Source
This commit is contained in:
Nikita Mikhaylov 2024-09-06 17:27:46 +02:00 committed by Nikita Mikhaylov
parent 50e1cba451
commit c9b053f07d
96 changed files with 604 additions and 730 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
docs/en/operations/settings/settings.md
View File

@ -1588,25 +1588,12 @@ This setting is useful for any replicated table.
An arbitrary integer expression that can be used to split work between replicas for a specific table.
The value can be any integer expression.
A query may be processed faster if it is executed on several servers in parallel but it depends on the used [parallel_replicas_custom_key](#parallel_replicas_custom_key)
and [parallel_replicas_custom_key_filter_type](#parallel_replicas_custom_key_filter_type).
Simple expressions using primary keys are preferred.
If the setting is used on a cluster that consists of a single shard with multiple replicas, those replicas will be converted into virtual shards.
Otherwise, it will behave same as for `SAMPLE` key, it will use multiple replicas of each shard.
## parallel_replicas_custom_key_filter_type {#parallel_replicas_custom_key_filter_type}
How to use `parallel_replicas_custom_key` expression for splitting work between replicas.
Possible values:
- `default` — Use the default implementation using modulo operation on the `parallel_replicas_custom_key`.
- `range` — Split the entire value space of the expression in the ranges. This type of filtering is useful if values of `parallel_replicas_custom_key` are uniformly spread across the entire integer space, e.g. hash values.
Default value: `default`.
## parallel_replicas_custom_key_range_lower {#parallel_replicas_custom_key_range_lower}
Allows the filter type `range` to split the work evenly between replicas based on the custom range `[parallel_replicas_custom_key_range_lower, INT_MAX]`.
@ -1621,9 +1608,9 @@ Allows the filter type `range` to split the work evenly between replicas based o
When used in conjuction with [parallel_replicas_custom_key_range_lower](#parallel_replicas_custom_key_range_lower), it lets the filter evenly split the work over replicas for the range `[parallel_replicas_custom_key_range_lower, parallel_replicas_custom_key_range_upper]`.
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing.
Note: This setting will not cause any additional data to be filtered during query processing, rather it changes the points at which the range filter breaks up the range `[0, INT_MAX]` for parallel processing
## use_parallel_replicas
## enable_parallel_replicas
Enables or disables sending SELECT queries to all replicas of a table (up to `max_parallel_replicas`). Reading is parallelized and coordinated dynamically. It will work for any kind of MergeTree table.

25
src/Core/ParallelReplicasMode.h Normal file
View File

@ -0,0 +1,25 @@
#pragma once
#include <base/types.h>
namespace DB
{
/// Deprecated. Used only to define a type for the obsolete setting.
enum class ParallelReplicasCustomKeyFilterType : UInt8
{
DEFAULT,
RANGE,
};
enum class ParallelReplicasMode : UInt8
{
AUTO = 0,
READ_TASKS = 1,
CUSTOM_KEY_SAMPLING = 2,
CUSTOM_KEY_RANGE = 3,
SAMPLING_KEY = 4,
};
}

8
src/Core/Settings.h
View File

@ -204,8 +204,6 @@ class IColumn;
\
M(Bool, group_by_use_nulls, false, "Treat columns mentioned in ROLLUP, CUBE or GROUPING SETS as Nullable", 0) \
\
M(NonZeroUInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled. Should be always greater than 0", 0) \
\
M(Bool, skip_unavailable_shards, false, "If true, ClickHouse silently skips unavailable shards. Shard is marked as unavailable when: 1) The shard cannot be reached due to a connection failure. 2) Shard is unresolvable through DNS. 3) Table does not exist on the shard.", 0) \
\
M(UInt64, parallel_distributed_insert_select, 0, "Process distributed INSERT SELECT query in the same cluster on local tables on every shard; if set to 1 - SELECT is executed on each shard; if set to 2 - SELECT and INSERT are executed on each shard", 0) \
@ -933,14 +931,15 @@ class IColumn;
M(Bool, allow_experimental_statistics, false, "Allows using statistics", 0) ALIAS(allow_experimental_statistic) \
\
/* Parallel replicas */ \
M(UInt64, enable_parallel_replicas, 0, "Use up to `max_parallel_replicas` the number of replicas from each shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) ALIAS(allow_experimental_parallel_reading_from_replicas) \
M(NonZeroUInt64, max_parallel_replicas, 1, "The maximum number of replicas of each shard used when the query is executed. For consistency (to get different parts of the same partition), this option only works for the specified sampling key. The lag of the replicas is not controlled. Should be always greater than 0", 0) \
M(ParallelReplicasMode, parallel_replicas_mode, ParallelReplicasMode::READ_TASKS, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \
M(UInt64, parallel_replicas_count, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the number of parallel replicas participating in query processing.", 0) \
M(UInt64, parallel_replica_offset, 0, "This is internal setting that should not be used directly and represents an implementation detail of the 'parallel replicas' mode. This setting will be automatically set up by the initiator server for distributed queries to the index of the replica participating in query processing among parallel replicas.", 0) \
M(String, parallel_replicas_custom_key, "", "Custom key assigning work to replicas when parallel replicas are used.", 0) \
M(ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT, "Type of filter to use with custom key for parallel replicas. default - use modulo operation on the custom key, range - use range filter on custom key using all possible values for the value type of custom key.", 0) \
M(UInt64, parallel_replicas_custom_key_range_lower, 0, "Lower bound for the universe that the parallel replicas custom range filter is calculated over", 0) \
M(UInt64, parallel_replicas_custom_key_range_upper, 0, "Upper bound for the universe that the parallel replicas custom range filter is calculated over. A value of 0 disables the upper bound, setting it to the max value of the custom key expression", 0) \
M(String, cluster_for_parallel_replicas, "", "Cluster for a shard in which current server is located", 0) \
M(UInt64, allow_experimental_parallel_reading_from_replicas, 0, "Use all the replicas from a shard for SELECT query execution. Reading is parallelized and coordinated dynamically. 0 - disabled, 1 - enabled, silently disable them in case of failure, 2 - enabled, throw an exception in case of failure", 0) \
M(Bool, parallel_replicas_allow_in_with_subquery, true, "If true, subquery for IN will be executed on every follower replica.", 0) \
M(Float, parallel_replicas_single_task_marks_count_multiplier, 2, "A multiplier which will be added during calculation for minimal number of marks to retrieve from coordinator. This will be applied only for remote replicas.", 0) \
M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
@ -1053,6 +1052,7 @@ class IColumn;
MAKE_OBSOLETE(M, Bool, query_plan_optimize_primary_key, true) \
MAKE_OBSOLETE(M, Bool, optimize_monotonous_functions_in_order_by, false) \
MAKE_OBSOLETE(M, UInt64, http_max_chunk_size, 100_GiB) \
MAKE_OBSOLETE(M, ParallelReplicasCustomKeyFilterType, parallel_replicas_custom_key_filter_type, ParallelReplicasCustomKeyFilterType::DEFAULT) \
/** The section above is for obsolete settings. Do not add anything there. */

262
src/Core/SettingsChangesHistory.cpp
View File

@ -86,6 +86,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"join_to_sort_minimum_perkey_rows", 0, 40, "The lower limit of per-key average rows in the right table to determine whether to rerange the right table by key in left or inner join. This setting ensures that the optimization is not applied for sparse table keys"},
{"join_to_sort_maximum_table_rows", 0, 10000, "The maximum number of rows in the right table to determine whether to rerange the right table by key in left or inner join"},
{"allow_experimental_join_right_table_sorting", false, false, "If it is set to true, and the conditions of `join_to_sort_minimum_perkey_rows` and `join_to_sort_maximum_table_rows` are met, rerange the right table by key to improve the performance in left or inner hash join"}
{"enable_parallel_replicas", false, false, "Parallel replicas with read tasks became the Beta tier feature."},
{"parallel_replicas_mode", "read_tasks", "read_tasks", "This setting was introduced as a part of making parallel replicas feature Beta"},
}
},
{"24.8",
@ -528,266 +530,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}
}
},
{"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
{"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
{"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
{"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
{"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
{"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
{"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
{"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
{"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
{"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
{"collect_hash_table_stats_during_joins", false, true, "New setting."},
{"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
{"input_format_orc_reader_time_zone_name", "GMT", "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT."},
{"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
{"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
{"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
{"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
{"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
{"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
{"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
{"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
{"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
{"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
{"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
{"use_parallel_replicas", false, false, "Parallel replicas with read tasks became the Beta tier feature."},
}},
{"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
{"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
{"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
{"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
{"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
{"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
{"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
{"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
{"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
{"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
{"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
{"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
{"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
{"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
{"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
{"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
{"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
{"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
{"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
{"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
{"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
{"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
{"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
{"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
{"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
{"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
{"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
{"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
{"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
}},
{"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
{"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
{"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
{"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
{"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
{"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
{"http_max_chunk_size", 0, 0, "Internal limitation"},
{"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
{"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
{"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
{"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
{"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
}},
{"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
{"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
{"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
{"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
{"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
{"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
{"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
{"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
{"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
{"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
{"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
{"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
{"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
}},
{"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
{"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
{"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
{"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
{"page_cache_inject_eviction", false, false, "Added userspace page cache"},
{"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
{"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
{"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
{"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
{"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
{"log_processors_profiles", false, true, "Enable by default"},
{"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
{"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
{"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
{"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
{"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
{"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
{"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
{"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
{"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
{"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
{"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
{"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
{"allow_get_client_http_header", false, false, "Introduced a new function."},
{"output_format_pretty_row_numbers", false, true, "It is better for usability."},
{"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
{"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
{"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
{"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
{"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
{"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
{"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
{"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
{"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
{"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
{"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
{"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
{"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
{"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
{"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
{"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
{"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
}},
{"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
{"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
{"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
{"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
{"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
{"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
{"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
{"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
{"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
{"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
{"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
{"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
{"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
{"format_template_row_format", "", "", "Template row format string can be set directly in query"},
{"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
{"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
{"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
{"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
{"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
{"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
{"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
{"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
{"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
{"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
{"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
{"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
{"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
{"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
}},
{"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
{"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
{"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
{"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
{"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
{"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
{"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
{"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
{"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
{"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
{"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
{"enable_vertical_final", false, true, "Use vertical final by default"},
{"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
{"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
{"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
{"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
{"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
{"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
{"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
{"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
{"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
{"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
{"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
{"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
{"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
{"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
{"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
{"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
{"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
{"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
{"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
{"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
{"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
{"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
{"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
{"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
{"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
{"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
{"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
{"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
{"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
{"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
{"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
{"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
{"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
{"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
{"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
{"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
{"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
{"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
{"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
{"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
{"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
{"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
{"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
{"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
{"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
{"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
{"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
{"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
{"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
{"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
{"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
{"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
{"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
{"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
{"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
{"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
{"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
{"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
{"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
{"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
{"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
{"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
{"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
{"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
{"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
{"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
{"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
{"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
{"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
{"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
{"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
{"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
{"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
{"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
{"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
{"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
{"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
{"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
{"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
{"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
{"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
{"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
{"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
{"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
{"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing UID of the table in its CREATE query for Engine=Atomic"}}},
{"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
{"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
{"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
{"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
{"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
{"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
{"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
{"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
};

8
src/Core/SettingsEnums.cpp
View File

@ -168,7 +168,6 @@ IMPLEMENT_SETTING_ENUM(Dialect, ErrorCodes::BAD_ARGUMENTS,
{"kusto", Dialect::kusto},
{"prql", Dialect::prql}})
IMPLEMENT_SETTING_ENUM(ParallelReplicasCustomKeyFilterType, ErrorCodes::BAD_ARGUMENTS,
{{"default", ParallelReplicasCustomKeyFilterType::DEFAULT},
{"range", ParallelReplicasCustomKeyFilterType::RANGE}})
@ -183,6 +182,13 @@ IMPLEMENT_SETTING_ENUM(DeduplicateMergeProjectionMode, ErrorCodes::BAD_ARGUMENTS
{"drop", DeduplicateMergeProjectionMode::DROP},
{"rebuild", DeduplicateMergeProjectionMode::REBUILD}})
IMPLEMENT_SETTING_ENUM(ParallelReplicasMode, ErrorCodes::BAD_ARGUMENTS,
{{"auto", ParallelReplicasMode::AUTO},
{"read_tasks", ParallelReplicasMode::READ_TASKS},
{"custom_key_sampling", ParallelReplicasMode::CUSTOM_KEY_SAMPLING},
{"custom_key_range", ParallelReplicasMode::CUSTOM_KEY_RANGE},
{"sampling_key", ParallelReplicasMode::SAMPLING_KEY}})
IMPLEMENT_SETTING_AUTO_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS)
IMPLEMENT_SETTING_ENUM(ParquetVersion, ErrorCodes::BAD_ARGUMENTS,

9
src/Core/SettingsEnums.h
View File

@ -7,6 +7,7 @@
#include <Core/SchemaInferenceMode.h>
#include <Core/SettingsFields.h>
#include <Core/ShortCircuitFunctionEvaluation.h>
#include <Core/ParallelReplicasMode.h>
#include <Formats/FormatSettings.h>
#include <IO/ReadSettings.h>
#include <Parsers/ASTSQLSecurity.h>
@ -300,12 +301,6 @@ enum class Dialect : uint8_t
DECLARE_SETTING_ENUM(Dialect)
enum class ParallelReplicasCustomKeyFilterType : uint8_t
{
DEFAULT,
RANGE,
};
DECLARE_SETTING_ENUM(ParallelReplicasCustomKeyFilterType)
enum class LightweightMutationProjectionMode : uint8_t
@ -326,6 +321,8 @@ enum class DeduplicateMergeProjectionMode : uint8_t
DECLARE_SETTING_ENUM(DeduplicateMergeProjectionMode)
DECLARE_SETTING_ENUM(ParallelReplicasMode)
DECLARE_SETTING_ENUM(LocalFSReadMethod)
enum class ObjectStorageQueueMode : uint8_t

14
src/Interpreters/ClusterProxy/executeQuery.cpp
View File

@ -164,7 +164,7 @@ ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & cluster,
}
}
if (disable_parallel_replicas)
new_settings.use_parallel_replicas = 0;
new_settings.enable_parallel_replicas = 0;
}
if (settings.max_execution_time_leaf.value > 0)
@ -242,7 +242,7 @@ getShardFilterGeneratorForCustomKey(const Cluster & cluster, ContextPtr context,
return [my_custom_key_ast = std::move(custom_key_ast),
column_description = columns,
custom_key_type = settings.parallel_replicas_custom_key_filter_type.value,
custom_key_type = settings.parallel_replicas_mode.value,
custom_key_range_lower = settings.parallel_replicas_custom_key_range_lower.value,
custom_key_range_upper = settings.parallel_replicas_custom_key_range_upper.value,
query_context = context,
@ -288,9 +288,9 @@ void executeQuery(
auto cluster = query_info.getCluster();
auto new_context = updateSettingsAndClientInfoForCluster(*cluster, is_remote_function, context,
settings, main_table, query_info.additional_filter_ast, log, &distributed_settings);
if (context->getSettingsRef().use_parallel_replicas
&& context->getSettingsRef().use_parallel_replicas.value
!= new_context->getSettingsRef().use_parallel_replicas.value)
if (context->getSettingsRef().enable_parallel_replicas
&& context->getSettingsRef().enable_parallel_replicas.value
!= new_context->getSettingsRef().enable_parallel_replicas.value)
{
LOG_TRACE(
log,
@ -462,14 +462,14 @@ void executeQueryWithParallelReplicas(
{
LOG_WARNING(
getLogger("executeQueryWithParallelReplicas"),
"Setting 'use_hedged_requests' explicitly with enabled 'use_parallel_replicas' has no effect. "
"Setting 'use_hedged_requests' explicitly with enabled 'enable_parallel_replicas' has no effect. "
"Hedged connections are not used for parallel reading from replicas");
}
else
{
LOG_INFO(
getLogger("executeQueryWithParallelReplicas"),
"Disabling 'use_hedged_requests' in favor of 'use_parallel_replicas'. Hedged connections are "
"Disabling 'use_hedged_requests' in favor of 'enable_parallel_replicas'. Hedged connections are "
"not used for parallel reading from replicas");
}

44
src/Interpreters/Context.cpp
View File

@ -5625,24 +5625,11 @@ std::shared_ptr<AsyncReadCounters> Context::getAsyncReadCounters() const
return async_read_counters;
}
Context::ParallelReplicasMode Context::getParallelReplicasMode() const
{
const auto & settings_ref = getSettingsRef();
using enum Context::ParallelReplicasMode;
if (!settings_ref.parallel_replicas_custom_key.value.empty())
return CUSTOM_KEY;
if (settings_ref.use_parallel_replicas > 0)
return READ_TASKS;
return SAMPLE_KEY;
}
bool Context::canUseTaskBasedParallelReplicas() const
{
const auto & settings_ref = getSettingsRef();
return getParallelReplicasMode() == ParallelReplicasMode::READ_TASKS && settings_ref.max_parallel_replicas > 1;
return settings->enable_parallel_replicas > 0
&& settings->parallel_replicas_mode == ParallelReplicasMode::READ_TASKS
&& settings->max_parallel_replicas > 1;
}
bool Context::canUseParallelReplicasOnInitiator() const
@ -5657,7 +5644,13 @@ bool Context::canUseParallelReplicasOnFollower() const
bool Context::canUseParallelReplicasCustomKey() const
{
return settings->max_parallel_replicas > 1 && getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY;
const bool has_enough_servers = settings->max_parallel_replicas > 1;
const bool parallel_replicas_enabled = settings->enable_parallel_replicas > 0;
const bool is_parallel_replicas_with_custom_key =
settings->parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_SAMPLING ||
settings->parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_RANGE;
return has_enough_servers && parallel_replicas_enabled && is_parallel_replicas_with_custom_key;
}
bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster) const
@ -5667,8 +5660,21 @@ bool Context::canUseParallelReplicasCustomKeyForCluster(const Cluster & cluster)
bool Context::canUseOffsetParallelReplicas() const
{
return offset_parallel_replicas_enabled && settings->max_parallel_replicas > 1
&& getParallelReplicasMode() != Context::ParallelReplicasMode::READ_TASKS;
/**
* Offset parallel replicas algorithm is not only the one which relies on native SAMPLING KEY,
* but also those which rely on customer-provided "custom" key.
* We combine them together into one group for convenience.
*/
const bool has_enough_servers = settings->max_parallel_replicas > 1;
const bool parallel_replicas_enabled = settings->enable_parallel_replicas > 0;
const bool is_parallel_replicas_with_custom_key_or_native_sampling_key =
settings->parallel_replicas_mode == ParallelReplicasMode::SAMPLING_KEY ||
settings->parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_SAMPLING ||
settings->parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_RANGE;
return offset_parallel_replicas_enabled &&
has_enough_servers &&
parallel_replicas_enabled &&
is_parallel_replicas_with_custom_key_or_native_sampling_key;
}
void Context::disableOffsetParallelReplicas()

10
src/Interpreters/Context.h
View File

@ -12,6 +12,7 @@
#include <Common/SharedMutexHelper.h>
#include <Core/NamesAndTypes.h>
#include <Core/UUID.h>
#include <Core/ParallelReplicasMode.h>
#include <Formats/FormatSettings.h>
#include <IO/AsyncReadCounters.h>
#include <IO/ReadSettings.h>
@ -1343,15 +1344,6 @@ public:
ClusterPtr getClusterForParallelReplicas() const;
enum class ParallelReplicasMode : uint8_t
{
SAMPLE_KEY,
CUSTOM_KEY,
READ_TASKS,
};
ParallelReplicasMode getParallelReplicasMode() const;
void setPreparedSetsCache(const PreparedSetsCachePtr & cache);
PreparedSetsCachePtr getPreparedSetsCache() const;

12
src/Interpreters/GlobalSubqueriesVisitor.h
View File

@ -219,13 +219,13 @@ private:
/// We don't enable parallel replicas for IN (subquery)
if (!settings.parallel_replicas_allow_in_with_subquery && ast->as<ASTSubquery>())
{
if (settings.use_parallel_replicas == 1)
if (settings.enable_parallel_replicas == 1)
{
LOG_DEBUG(getLogger("GlobalSubqueriesMatcher"), "IN with subquery is not supported with parallel replicas");
data.getContext()->getQueryContext()->setSetting("use_parallel_replicas", Field(0));
data.getContext()->getQueryContext()->setSetting("enable_parallel_replicas", Field(0));
return;
}
else if (settings.use_parallel_replicas >= 2)
else if (settings.enable_parallel_replicas >= 2)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "IN with subquery is not supported with parallel replicas");
}
}
@ -277,13 +277,13 @@ private:
if (!is_subquery)
{
if (settings.use_parallel_replicas == 1)
if (settings.enable_parallel_replicas == 1)
{
LOG_DEBUG(getLogger("GlobalSubqueriesMatcher"), "JOIN with parallel replicas is only supported with subqueries");
data.getContext()->getQueryContext()->setSetting("use_parallel_replicas", Field(0));
data.getContext()->getQueryContext()->setSetting("enable_parallel_replicas", Field(0));
return;
}
else if (settings.use_parallel_replicas >= 2)
else if (settings.enable_parallel_replicas >= 2)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JOIN with parallel replicas is only supported with subqueries");
}
}

34
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -504,12 +504,12 @@ InterpreterSelectQuery::InterpreterSelectQuery(
bool is_query_with_final = isQueryWithFinal(query_info);
if (is_query_with_final && context->canUseTaskBasedParallelReplicas())
{
if (settings.use_parallel_replicas == 1)
if (settings.enable_parallel_replicas == 1)
{
LOG_DEBUG(log, "FINAL modifier is not supported with parallel replicas. Query will be executed without using them.");
context->setSetting("use_parallel_replicas", Field(0));
context->setSetting("enable_parallel_replicas", Field(0));
}
else if (settings.use_parallel_replicas >= 2)
else if (settings.enable_parallel_replicas >= 2)
{
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "FINAL modifier is not supported with parallel replicas");
}
@ -517,14 +517,14 @@ InterpreterSelectQuery::InterpreterSelectQuery(
/// Check support for parallel replicas for non-replicated storage (plain MergeTree)
bool is_plain_merge_tree = storage && storage->isMergeTree() && !storage->supportsReplication();
if (is_plain_merge_tree && settings.use_parallel_replicas > 0 && !settings.parallel_replicas_for_non_replicated_merge_tree)
if (is_plain_merge_tree && settings.enable_parallel_replicas > 0 && !settings.parallel_replicas_for_non_replicated_merge_tree)
{
if (settings.use_parallel_replicas == 1)
if (settings.enable_parallel_replicas == 1)
{
LOG_DEBUG(log, "To use parallel replicas with plain MergeTree tables please enable setting `parallel_replicas_for_non_replicated_merge_tree`. For now query will be executed without using them.");
context->setSetting("use_parallel_replicas", Field(0));
context->setSetting("enable_parallel_replicas", Field(0));
}
else if (settings.use_parallel_replicas >= 2)
else if (settings.enable_parallel_replicas >= 2)
{
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "To use parallel replicas with plain MergeTree tables please enable setting `parallel_replicas_for_non_replicated_merge_tree`");
}
@ -598,7 +598,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(
settings.parallel_replicas_count,
settings.parallel_replica_offset,
std::move(custom_key_ast),
{settings.parallel_replicas_custom_key_filter_type,
{settings.parallel_replicas_mode,
settings.parallel_replicas_custom_key_range_lower,
settings.parallel_replicas_custom_key_range_upper},
storage->getInMemoryMetadataPtr()->columns,
@ -814,10 +814,10 @@ InterpreterSelectQuery::InterpreterSelectQuery(
};
/// This is a hack to make sure we reanalyze if GlobalSubqueriesVisitor changed use_parallel_replicas
/// This is a hack to make sure we reanalyze if GlobalSubqueriesVisitor changed enable_parallel_replicas
/// inside the query context (because it doesn't have write access to the main context)
UInt64 parallel_replicas_before_analysis
= context->hasQueryContext() ? context->getQueryContext()->getSettingsRef().use_parallel_replicas : 0;
= context->hasQueryContext() ? context->getQueryContext()->getSettingsRef().enable_parallel_replicas : 0;
/// Conditionally support AST-based PREWHERE optimization.
analyze(shouldMoveToPrewhere() && (!settings.query_plan_optimize_prewhere || !settings.query_plan_enable_optimizations));
@ -829,10 +829,10 @@ InterpreterSelectQuery::InterpreterSelectQuery(
if (context->hasQueryContext())
{
/// As this query can't be executed with parallel replicas, we must reanalyze it
if (context->getQueryContext()->getSettingsRef().use_parallel_replicas
if (context->getQueryContext()->getSettingsRef().enable_parallel_replicas
!= parallel_replicas_before_analysis)
{
context->setSetting("use_parallel_replicas", Field(0));
context->setSetting("enable_parallel_replicas", Field(0));
context->setSetting("max_parallel_replicas", UInt64{1});
need_analyze_again = true;
}
@ -924,7 +924,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis()
if (getTrivialCount(0).has_value())
{
/// The query could use trivial count if it didn't use parallel replicas, so let's disable it and reanalyze
context->setSetting("use_parallel_replicas", Field(0));
context->setSetting("enable_parallel_replicas", Field(0));
context->setSetting("max_parallel_replicas", UInt64{1});
LOG_DEBUG(log, "Disabling parallel replicas to be able to use a trivial count optimization");
return true;
@ -980,7 +980,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis()
if (number_of_replicas_to_use <= 1)
{
context->setSetting("use_parallel_replicas", Field(0));
context->setSetting("enable_parallel_replicas", Field(0));
context->setSetting("max_parallel_replicas", UInt64{1});
LOG_DEBUG(log, "Disabling parallel replicas because there aren't enough rows to read");
return true;
@ -2319,12 +2319,12 @@ void InterpreterSelectQuery::addPrewhereAliasActions()
}
/// Based on the query analysis, check if using a trivial count (storage or partition metadata) is possible
std::optional<UInt64> InterpreterSelectQuery::getTrivialCount(UInt64 max_parallel_replicas)
std::optional<UInt64> InterpreterSelectQuery::getTrivialCount(UInt64 enable_parallel_replicas)
{
const Settings & settings = context->getSettingsRef();
bool optimize_trivial_count =
syntax_analyzer_result->optimize_trivial_count
&& (max_parallel_replicas <= 1)
&& (enable_parallel_replicas == 0)
&& !settings.allow_experimental_query_deduplication
&& !settings.empty_result_for_aggregation_by_empty_set
&& storage
@ -2408,7 +2408,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
std::optional<UInt64> num_rows;
/// Optimization for trivial query like SELECT count() FROM table.
if (processing_stage == QueryProcessingStage::FetchColumns && (num_rows = getTrivialCount(settings.max_parallel_replicas)))
if (processing_stage == QueryProcessingStage::FetchColumns && (num_rows = getTrivialCount(settings.enable_parallel_replicas)))
{
const auto & desc = query_analyzer->aggregates()[0];
const auto & func = desc.function;

2
src/Interpreters/InterpreterSelectQuery.h
View File

@ -195,7 +195,7 @@ private:
void executeExtremes(QueryPlan & query_plan);
void executeSubqueriesInSetsAndJoins(QueryPlan & query_plan);
bool autoFinalOnQuery(ASTSelectQuery & select_query);
std::optional<UInt64> getTrivialCount(UInt64 max_parallel_replicas);
std::optional<UInt64> getTrivialCount(UInt64 enable_parallel_replicas);
/// Check if we can limit block size to read based on LIMIT clause
UInt64 maxBlockSizeByLimit() const;

5
src/Interpreters/getCustomKeyFilterForParallelReplicas.cpp
View File

@ -31,7 +31,8 @@ ASTPtr getCustomKeyFilterForParallelReplica(
const ContextPtr & context)
{
chassert(replicas_count > 1);
if (filter.filter_type == ParallelReplicasCustomKeyFilterType::DEFAULT)
chassert(filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_SAMPLING || filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_RANGE);
if (filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_SAMPLING)
{
// first we do modulo with replica count
auto modulo_function = makeASTFunction("positiveModulo", custom_key_ast, std::make_shared<ASTLiteral>(replicas_count));
@ -42,7 +43,7 @@ ASTPtr getCustomKeyFilterForParallelReplica(
return equals_function;
}
chassert(filter.filter_type == ParallelReplicasCustomKeyFilterType::RANGE);
chassert(filter.filter_type == ParallelReplicasMode::CUSTOM_KEY_RANGE);
KeyDescription custom_key_description
= KeyDescription::getKeyFromAST(custom_key_ast, columns, context);

2
src/Interpreters/getCustomKeyFilterForParallelReplicas.h
View File

@ -12,7 +12,7 @@ namespace DB
{
struct ParallelReplicasCustomKeyFilter
{
ParallelReplicasCustomKeyFilterType filter_type;
ParallelReplicasMode filter_type;
UInt64 range_lower;
UInt64 range_upper;
};

12
src/Planner/Planner.cpp
View File

@ -1424,11 +1424,11 @@ void Planner::buildPlanForQueryNode()
{
if (!settings.parallel_replicas_allow_in_with_subquery && planner_context->getPreparedSets().hasSubqueries())
{
if (settings.use_parallel_replicas >= 2)
if (settings.enable_parallel_replicas >= 2)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "IN with subquery is not supported with parallel replicas");
auto & mutable_context = planner_context->getMutableQueryContext();
mutable_context->setSetting("use_parallel_replicas", Field(0));
mutable_context->setSetting("enable_parallel_replicas", Field(0));
LOG_DEBUG(getLogger("Planner"), "Disabling parallel replicas to execute a query with IN with subquery");
}
}
@ -1463,7 +1463,7 @@ void Planner::buildPlanForQueryNode()
const auto & modifiers = table_node->getTableExpressionModifiers();
if (modifiers.has_value() && modifiers->hasFinal())
{
if (settings.use_parallel_replicas >= 2)
if (settings.enable_parallel_replicas >= 2)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "FINAL modifier is not supported with parallel replicas");
else
{
@ -1471,7 +1471,7 @@ void Planner::buildPlanForQueryNode()
getLogger("Planner"),
"FINAL modifier is not supported with parallel replicas. Query will be executed without using them.");
auto & mutable_context = planner_context->getMutableQueryContext();
mutable_context->setSetting("use_parallel_replicas", Field(0));
mutable_context->setSetting("enable_parallel_replicas", Field(0));
}
}
}
@ -1482,7 +1482,7 @@ void Planner::buildPlanForQueryNode()
/// Check support for JOIN for parallel replicas with custom key
if (planner_context->getTableExpressionNodeToData().size() > 1)
{
if (settings.use_parallel_replicas >= 2)
if (settings.enable_parallel_replicas >= 2)
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JOINs are not supported with parallel replicas");
else
{
@ -1491,7 +1491,7 @@ void Planner::buildPlanForQueryNode()
"JOINs are not supported with parallel replicas. Query will be executed without using them.");
auto & mutable_context = planner_context->getMutableQueryContext();
mutable_context->setSetting("use_parallel_replicas", Field(0));
mutable_context->setSetting("enable_parallel_replicas", Field(0));
mutable_context->setSetting("parallel_replicas_custom_key", String{""});
}
}

24
src/Planner/PlannerJoinTree.cpp
View File

@ -3,6 +3,7 @@
#include <Core/Settings.h>
#include <Common/scope_guard_safe.h>
#include <Core/ParallelReplicasMode.h>
#include <Columns/ColumnAggregateFunction.h>
@ -290,14 +291,21 @@ bool applyTrivialCountIfPossible(
if (!num_rows)
return false;
if (settings.max_parallel_replicas > 1)
if (settings.enable_parallel_replicas > 0 && settings.max_parallel_replicas > 1)
{
if (!settings.parallel_replicas_custom_key.value.empty() || settings.use_parallel_replicas == 0)
/// Imagine the situation when we have a query with parallel replicas and
/// this code executed on the remote server.
/// If we will apply trivial count optimization, then each remote server will do the same
/// and we will have N times more rows as the result on the initiator.
/// TODO: This condition seems unneeded when we will make the parallel replicas with custom key
/// to work on top of MergeTree instead of Distributed.
if (settings.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_RANGE ||
settings.parallel_replicas_mode == ParallelReplicasMode::CUSTOM_KEY_SAMPLING ||
settings.parallel_replicas_mode == ParallelReplicasMode::SAMPLING_KEY)
return false;
/// The query could use trivial count if it didn't use parallel replicas, so let's disable it
query_context->setSetting("use_parallel_replicas", Field(0));
query_context->setSetting("max_parallel_replicas", UInt64{1});
query_context->setSetting("enable_parallel_replicas", Field(0));
LOG_TRACE(getLogger("Planner"), "Disabling parallel replicas to be able to use a trivial count optimization");
}
@ -504,7 +512,7 @@ std::optional<FilterDAGInfo> buildCustomKeyFilterIfNeeded(const StoragePtr & sto
settings.parallel_replicas_count,
settings.parallel_replica_offset,
std::move(custom_key_ast),
{settings.parallel_replicas_custom_key_filter_type,
{settings.parallel_replicas_mode,
settings.parallel_replicas_custom_key_range_lower,
settings.parallel_replicas_custom_key_range_upper},
storage->getInMemoryMetadataPtr()->columns,
@ -836,6 +844,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
if (query_context->canUseParallelReplicasCustomKey())
{
LOG_DEBUG(getLogger("Planner"), "We can use parallel replicas with custom key");
if (settings.parallel_replicas_count > 1)
{
if (auto parallel_replicas_custom_key_filter_info= buildCustomKeyFilterIfNeeded(storage, table_expression_query_info, planner_context))
@ -844,6 +853,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
else if (auto * distributed = typeid_cast<StorageDistributed *>(storage.get());
distributed && query_context->canUseParallelReplicasCustomKeyForCluster(*distributed->getCluster()))
{
LOG_DEBUG(getLogger("Planner"), "We can use parallel replicas with custom key with distributed");
planner_context->getMutableQueryContext()->setSetting("distributed_group_by_no_merge", 2);
/// We disable prefer_localhost_replica because if one of the replicas is local it will create a single local plan
/// instead of executing the query with multiple replicas
@ -867,7 +877,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
= planner_context->getGlobalPlannerContext()->parallel_replicas_table
&& !table_expression_query_info.current_table_chosen_for_reading_with_parallel_replicas;
if (other_table_already_chosen_for_reading_with_parallel_replicas)
planner_context->getMutableQueryContext()->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0));
planner_context->getMutableQueryContext()->setSetting("enable_parallel_replicas", Field(0));
storage->read(
query_plan,
@ -968,7 +978,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
if (number_of_replicas_to_use <= 1)
{
planner_context->getMutableQueryContext()->setSetting(
"allow_experimental_parallel_reading_from_replicas", Field(0));
"enable_parallel_replicas", Field(0));
planner_context->getMutableQueryContext()->setSetting("max_parallel_replicas", UInt64{1});
LOG_DEBUG(getLogger("Planner"), "Disabling parallel replicas because there aren't enough rows to read");
}

2
src/Planner/findParallelReplicasQuery.cpp
View File

@ -270,7 +270,7 @@ const QueryNode * findQueryForParallelReplicas(const QueryTreeNodePtr & query_tr
/// This is needed to avoid infinite recursion.
auto mutable_context = Context::createCopy(context);
mutable_context->setSetting("use_parallel_replicas", Field(0));
mutable_context->setSetting("enable_parallel_replicas", Field(0));
/// Here we replace tables to dummy, in order to build a temporary query plan for parallel replicas analysis.
ResultReplacementMap replacement_map;

2
src/Processors/QueryPlan/ReadFromLoopStep.cpp
View File

@ -115,7 +115,7 @@ namespace DB
static ContextPtr disableParallelReplicas(ContextPtr context)
{
auto modified_context = Context::createCopy(context);
modified_context->setSetting("allow_experimental_parallel_reading_from_replicas", Field(0));
modified_context->setSetting("enable_parallel_replicas", Field(0));
return modified_context;
}

13
src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
View File

@ -277,11 +277,14 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
* It is also important that the entire universe can be covered using SAMPLE 0.1 OFFSET 0, ... OFFSET 0.9 and similar decimals.
*/
auto parallel_replicas_mode = context->getParallelReplicasMode();
const bool can_use_sampling_key_parallel_replicas =
settings.enable_parallel_replicas > 0
&& settings.max_parallel_replicas > 1
&& settings.parallel_replicas_mode == ParallelReplicasMode::SAMPLING_KEY;
/// Parallel replicas has been requested but there is no way to sample data.
/// Select all data from first replica and no data from other replicas.
if (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY
&& !data.supportsSampling() && settings.parallel_replica_offset > 0)
if (can_use_sampling_key_parallel_replicas && settings.parallel_replicas_count > 1 && !data.supportsSampling() && settings.parallel_replica_offset > 0)
{
LOG_DEBUG(
log,
@ -291,9 +294,7 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
return sampling;
}
sampling.use_sampling = relative_sample_size > 0
|| (settings.parallel_replicas_count > 1 && parallel_replicas_mode == Context::ParallelReplicasMode::SAMPLE_KEY
&& data.supportsSampling());
sampling.use_sampling = relative_sample_size > 0 || (can_use_sampling_key_parallel_replicas && settings.parallel_replicas_count > 1 && data.supportsSampling());
bool no_data = false; /// There is nothing left after sampling.
if (sampling.use_sampling)

2
tests/ci/stress.py
View File

@ -78,7 +78,7 @@ def get_options(i: int, upgrade_check: bool) -> str:
client_options.append("ignore_drop_queries_probability=0.5")
if random.random() < 0.2:
client_options.append("allow_experimental_parallel_reading_from_replicas=1")
client_options.append("enable_parallel_replicas=1")
client_options.append("max_parallel_replicas=3")
client_options.append("cluster_for_parallel_replicas='parallel_replicas'")
client_options.append("parallel_replicas_for_non_replicated_merge_tree=1")

2
tests/docker_scripts/stateful_runner.sh
View File

@ -243,7 +243,7 @@ function run_tests()
)
if [[ -n "$USE_PARALLEL_REPLICAS" ]] && [[ "$USE_PARALLEL_REPLICAS" -eq 1 ]]; then
TEST_ARGS+=(
--client="clickhouse-client --use_parallel_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 --max_parallel_replicas=100 --cluster_for_parallel_replicas='parallel_replicas'"
--client="clickhouse-client --enable_parallel_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 --max_parallel_replicas=100 --cluster_for_parallel_replicas='parallel_replicas'"
--no-parallel-replicas
)
fi

4
tests/integration/test_parallel_replicas_all_marks_read/test.py
View File

@ -54,7 +54,7 @@ def _get_result_without_parallel_replicas(query):
return nodes[0].query(
query,
settings={
"use_parallel_replicas": 0,
"enable_parallel_replicas": 0,
},
)
@ -65,7 +65,7 @@ def _get_result_with_parallel_replicas(
return nodes[0].query(
query,
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 6,
"cluster_for_parallel_replicas": f"{cluster_name}",
"parallel_replicas_mark_segment_size": parallel_replicas_mark_segment_size,

25
tests/integration/test_parallel_replicas_custom_key/test.py
View File

@ -37,13 +37,13 @@ def insert_data(table_name, row_num, all_nodes=False):
@pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"])
@pytest.mark.parametrize("filter_type", ["default", "range"])
@pytest.mark.parametrize("parallel_replicas_mode", ["custom_key_sampling"])
@pytest.mark.parametrize(
"cluster",
["test_multiple_shards_multiple_replicas", "test_single_shard_multiple_replicas"],
)
def test_parallel_replicas_custom_key_distributed(
start_cluster, cluster, custom_key, filter_type
start_cluster, cluster, custom_key, parallel_replicas_mode
):
for node in nodes:
node.rotate_logs()
@ -84,7 +84,8 @@ def test_parallel_replicas_custom_key_distributed(
settings={
"max_parallel_replicas": 4,
"parallel_replicas_custom_key": custom_key,
"parallel_replicas_custom_key_filter_type": filter_type,
"enable_parallel_replicas": 1,
"parallel_replicas_mode": parallel_replicas_mode,
"prefer_localhost_replica": 0,
},
)
@ -100,13 +101,15 @@ def test_parallel_replicas_custom_key_distributed(
@pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"])
@pytest.mark.parametrize("filter_type", ["default", "range"])
@pytest.mark.parametrize(
"parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"]
)
@pytest.mark.parametrize(
"cluster",
["test_single_shard_multiple_replicas"],
)
def test_parallel_replicas_custom_key_mergetree(
start_cluster, cluster, custom_key, filter_type
start_cluster, cluster, custom_key, parallel_replicas_mode
):
for node in nodes:
node.rotate_logs()
@ -130,8 +133,9 @@ def test_parallel_replicas_custom_key_mergetree(
"SELECT key, count() FROM test_table_for_mt GROUP BY key ORDER BY key",
settings={
"max_parallel_replicas": 4,
"enable_parallel_replicas": 1,
"parallel_replicas_custom_key": custom_key,
"parallel_replicas_custom_key_filter_type": filter_type,
"parallel_replicas_mode": parallel_replicas_mode,
"parallel_replicas_for_non_replicated_merge_tree": 1,
"cluster_for_parallel_replicas": cluster,
},
@ -141,13 +145,15 @@ def test_parallel_replicas_custom_key_mergetree(
@pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"])
@pytest.mark.parametrize("filter_type", ["default", "range"])
@pytest.mark.parametrize(
"parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"]
)
@pytest.mark.parametrize(
"cluster",
["test_single_shard_multiple_replicas"],
)
def test_parallel_replicas_custom_key_replicatedmergetree(
start_cluster, cluster, custom_key, filter_type
start_cluster, cluster, custom_key, parallel_replicas_mode
):
for node in nodes:
node.rotate_logs()
@ -174,8 +180,9 @@ def test_parallel_replicas_custom_key_replicatedmergetree(
"SELECT key, count() FROM test_table_for_rmt GROUP BY key ORDER BY key",
settings={
"max_parallel_replicas": 4,
"enable_parallel_replicas": 1,
"parallel_replicas_custom_key": custom_key,
"parallel_replicas_custom_key_filter_type": filter_type,
"parallel_replicas_mode": parallel_replicas_mode,
"cluster_for_parallel_replicas": cluster,
},
)

9
tests/integration/test_parallel_replicas_custom_key_failover/test.py
View File

@ -52,13 +52,15 @@ def create_tables(cluster, table_name):
@pytest.mark.parametrize("use_hedged_requests", [1, 0])
@pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"])
@pytest.mark.parametrize("filter_type", ["default", "range"])
@pytest.mark.parametrize(
"parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"]
)
@pytest.mark.parametrize("prefer_localhost_replica", [0, 1])
def test_parallel_replicas_custom_key_failover(
start_cluster,
use_hedged_requests,
custom_key,
filter_type,
parallel_replicas_mode,
prefer_localhost_replica,
):
cluster_name = "test_single_shard_multiple_replicas"
@ -77,8 +79,9 @@ def test_parallel_replicas_custom_key_failover(
settings={
"log_comment": log_comment,
"max_parallel_replicas": 4,
"enable_parallel_replicas": 1,
"parallel_replicas_custom_key": custom_key,
"parallel_replicas_custom_key_filter_type": filter_type,
"parallel_replicas_mode": parallel_replicas_mode,
"use_hedged_requests": use_hedged_requests,
"prefer_localhost_replica": prefer_localhost_replica,
# avoid considering replica delay on connection choice

9
tests/integration/test_parallel_replicas_custom_key_load_balancing/test.py
View File

@ -56,12 +56,14 @@ def create_tables(table_name):
@pytest.mark.parametrize("use_hedged_requests", [1, 0])
@pytest.mark.parametrize("custom_key", ["sipHash64(key)", "key"])
@pytest.mark.parametrize("filter_type", ["default", "range"])
@pytest.mark.parametrize(
"parallel_replicas_mode", ["custom_key_sampling", "custom_key_range"]
)
def test_parallel_replicas_custom_key_load_balancing(
start_cluster,
use_hedged_requests,
custom_key,
filter_type,
parallel_replicas_mode,
):
cluster_name = "test_single_shard_multiple_replicas"
table = "test_table"
@ -80,8 +82,9 @@ def test_parallel_replicas_custom_key_load_balancing(
"log_comment": log_comment,
"prefer_localhost_replica": 0,
"max_parallel_replicas": 4,
"enable_parallel_replicas": 1,
"parallel_replicas_custom_key": custom_key,
"parallel_replicas_custom_key_filter_type": filter_type,
"parallel_replicas_mode": parallel_replicas_mode,
"use_hedged_requests": use_hedged_requests,
# avoid considering replica delay on connection choice
# otherwise connection can be not distributed evenly among available nodes

6
tests/integration/test_parallel_replicas_distributed_skip_shards/test.py
View File

@ -85,7 +85,7 @@ def test_skip_unavailable_shards(start_cluster, prefer_localhost_replica):
node1.query(
f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d",
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 3,
"prefer_localhost_replica": prefer_localhost_replica,
"skip_unavailable_shards": 1,
@ -119,7 +119,7 @@ def test_error_on_unavailable_shards(start_cluster, prefer_localhost_replica):
node1.query(
f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d",
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 3,
"prefer_localhost_replica": prefer_localhost_replica,
"skip_unavailable_shards": 0,
@ -154,7 +154,7 @@ def test_no_unavailable_shards(start_cluster, skip_unavailable_shards):
node1.query(
f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d",
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 3,
"prefer_localhost_replica": 0,
"skip_unavailable_shards": skip_unavailable_shards,

7
tests/integration/test_parallel_replicas_failover/test.py
View File

@ -1,5 +1,6 @@
import pytest
import uuid
from helpers.cluster import ClickHouseCluster
cluster = ClickHouseCluster(__file__)
@ -68,7 +69,7 @@ def test_skip_replicas_without_table(start_cluster):
node1.query(
f"SELECT key, count() FROM {table_name} GROUP BY key ORDER BY key",
settings={
"allow_experimental_parallel_reading_from_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 3,
"cluster_for_parallel_replicas": cluster_name,
"log_comment": log_comment,
@ -80,7 +81,7 @@ def test_skip_replicas_without_table(start_cluster):
node1.query("SYSTEM FLUSH LOGS")
assert (
node1.query(
f"SELECT ProfileEvents['DistributedConnectionMissingTable'], ProfileEvents['ParallelReplicasUnavailableCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '{log_comment}' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS allow_experimental_parallel_reading_from_replicas=0"
f"SELECT ProfileEvents['DistributedConnectionMissingTable'], ProfileEvents['ParallelReplicasUnavailableCount'] FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '{log_comment}' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS enable_parallel_replicas=0"
)
== "1\t1\n"
)
@ -103,7 +104,7 @@ def test_skip_unresponsive_replicas(start_cluster):
node1.query(
f"SELECT key, count() FROM {table_name} GROUP BY key ORDER BY key",
settings={
"allow_experimental_parallel_reading_from_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 3,
"cluster_for_parallel_replicas": cluster_name,
},

2
tests/integration/test_parallel_replicas_invisible_parts/test.py
View File

@ -80,7 +80,7 @@ def _get_result_with_parallel_replicas(
return nodes[0].query(
query,
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": len(nodes),
"cluster_for_parallel_replicas": f"{cluster_name}",
"parallel_replicas_mark_segment_size": parallel_replicas_mark_segment_size,

2
tests/integration/test_parallel_replicas_no_replicas/test.py
View File

@ -45,7 +45,7 @@ def test_skip_all_replicas(
initiator.query(
f"SELECT key, count() FROM {table_name} GROUP BY key ORDER BY key",
settings={
"allow_experimental_parallel_reading_from_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": max_parallel_replicas,
"cluster_for_parallel_replicas": cluster_name,
"skip_unavailable_shards": skip_unavailable_shards,

4
tests/integration/test_parallel_replicas_over_distributed/test.py
View File

@ -137,7 +137,7 @@ def test_parallel_replicas_over_distributed(
node.query(
f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d",
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"prefer_localhost_replica": prefer_localhost_replica,
"max_parallel_replicas": max_parallel_replicas,
},
@ -150,7 +150,7 @@ def test_parallel_replicas_over_distributed(
node.query(
f"SELECT count(), min(key), max(key), sum(key) FROM {table_name}_d",
settings={
"use_parallel_replicas": 0,
"enable_parallel_replicas": 0,
},
)
== expected_result

8
tests/integration/test_parallel_replicas_skip_shards/test.py
View File

@ -25,7 +25,7 @@ def test_skip_unavailable_shards(start_cluster):
node1.query(
"SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h",
settings={
"use_parallel_replicas": 0,
"enable_parallel_replicas": 0,
"skip_unavailable_shards": 1,
},
)
@ -36,7 +36,7 @@ def test_skip_unavailable_shards(start_cluster):
node1.query(
"SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h",
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 3,
"skip_unavailable_shards": 1,
# "async_socket_for_remote" : 0,
@ -53,7 +53,7 @@ def test_error_on_unavailable_shards(start_cluster):
node1.query(
"SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h",
settings={
"use_parallel_replicas": 0,
"enable_parallel_replicas": 0,
"skip_unavailable_shards": 0,
},
)
@ -62,7 +62,7 @@ def test_error_on_unavailable_shards(start_cluster):
node1.query(
"SELECT hostName() as h FROM clusterAllReplicas('two_shards', system.one) order by h",
settings={
"use_parallel_replicas": 2,
"enable_parallel_replicas": 2,
"max_parallel_replicas": 3,
"skip_unavailable_shards": 0,
},

2
tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml
View File

@ -2,7 +2,7 @@
<profiles>
<default>
<enable_analyzer>1</enable_analyzer>
<allow_experimental_parallel_reading_from_replicas>1</allow_experimental_parallel_reading_from_replicas>
<enable_parallel_replicas>1</enable_parallel_replicas>
<cluster_for_parallel_replicas>default</cluster_for_parallel_replicas>
<max_parallel_replicas>100</max_parallel_replicas>
<use_hedged_requests>0</use_hedged_requests>

4
tests/queries/0_stateless/00124_shard_distributed_with_many_replicas.sql
View File

@ -1,7 +1,9 @@
-- Tags: replica, distributed
SET use_parallel_replicas = 0;
SET enable_parallel_replicas = 1;
SET parallel_replicas_mode = 'sampling_key';
SET max_parallel_replicas = 2;
SET parallel_replicas_for_non_replicated_merge_tree = 1;
DROP TABLE IF EXISTS report;
set allow_deprecated_syntax_for_merge_tree=1;

14
tests/queries/0_stateless/00193_parallel_replicas.sql
View File

@ -13,12 +13,14 @@ INSERT INTO parallel_replicas (x, u, s) VALUES (41, 42, 'U'),(43, 44, 'V'),(45,
INSERT INTO parallel_replicas (x, u, s) VALUES (51, 52, 'Z');
/*
* Проверяем, что:
* - на каждой реплике таблица не пустая;
* - объединение данных всех реплик совпадает с содержанием таблицы parallel_replicas.
*/
* Check that:
* - the table is not empty on each replica;
* - combining the data of all replicas coincides with the contents of the parallel_replicas table.
*/
/* Две реплики */
/* Two replicas */
SET enable_parallel_replicas=1, parallel_replicas_mode='sampling_key', max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree = 1;
CREATE TABLE parallel_replicas_backup(d Date DEFAULT today(), x UInt32, u UInt64, s String) ENGINE = Memory;
@ -38,7 +40,7 @@ SELECT x, u, s FROM parallel_replicas_backup ORDER BY x, u, s ASC;
DROP TABLE parallel_replicas_backup;
CREATE TABLE parallel_replicas_backup(d Date DEFAULT today(), x UInt32, u UInt64, s String) ENGINE = Memory;
/* Три реплики */
/* Three replicas */
SET parallel_replicas_count = 3;

5
tests/queries/0_stateless/01034_prewhere_max_parallel_replicas_distributed.sql
View File

@ -1,7 +1,5 @@
-- Tags: replica, distributed
set use_parallel_replicas=0;
drop table if exists test_max_parallel_replicas_lr;
-- If you wonder why the table is named with "_lr" suffix in this test.
@ -10,7 +8,10 @@ drop table if exists test_max_parallel_replicas_lr;
CREATE TABLE test_max_parallel_replicas_lr (timestamp UInt64) ENGINE = MergeTree ORDER BY (intHash32(timestamp)) SAMPLE BY intHash32(timestamp);
INSERT INTO test_max_parallel_replicas_lr select number as timestamp from system.numbers limit 100;
SET enable_parallel_replicas = 1;
SET parallel_replicas_mode='sampling_key';
SET max_parallel_replicas = 2;
SET parallel_replicas_for_non_replicated_merge_tree = 1;
select count() FROM remote('127.0.0.{2|3}', currentDatabase(), test_max_parallel_replicas_lr) PREWHERE timestamp > 0;
drop table test_max_parallel_replicas_lr;

5
tests/queries/0_stateless/01034_sample_final_distributed.sql
View File

@ -1,6 +1,9 @@
-- Tags: distributed
set use_parallel_replicas = 0;
set enable_parallel_replicas = 1;
set parallel_replicas_mode = 'sampling_key';
set max_parallel_replicas = 3;
set parallel_replicas_for_non_replicated_merge_tree = 1;
drop table if exists sample_final;
create table sample_final (CounterID UInt32, EventDate Date, EventTime DateTime, UserID UInt64, Sign Int8) engine = CollapsingMergeTree(Sign) order by (CounterID, EventDate, intHash32(UserID), EventTime) sample by intHash32(UserID) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';

2
tests/queries/0_stateless/01099_parallel_distributed_insert_select.sql
View File

@ -2,7 +2,7 @@
-- set distributed_foreground_insert = 1; -- see https://github.com/ClickHouse/ClickHouse/issues/18971
SET use_parallel_replicas = 0; -- see https://github.com/ClickHouse/ClickHouse/issues/34525
SET enable_parallel_replicas = 0; -- see https://github.com/ClickHouse/ClickHouse/issues/34525
SET prefer_localhost_replica = 1;
DROP TABLE IF EXISTS local_01099_a;

2
tests/queries/0_stateless/01517_select_final_distributed.sql
View File

@ -1,6 +1,6 @@
-- Tags: distributed
SET use_parallel_replicas = 0;
SET enable_parallel_replicas = 0;
DROP TABLE IF EXISTS test5346;

5
tests/queries/0_stateless/01557_max_parallel_replicas_no_sample.sql
View File

@ -1,6 +1,9 @@
-- Tags: replica
SET use_parallel_replicas=0;
SET enable_parallel_replicas=1;
SET max_parallel_replicas=3;
SET parallel_replicas_mode='sampling_key';
SET parallel_replicas_for_non_replicated_merge_tree = 1;
DROP TABLE IF EXISTS t;
CREATE TABLE t (x String) ENGINE = MergeTree ORDER BY x;

2
tests/queries/0_stateless/02221_parallel_replicas_bug.sh
View File

@ -4,4 +4,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
${CLICKHOUSE_CLIENT} --use_parallel_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 -m < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null
${CLICKHOUSE_CLIENT} --enable_parallel_replicas=1 --parallel_replicas_for_non_replicated_merge_tree=1 -m < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null

2
tests/queries/0_stateless/02226_parallel_reading_from_replicas_benchmark.sh
View File

@ -14,7 +14,7 @@ as select * from numbers(1);
#
# Logical error: 'Coordinator for parallel reading from replicas is not initialized'.
opts=(
--use_parallel_replicas 1
--enable_parallel_replicas 1
--parallel_replicas_for_non_replicated_merge_tree 1
--max_parallel_replicas 3
--cluster_for_parallel_replicas parallel_replicas

2
tests/queries/0_stateless/02404_memory_bound_merging.sql
View File

@ -63,7 +63,7 @@ drop table if exists pr_t;
create table pr_t(a UInt64, b UInt64) engine=MergeTree order by a;
insert into pr_t select number % 1000, number % 1000 from numbers_mt(1e6);
set use_parallel_replicas = 1;
set enable_parallel_replicas = 1;
set parallel_replicas_for_non_replicated_merge_tree = 1;
set max_parallel_replicas = 3;
set cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost';

96
tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.reference
View File

@ -1,176 +1,176 @@
query='SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt)' with custom_key='sipHash64(x)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
Hello
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
Hello
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
Hello
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
Hello
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
Hello
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
Hello
query='SELECT * FROM 02535_custom_key_mt' with custom_key='sipHash64(x)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
Hello
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
Hello
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
Hello
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
Hello
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
Hello
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
Hello
query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt) GROUP BY y ORDER BY y' with custom_key='y'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt) GROUP BY y ORDER BY y' with custom_key='cityHash64(y)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_mt) GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM 02535_custom_key_mt GROUP BY y ORDER BY y' with custom_key='y'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM 02535_custom_key_mt GROUP BY y ORDER BY y' with custom_key='cityHash64(y)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM 02535_custom_key_mt GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333

5
tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_mt.sh
View File

@ -7,12 +7,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
function run_with_custom_key {
echo "query='$1' with custom_key='$2'"
for filter_type in 'default' 'range'; do
for filter_type in 'custom_key_sampling' 'custom_key_range'; do
for max_replicas in {1..3}; do
echo "filter_type='$filter_type' max_replicas=$max_replicas"
query="$1 SETTINGS max_parallel_replicas=$max_replicas\
, enable_parallel_replicas='1' \
, parallel_replicas_mode ='$filter_type'\
, parallel_replicas_custom_key='$2'\
, parallel_replicas_custom_key_filter_type='$filter_type'\
, parallel_replicas_for_non_replicated_merge_tree=1 \
, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'"
$CLICKHOUSE_CLIENT --query="$query"

96
tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.reference
View File

@ -1,176 +1,176 @@
query='SELECT * FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt)' with custom_key='sipHash64(x)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
Hello
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
Hello
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
Hello
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
Hello
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
Hello
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
Hello
query='SELECT * FROM 02535_custom_key_rmt' with custom_key='sipHash64(x)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
Hello
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
Hello
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
Hello
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
Hello
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
Hello
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
Hello
query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt_hash) GROUP BY y ORDER BY y' with custom_key='y'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt_hash) GROUP BY y ORDER BY y' with custom_key='cityHash64(y)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02535_custom_key_rmt_hash) GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM 02535_custom_key_rmt_hash GROUP BY y ORDER BY y' with custom_key='y'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM 02535_custom_key_rmt_hash GROUP BY y ORDER BY y' with custom_key='cityHash64(y)'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333
query='SELECT y, count() FROM 02535_custom_key_rmt_hash GROUP BY y ORDER BY y' with custom_key='cityHash64(y) + 1'
filter_type='default' max_replicas=1
filter_type='custom_key_sampling' max_replicas=1
0 334
1 333
2 333
filter_type='default' max_replicas=2
filter_type='custom_key_sampling' max_replicas=2
0 334
1 333
2 333
filter_type='default' max_replicas=3
filter_type='custom_key_sampling' max_replicas=3
0 334
1 333
2 333
filter_type='range' max_replicas=1
filter_type='custom_key_range' max_replicas=1
0 334
1 333
2 333
filter_type='range' max_replicas=2
filter_type='custom_key_range' max_replicas=2
0 334
1 333
2 333
filter_type='range' max_replicas=3
filter_type='custom_key_range' max_replicas=3
0 334
1 333
2 333

5
tests/queries/0_stateless/02535_max_parallel_replicas_custom_key_rmt.sh
View File

@ -7,12 +7,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
function run_with_custom_key {
echo "query='$1' with custom_key='$2'"
for filter_type in 'default' 'range'; do
for filter_type in 'custom_key_sampling' 'custom_key_range'; do
for max_replicas in {1..3}; do
echo "filter_type='$filter_type' max_replicas=$max_replicas"
query="$1 SETTINGS max_parallel_replicas=$max_replicas\
, enable_parallel_replicas='1' \
, parallel_replicas_mode ='$filter_type'\
, parallel_replicas_custom_key='$2'\
, parallel_replicas_custom_key_filter_type='$filter_type'\
, parallel_replicas_for_non_replicated_merge_tree=1 \
, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost'"
$CLICKHOUSE_CLIENT --query="$query"

2
tests/queries/0_stateless/02708_parallel_replicas_not_found_column.sql
View File

@ -1,4 +1,4 @@
CREATE TABLE IF NOT EXISTS t_02708(x DateTime) ENGINE = MergeTree ORDER BY tuple();
SET send_logs_level='error';
SELECT count() FROM t_02708 SETTINGS use_parallel_replicas=1;
SELECT count() FROM t_02708 SETTINGS enable_parallel_replicas=1;
DROP TABLE t_02708;

8
tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference
View File

@ -33,8 +33,8 @@
=============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE ===============
0 2 SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\')
0 3 SELECT `key`, `value1`, `value2`, toUInt64(min(`time`)) AS `start_ts` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` ORDER BY `key` ASC, `value1` ASC, `value2` ASC LIMIT 10
1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS use_parallel_replicas = 1, enable_analyzer=0;
1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS use_parallel_replicas = 1, enable_analyzer=1;
1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=0;
1 1 -- Parallel inner query alone\nSELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=1;
=============== OUTER QUERY (NO PARALLEL) ===============
>T%O ,z< 10
NQTpY# W\\Xx4 10
@ -61,5 +61,5 @@ t<iT X48q:Z]t0 10
0 2 SELECT `__table2`.`value1` AS `value1`, `__table2`.`value2` AS `value2`, count() AS `count` FROM `default`.`join_outer_table` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`key` AS `key`, `__table3`.`value1` AS `value1`, `__table3`.`value2` AS `value2` FROM `default`.`join_inner_table` AS `__table3` PREWHERE (`__table3`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table3`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table3`.`key`, `__table3`.`value1`, `__table3`.`value2`) AS `__table2` USING (`key`) GROUP BY `__table1`.`key`, `__table2`.`value1`, `__table2`.`value2`
0 3 SELECT `key`, `value1`, `value2` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2`
0 3 SELECT `value1`, `value2`, count() AS `count` FROM `default`.`join_outer_table` ALL INNER JOIN `_data_` USING (`key`) GROUP BY `key`, `value1`, `value2`
1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS use_parallel_replicas = 1, enable_analyzer=0;
1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS use_parallel_replicas = 1, enable_analyzer=1;
1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=0;
1 1 -- Parallel full query\nSELECT\n value1,\n value2,\n avg(count) AS avg\nFROM\n (\n SELECT\n key,\n value1,\n value2,\n count() AS count\n FROM join_outer_table\n INNER JOIN\n (\n SELECT\n key,\n value1,\n value2,\n toUInt64(min(time)) AS start_ts\n FROM join_inner_table\n PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n GROUP BY key, value1, value2\n ) USING (key)\n GROUP BY key, value1, value2\n )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS enable_parallel_replicas = 1, enable_analyzer=1;

8
tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql
View File

@ -64,7 +64,7 @@ PREWHERE (id = '833c9e22-c245-4eb5-8745-117a9a1f26b1') AND (number > toUInt64('1
GROUP BY key, value1, value2
ORDER BY key, value1, value2
LIMIT 10
SETTINGS use_parallel_replicas = 1, enable_analyzer=0;
SETTINGS enable_parallel_replicas = 1, enable_analyzer=0;
-- Parallel inner query alone
SELECT
@ -77,7 +77,7 @@ PREWHERE (id = '833c9e22-c245-4eb5-8745-117a9a1f26b1') AND (number > toUInt64('1
GROUP BY key, value1, value2
ORDER BY key, value1, value2
LIMIT 10
SETTINGS use_parallel_replicas = 1, enable_analyzer=1;
SETTINGS enable_parallel_replicas = 1, enable_analyzer=1;
SELECT '=============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE ===============';
@ -184,7 +184,7 @@ FROM
)
GROUP BY value1, value2
ORDER BY value1, value2
SETTINGS use_parallel_replicas = 1, enable_analyzer=0;
SETTINGS enable_parallel_replicas = 1, enable_analyzer=0;
-- Parallel full query
SELECT
@ -214,7 +214,7 @@ FROM
)
GROUP BY value1, value2
ORDER BY value1, value2
SETTINGS use_parallel_replicas = 1, enable_analyzer=1;
SETTINGS enable_parallel_replicas = 1, enable_analyzer=1;
SYSTEM FLUSH LOGS;

2
tests/queries/0_stateless/02751_parallel_replicas_bug_chunkinfo_not_set.sql
View File

@ -18,7 +18,7 @@ INSERT INTO join_inner_table__fuzz_1 SELECT
FROM generateRandom('number Int64, value1 String, value2 String, time Int64', 1, 10, 2)
LIMIT 100;
SET max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree=1;
SET max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree=1;
-- SELECT query will write a Warning to the logs
SET send_logs_level='error';

2
tests/queries/0_stateless/02764_parallel_replicas_plain_merge_tree.sql
View File

@ -2,7 +2,7 @@ DROP TABLE IF EXISTS parallel_replicas_plain;
CREATE TABLE parallel_replicas_plain (x String) ENGINE=MergeTree() ORDER BY x;
INSERT INTO parallel_replicas_plain SELECT toString(number) FROM numbers(10);
SET max_parallel_replicas=3, use_parallel_replicas=1, cluster_for_parallel_replicas='parallel_replicas';
SET max_parallel_replicas=3, enable_parallel_replicas=1, cluster_for_parallel_replicas='parallel_replicas';
SET send_logs_level='error';
SET parallel_replicas_for_non_replicated_merge_tree = 0;

4
tests/queries/0_stateless/02765_parallel_replicas_final_modifier.sql
View File

@ -2,12 +2,12 @@ CREATE TABLE IF NOT EXISTS parallel_replicas_final (x String) ENGINE=ReplacingMe
INSERT INTO parallel_replicas_final SELECT toString(number) FROM numbers(10);
SET max_parallel_replicas=3, use_parallel_replicas=1, cluster_for_parallel_replicas='parallel_replicas';
SET max_parallel_replicas=3, enable_parallel_replicas=1, cluster_for_parallel_replicas='parallel_replicas';
SET parallel_replicas_for_non_replicated_merge_tree = 1;
SELECT * FROM parallel_replicas_final FINAL FORMAT Null;
SET use_parallel_replicas=2;
SET enable_parallel_replicas=2;
SELECT * FROM parallel_replicas_final FINAL FORMAT Null; -- { serverError SUPPORT_IS_DISABLED }

4
tests/queries/0_stateless/02769_parallel_replicas_unavailable_shards.sql
View File

@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test_parallel_replicas_unavailable_shards;
CREATE TABLE test_parallel_replicas_unavailable_shards (n UInt64) ENGINE=MergeTree() ORDER BY tuple();
INSERT INTO test_parallel_replicas_unavailable_shards SELECT * FROM numbers(10);
SET use_parallel_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1;
SET enable_parallel_replicas=2, max_parallel_replicas=11, cluster_for_parallel_replicas='parallel_replicas', parallel_replicas_for_non_replicated_merge_tree=1;
SET send_logs_level='error';
-- with local plan for initiator, the query can be executed fast on initator, we can simply not come to the point where unavailable replica can be detected
-- therefore disable local plan for now
@ -10,7 +10,7 @@ SELECT count() FROM test_parallel_replicas_unavailable_shards WHERE NOT ignore(*
SYSTEM FLUSH LOGS;
SET use_parallel_replicas=0;
SET enable_parallel_replicas=0;
SELECT ProfileEvents['ParallelReplicasUnavailableCount'] FROM system.query_log WHERE yesterday() <= event_date AND query_id in (select query_id from system.query_log where log_comment = '02769_7b513191-5082-4073-8568-53b86a49da79' and current_database = currentDatabase()) and type = 'QueryFinish' and query_id == initial_query_id;
DROP TABLE test_parallel_replicas_unavailable_shards;

7
tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference
View File

@ -8,10 +8,3 @@
5935810273536892891
7885388429666205427
8124171311239967992
<<<<<<< HEAD
1 1 -- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n SETTINGS\n enable_analyzer = 1,\n max_parallel_replicas = 2,\n cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n allow_experimental_parallel_reading_from_replicas = 1;
0 1 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1`
=======
1 1 -- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n SETTINGS\n enable_analyzer = 1,\n max_parallel_replicas = 2,\n cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n use_parallel_replicas = 1;
0 2 SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1`
>>>>>>> Parallel replicas feature is Beta

2
tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql
View File

@ -26,7 +26,7 @@ FROM join_inner_table__fuzz_146_replicated
enable_analyzer = 1,
max_parallel_replicas = 2,
cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost',
use_parallel_replicas = 1;
enable_parallel_replicas = 1;
SYSTEM FLUSH LOGS;
-- There should be 2 different queries

8
tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh
View File

@ -33,7 +33,7 @@ function run_query_with_pure_parallel_replicas () {
--max_parallel_replicas 3 \
--prefer_localhost_replica 1 \
--cluster_for_parallel_replicas 'test_cluster_one_shard_three_replicas_localhost' \
--use_parallel_replicas 1 \
--enable_parallel_replicas 1 \
--enable_analyzer 0
$CLICKHOUSE_CLIENT \
@ -42,7 +42,7 @@ function run_query_with_pure_parallel_replicas () {
--max_parallel_replicas 3 \
--prefer_localhost_replica 1 \
--cluster_for_parallel_replicas 'test_cluster_one_shard_three_replicas_localhost' \
--use_parallel_replicas 1 \
--enable_parallel_replicas 1 \
--enable_analyzer 1
}
@ -56,7 +56,7 @@ function run_query_with_custom_key_parallel_replicas () {
--query "$2" \
--query_id "${1}_custom_key" \
--max_parallel_replicas 3 \
--parallel_replicas_custom_key_filter_type 'default' \
--parallel_replicas_mode 'custom_key_sampling' \
--parallel_replicas_custom_key "$2" \
--enable_analyzer 0
@ -64,7 +64,7 @@ function run_query_with_custom_key_parallel_replicas () {
--query "$2" \
--query_id "${1}_custom_key_analyzer" \
--max_parallel_replicas 3 \
--parallel_replicas_custom_key_filter_type 'default' \
--parallel_replicas_mode 'custom_key_sampling' \
--parallel_replicas_custom_key "$2" \
--enable_analyzer 1
}

2
tests/queries/0_stateless/02784_parallel_replicas_automatic_decision.sh
View File

@ -50,7 +50,7 @@ function run_query_with_pure_parallel_replicas () {
--query_id "${1}_pure" \
--max_parallel_replicas 3 \
--cluster_for_parallel_replicas "parallel_replicas" \
--use_parallel_replicas 1 \
--enable_parallel_replicas 1 \
--parallel_replicas_for_non_replicated_merge_tree 1 \
--parallel_replicas_min_number_of_rows_per_replica "$2" \
--max_threads 5 \

2
tests/queries/0_stateless/02784_parallel_replicas_automatic_decision_join.sh
View File

@ -67,7 +67,7 @@ function run_query_with_pure_parallel_replicas () {
--prefer_localhost_replica 1 \
--parallel_replicas_prefer_local_join 0 \
--cluster_for_parallel_replicas "parallel_replicas" \
--use_parallel_replicas 1 \
--enable_parallel_replicas 1 \
--parallel_replicas_for_non_replicated_merge_tree 1 \
--parallel_replicas_min_number_of_rows_per_replica "$2" \
|& grep "It is enough work for" | awk '{ print substr($7, 2, length($7) - 2) "\t" $20 " estimated parallel replicas" }' | sort -n -k2 -b | grep -Pv "\t0 estimated parallel replicas"

2
tests/queries/0_stateless/02811_parallel_replicas_prewhere_count.sql
View File

@ -11,7 +11,7 @@ SELECT count() FROM users PREWHERE uid > 2000;
-- enable parallel replicas but with high rows threshold
SET
skip_unavailable_shards=1,
use_parallel_replicas=1,
enable_parallel_replicas=1,
max_parallel_replicas=3,
cluster_for_parallel_replicas='parallel_replicas',
parallel_replicas_for_non_replicated_merge_tree=1,

8
tests/queries/0_stateless/02835_parallel_replicas_over_distributed.sql
View File

@ -14,13 +14,13 @@ insert into test select *, today() from numbers(100);
SELECT count(), min(id), max(id), avg(id)
FROM test_d
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
insert into test select *, today() from numbers(100);
SELECT count(), min(id), max(id), avg(id)
FROM test_d
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
-- 2 shards
@ -38,10 +38,10 @@ insert into test2 select *, today() from numbers(100);
SELECT count(), min(id), max(id), avg(id)
FROM test2_d
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
insert into test2 select *, today() from numbers(100);
SELECT count(), min(id), max(id), avg(id)
FROM test2_d
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;

4
tests/queries/0_stateless/02841_parallel_replicas_summary.sh
View File

@ -35,7 +35,7 @@ echo "
SETTINGS
max_parallel_replicas = 2,
cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost',
use_parallel_replicas = 2,
enable_parallel_replicas = 2,
parallel_replicas_for_non_replicated_merge_tree = 1,
interactive_delay=0,
parallel_replicas_local_plan=0
@ -50,7 +50,7 @@ echo "
SETTINGS
max_parallel_replicas = 2,
cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost',
use_parallel_replicas = 2,
enable_parallel_replicas = 2,
parallel_replicas_for_non_replicated_merge_tree = 1,
interactive_delay=99999999999,
parallel_replicas_local_plan=0

2
tests/queries/0_stateless/02861_index_set_incorrect_args.sql
View File

@ -2,5 +2,5 @@
DROP TABLE IF EXISTS set_index__fuzz_41;
CREATE TABLE set_index__fuzz_41 (`a` Date, `b` Nullable(DateTime64(3)), INDEX b_set b TYPE set(0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple();
INSERT INTO set_index__fuzz_41 (a) VALUES (today());
SELECT b FROM set_index__fuzz_41 WHERE and(b = 256) SETTINGS force_data_skipping_indices = 'b_set', optimize_move_to_prewhere = 0, max_parallel_replicas=2, parallel_replicas_for_non_replicated_merge_tree=1, use_parallel_replicas=2; -- { serverError TOO_FEW_ARGUMENTS_FOR_FUNCTION }
SELECT b FROM set_index__fuzz_41 WHERE and(b = 256) SETTINGS force_data_skipping_indices = 'b_set', optimize_move_to_prewhere = 0, max_parallel_replicas=2, parallel_replicas_for_non_replicated_merge_tree=1, enable_parallel_replicas=2; -- { serverError TOO_FEW_ARGUMENTS_FOR_FUNCTION }
DROP TABLE set_index__fuzz_41;

2
tests/queries/0_stateless/02869_parallel_replicas_read_from_several.sql
View File

@ -24,5 +24,5 @@ system sync replica t3;
SELECT count(), min(k), max(k), avg(k)
FROM t1
SETTINGS use_parallel_replicas = 1, max_parallel_replicas = 3, prefer_localhost_replica = 0,
SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 3, prefer_localhost_replica = 0,
cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = 0.001;

4
tests/queries/0_stateless/02875_parallel_replicas_cluster_all_replicas.sql
View File

@ -2,12 +2,12 @@ DROP TABLE IF EXISTS tt;
CREATE TABLE tt (n UInt64) ENGINE=MergeTree() ORDER BY tuple();
INSERT INTO tt SELECT * FROM numbers(10);
SET use_parallel_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1;
SET enable_parallel_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1;
SELECT count() FROM clusterAllReplicas('test_cluster_two_shard_three_replicas_localhost', currentDatabase(), tt) settings log_comment='02875_190aed82-2423-413b-ad4c-24dcca50f65b';
SYSTEM FLUSH LOGS;
SET use_parallel_replicas=0;
SET enable_parallel_replicas=0;
SELECT count() > 0 FROM system.text_log
WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment = '02875_190aed82-2423-413b-ad4c-24dcca50f65b')
AND message LIKE '%Parallel reading from replicas is disabled for cluster%';

4
tests/queries/0_stateless/02875_parallel_replicas_remote.sql
View File

@ -2,12 +2,12 @@ DROP TABLE IF EXISTS tt;
CREATE TABLE tt (n UInt64) ENGINE=MergeTree() ORDER BY tuple();
INSERT INTO tt SELECT * FROM numbers(10);
SET use_parallel_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1;
SET enable_parallel_replicas=1, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1;
SELECT count() FROM remote('127.0.0.{1..6}', currentDatabase(), tt) settings log_comment='02875_89f3c39b-1919-48cb-b66e-ef9904e73146';
SYSTEM FLUSH LOGS;
SET use_parallel_replicas=0;
SET enable_parallel_replicas=0;
SELECT count() > 0 FROM system.text_log
WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment = '02875_89f3c39b-1919-48cb-b66e-ef9904e73146')
AND message LIKE '%Parallel reading from replicas is disabled for cluster%';

6
tests/queries/0_stateless/02898_parallel_replicas_custom_key_final.sql
View File

@ -8,12 +8,12 @@ SELECT y, count()
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02898_parallel_replicas_final) FINAL
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling';
SELECT y, count()
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), 02898_parallel_replicas_final) FINAL
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='range';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_range';
DROP TABLE 02898_parallel_replicas_final;
DROP TABLE 02898_parallel_replicas_final;

6
tests/queries/0_stateless/02898_parallel_replicas_progress_bar.sql
View File

@ -14,7 +14,7 @@ system sync replica t1;
system sync replica t2;
system sync replica t3;
SET use_parallel_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost';
SET enable_parallel_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost';
-- default coordinator
SELECT count(), min(k), max(k), avg(k) FROM t1 SETTINGS log_comment='02898_default_190aed82-2423-413b-ad4c-24dcca50f65b';
@ -23,7 +23,7 @@ SELECT count(), min(k), max(k), avg(k) FROM t1 SETTINGS log_comment='02898_defau
SYSTEM FLUSH LOGS;
SELECT count() > 0 FROM system.text_log
WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_default_190aed82-2423-413b-ad4c-24dcca50f65b')
AND message LIKE '%Total rows to read: 3000%' SETTINGS use_parallel_replicas=0;
AND message LIKE '%Total rows to read: 3000%' SETTINGS enable_parallel_replicas=0;
-- reading in order coordinator
-- disable parallel_replicas_local_plan since the test relay on traces which only present in case of no local plan
@ -32,7 +32,7 @@ SELECT k, sipHash64(v) FROM t1 order by k limit 5 offset 998 SETTINGS optimize_r
SYSTEM FLUSH LOGS;
SELECT count() > 0 FROM system.text_log
WHERE query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment='02898_inorder_190aed82-2423-413b-ad4c-24dcca50f65b')
AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS use_parallel_replicas=0;
AND message LIKE '%Updated total rows to read: added % rows, total 3000 rows%' SETTINGS enable_parallel_replicas=0;
DROP TABLE t1 SYNC;
DROP TABLE t2 SYNC;

4
tests/queries/0_stateless/02901_parallel_replicas_rollup.sh
View File

@ -30,7 +30,7 @@ $CLICKHOUSE_CLIENT \
--max_parallel_replicas 3 \
--prefer_localhost_replica 1 \
--cluster_for_parallel_replicas "test_cluster_one_shard_three_replicas_localhost" \
--use_parallel_replicas 1 \
--enable_parallel_replicas 1 \
--parallel_replicas_for_non_replicated_merge_tree 1 \
--parallel_replicas_min_number_of_rows_per_replica 0 \
--query "
@ -68,7 +68,7 @@ $CLICKHOUSE_CLIENT \
--max_parallel_replicas 3 \
--prefer_localhost_replica 1 \
--cluster_for_parallel_replicas "test_cluster_one_shard_three_replicas_localhost" \
--use_parallel_replicas 1 \
--enable_parallel_replicas 1 \
--parallel_replicas_for_non_replicated_merge_tree 1 \
--parallel_replicas_min_number_of_rows_per_replica 0 \
--query "SELECT * FROM (SELECT year, month, day, count(*) FROM days GROUP BY year, month, day WITH ROLLUP) ORDER BY 1, 2, 3";

6
tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.reference
View File

@ -3,7 +3,7 @@ SELECT y, count()
FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas)
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling';
0 250
1 250
2 250
@ -12,7 +12,7 @@ SELECT y, count()
FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas)
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='range';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_range';
0 250
1 250
2 250
@ -22,7 +22,7 @@ SELECT y, count()
FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas)
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling';
0 250
1 250
2 250

6
tests/queries/0_stateless/02918_parallel_replicas_custom_key_unavailable_replica.sql
View File

@ -14,20 +14,20 @@ SELECT y, count()
FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas)
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling';
SELECT y, count()
FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas)
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='range';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_range';
SET use_hedged_requests=0;
SELECT y, count()
FROM cluster(test_cluster_1_shard_3_replicas_1_unavailable, currentDatabase(), 02918_parallel_replicas)
GROUP BY y
ORDER BY y
SETTINGS max_parallel_replicas=3, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_custom_key_filter_type='default';
SETTINGS max_parallel_replicas=3, enable_parallel_replicas=1, parallel_replicas_custom_key='cityHash64(y)', parallel_replicas_mode='custom_key_sampling';
-- { echoOff }
SET send_logs_level='warning';

2
tests/queries/0_stateless/02922_analyzer_aggregate_nothing_type.sql
View File

@ -88,7 +88,7 @@ CREATE TABLE t1 (`n` UInt64) ENGINE = MergeTree ORDER BY tuple();
INSERT INTO t1 SELECT * FROM numbers(10);
SET
use_parallel_replicas=1,
enable_parallel_replicas=1,
max_parallel_replicas=2,
use_hedged_requests=0,
cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost',

6
tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql
View File

@ -9,7 +9,7 @@ SELECT key, value1, value2, toUInt64(min(time)) AS start_ts FROM join_inner_tabl
max_parallel_replicas = 3,
prefer_localhost_replica = 1,
cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost',
use_parallel_replicas = 1,
enable_parallel_replicas = 1,
use_hedged_requests = 0;
@ -22,7 +22,7 @@ ORDER BY nan DESC, [0, NULL, NULL, NULL, NULL] DESC
FORMAT Null
SETTINGS
max_parallel_replicas = 3,
use_parallel_replicas = 1,
enable_parallel_replicas = 1,
use_hedged_requests = 0,
cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost';
@ -35,4 +35,4 @@ ORDER BY
nan DESC,
_CAST([0, NULL, NULL, NULL, NULL], 'Array(Nullable(UInt8))') DESC
FORMAT Null
SETTINGS receive_timeout = 10., receive_data_timeout_ms = 10000, use_hedged_requests = 0, allow_suspicious_low_cardinality_types = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, log_queries = 1, table_function_remote_max_addresses = 200, enable_analyzer = 1;
SETTINGS receive_timeout = 10., receive_data_timeout_ms = 10000, use_hedged_requests = 0, allow_suspicious_low_cardinality_types = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, log_queries = 1, table_function_remote_max_addresses = 200, enable_analyzer = 1;

12
tests/queries/0_stateless/02935_parallel_replicas_settings.sql
View File

@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test_parallel_replicas_settings;
CREATE TABLE test_parallel_replicas_settings (n UInt64) ENGINE=MergeTree() ORDER BY tuple();
INSERT INTO test_parallel_replicas_settings SELECT * FROM numbers(10);
SET use_parallel_replicas=2, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1;
SET enable_parallel_replicas=2, max_parallel_replicas=3, parallel_replicas_for_non_replicated_merge_tree=1;
SET cluster_for_parallel_replicas='';
SELECT count() FROM test_parallel_replicas_settings WHERE NOT ignore(*); -- { serverError CLUSTER_DOESNT_EXIST }
@ -16,20 +16,20 @@ SELECT count() > 0 FROM system.text_log
WHERE yesterday() <= event_date
AND query_id in (select query_id from system.query_log where current_database=currentDatabase() AND log_comment='0_f621c4f2-4da7-4a7c-bb6d-052c442d0f7f')
AND level = 'Information'
AND message ILIKE '%Disabling ''use_hedged_requests'' in favor of ''use_parallel_replicas''%'
SETTINGS use_parallel_replicas=0;
AND message ILIKE '%Disabling ''use_hedged_requests'' in favor of ''enable_parallel_replicas''%'
SETTINGS enable_parallel_replicas=0;
SET use_hedged_requests=1;
SELECT count() FROM test_parallel_replicas_settings WHERE NOT ignore(*) settings log_comment='1_f621c4f2-4da7-4a7c-bb6d-052c442d0f7f';
SYSTEM FLUSH LOGS;
SET use_parallel_replicas=0;
SET enable_parallel_replicas=0;
SELECT count() > 0 FROM system.text_log
WHERE yesterday() <= event_date
AND query_id in (select query_id from system.query_log where current_database = currentDatabase() AND log_comment = '1_f621c4f2-4da7-4a7c-bb6d-052c442d0f7f')
AND level = 'Warning'
AND message ILIKE '%Setting ''use_hedged_requests'' explicitly with enabled ''use_parallel_replicas'' has no effect%'
SETTINGS use_parallel_replicas=0;
AND message ILIKE '%Setting ''use_hedged_requests'' explicitly with enabled ''enable_parallel_replicas'' has no effect%'
SETTINGS enable_parallel_replicas=0;
DROP TABLE test_parallel_replicas_settings;

2
tests/queries/0_stateless/02946_parallel_replicas_distributed.sql
View File

@ -11,7 +11,7 @@ ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDat
SELECT count(), sum(id)
FROM test_d
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 3, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree=1;
DROP TABLE test_d;
DROP TABLE test;

4
tests/queries/0_stateless/02946_parallel_replicas_force_primary_key.sql
View File

@ -22,10 +22,10 @@ FROM t1
WHERE k > 0
GROUP BY k
ORDER BY k
SETTINGS force_primary_key = 1, use_parallel_replicas = 0;
SETTINGS force_primary_key = 1, enable_parallel_replicas = 0;
-- parallel replicas, primary key is used
SET use_parallel_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost';
SET enable_parallel_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost';
SELECT
k,
count()

2
tests/queries/0_stateless/02947_parallel_replicas_remote.sql
View File

@ -7,6 +7,6 @@ AS select *, '2023-12-25' from numbers(100);
SELECT count(), sum(id)
FROM remote('127.0.0.1|127.0.0.2|127.0.0.3|127.0.0.4', currentDatabase(), test)
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 4, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1; -- { serverError CLUSTER_DOESNT_EXIST }
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 4, prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1; -- { serverError CLUSTER_DOESNT_EXIST }
DROP TABLE test;

22
tests/queries/0_stateless/02949_parallel_replicas_in_subquery.sql
View File

@ -8,23 +8,23 @@ INSERT INTO merge_tree_in_subqueries VALUES(5, 'test5', 0);
SET max_parallel_replicas=3, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED }
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS use_parallel_replicas=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED }
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 0) SETTINGS enable_parallel_replicas=1;
SELECT '---';
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED };
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS use_parallel_replicas=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED };
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT * FROM system.numbers LIMIT 2, 3) ORDER BY id SETTINGS enable_parallel_replicas=1;
SELECT '---';
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED };
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS use_parallel_replicas=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0; -- { serverError SUPPORT_IS_DISABLED };
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
SELECT * FROM merge_tree_in_subqueries WHERE id IN (SELECT 1) ORDER BY id SETTINGS enable_parallel_replicas=1;
-- IN with tuples is allowed
SELECT '---';
SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0;
SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS use_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=0;
SELECT id, name FROM merge_tree_in_subqueries WHERE (id, name) IN (3, 'test3') SETTINGS enable_parallel_replicas=2, parallel_replicas_allow_in_with_subquery=1;
DROP TABLE IF EXISTS merge_tree_in_subqueries;

2
tests/queries/0_stateless/02949_parallel_replicas_scalar_subquery_big_integer.sql
View File

@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test;
CREATE TABLE test (x UInt8) ENGINE = MergeTree ORDER BY x;
INSERT INTO test VALUES (1), (2), (3);
SET use_parallel_replicas = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1;
SET enable_parallel_replicas = 1, max_parallel_replicas = 2, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', prefer_localhost_replica = 0, parallel_replicas_for_non_replicated_merge_tree = 1;
WITH (SELECT '111111111111111111111111111111111111111'::UInt128) AS v SELECT sum(x), max(v) FROM test;

7
tests/queries/0_stateless/02950_parallel_replicas_used_count.sql
View File

@ -7,21 +7,20 @@ SETTINGS index_granularity=1;
INSERT INTO test SELECT number, toString(number) FROM numbers(10_000);
SET use_parallel_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost';
SET enable_parallel_replicas = 2, max_parallel_replicas = 3, parallel_replicas_for_non_replicated_merge_tree=1, cluster_for_parallel_replicas='test_cluster_one_shard_three_replicas_localhost';
-- default coordinator
SELECT count(), sum(k)
FROM test
SETTINGS log_comment = '02950_parallel_replicas_used_replicas_count';
SYSTEM FLUSH LOGS;
SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS use_parallel_replicas=0;
SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS enable_parallel_replicas=0;
-- In order coordinator
SELECT k FROM test order by k limit 5 offset 89 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_2', merge_tree_min_rows_for_concurrent_read=1, max_threads=1;
SYSTEM FLUSH LOGS;
SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_2' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS use_parallel_replicas=0;
SELECT ProfileEvents['ParallelReplicasUsedCount'] > 0 FROM system.query_log WHERE type = 'QueryFinish' AND query_id IN (SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment = '02950_parallel_replicas_used_replicas_count_2' AND type = 'QueryFinish' AND initial_query_id = query_id) SETTINGS enable_parallel_replicas=0;
-- In reverse order coordinator
SELECT k FROM test order by k desc limit 5 offset 9906 SETTINGS optimize_read_in_order=1, log_comment='02950_parallel_replicas_used_replicas_count_3', merge_tree_min_rows_for_concurrent_read=1, max_threads=1;

2
tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh
View File

@ -17,7 +17,7 @@ insert into num_1 select number * 2, toString(number * 2) from numbers(1e7);
insert into num_2 select number * 3, -number from numbers(1.5e6);
"
PARALLEL_REPLICAS_SETTINGS="use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_local_plan=1"
PARALLEL_REPLICAS_SETTINGS="enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_local_plan=1"
##############
echo

74
tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference
View File

@ -2,7 +2,7 @@
set parallel_replicas_prefer_local_join = 0;
-- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode.
select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -18,7 +18,7 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x
13 13 0 0 0 0
14 14 14 14 0 0
15 15 0 0 0 0
explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Union
Expression
Join
@ -42,7 +42,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -65,7 +65,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Union
@ -92,7 +92,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
select * from sub5 order by key
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
54 54 50 50 12 12 0
64 64 0 0 0 0 1
explain description=0
@ -102,7 +102,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
select * from sub5 order by key
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -131,7 +131,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -154,7 +154,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -183,7 +183,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -206,7 +206,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -239,7 +239,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
6 6 6 6 0 0
8 8 8 8 0 0
@ -262,7 +262,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Join
Expression
@ -292,7 +292,7 @@ sub2 as (select y, z from tab2 where y != 4),
sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
select * from sub5 order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
0 0 1 1 0 0
0 0 3 3 0 0
@ -314,7 +314,7 @@ sub2 as (select y, z from tab2 where y != 4),
sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
select * from sub5 order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -347,7 +347,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -370,7 +370,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Union
@ -403,7 +403,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -426,7 +426,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;-- { echoOn }
SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;-- { echoOn }
Expression
Sorting
Expression
@ -456,7 +456,7 @@ Expression
ReadFromRemoteParallelReplicas
set parallel_replicas_prefer_local_join = 1;
-- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode.
select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -473,6 +473,7 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x
14 14 14 14 0 0
15 15 0 0 0 0
<<<<<<< HEAD
<<<<<<< HEAD
explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Union
Expression
@ -492,6 +493,9 @@ Union
ReadFromRemoteParallelReplicas
=======
explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
=======
explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
>>>>>>> Parallel replicas is Beta
Expression
ReadFromRemoteParallelReplicas
>>>>>>> Parallel replicas feature is Beta
@ -503,7 +507,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -526,7 +530,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Union
@ -555,7 +559,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
select * from sub5 order by key
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
54 54 50 50 12 12 0
64 64 0 0 0 0 1
explain description=0
@ -565,7 +569,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
select * from sub5 order by key
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -596,7 +600,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -619,7 +623,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -649,7 +653,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -672,7 +676,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -705,7 +709,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
6 6 6 6 0 0
8 8 8 8 0 0
@ -728,7 +732,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Join
Expression
@ -758,7 +762,7 @@ sub2 as (select y, z from tab2 where y != 4),
sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
select * from sub5 order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
0 0 1 1 0 0
0 0 3 3 0 0
@ -780,7 +784,7 @@ sub2 as (select y, z from tab2 where y != 4),
sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
select * from sub5 order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Expression
@ -813,7 +817,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -836,7 +840,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
Expression
Sorting
Union
@ -871,7 +875,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
0 0 0 0 0 0
1 1 0 0 0 0
3 3 0 0 0 0
@ -894,7 +898,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
Expression
Sorting
Expression

36
tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2
View File

@ -19,8 +19,8 @@ set parallel_replicas_local_plan=1;
set parallel_replicas_prefer_local_join = {{use_global_in}};
-- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode.
select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- The same query with cte;
with sub1 as (select x, y from tab1 where x != 2),
@ -29,7 +29,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0
with sub1 as (select x, y from tab1 where x != 2),
@ -38,7 +38,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- GROUP BY should work up to WithMergableStage
with sub1 as (select x, y from tab1 where x != 2),
@ -47,7 +47,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
select * from sub5 order by key
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0
with sub1 as (select x, y from tab1 where x != 2),
@ -56,7 +56,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
select * from sub5 order by key
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- ORDER BY in sub3 : sub1 is fully pushed, sub3 -> WithMergableStage
with sub1 as (select x, y from tab1 where x != 2),
@ -65,7 +65,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0
with sub1 as (select x, y from tab1 where x != 2),
@ -74,7 +74,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- ORDER BY in sub1 : sub1 -> WithMergableStage
with sub1 as (select x, y from tab1 where x != 2 order by y),
@ -83,7 +83,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0
with sub1 as (select x, y from tab1 where x != 2 order by y),
@ -92,7 +92,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- RIGHT JOIN in sub3: sub3 -> WithMergableStage
with sub1 as (select x, y from tab1 where x != 2),
@ -101,7 +101,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0
with sub1 as (select x, y from tab1 where x != 2),
@ -110,7 +110,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- RIGHT JOIN in sub5: sub5 -> WithMergableStage
with sub1 as (select x, y from tab1 where x != 2),
@ -118,7 +118,7 @@ sub2 as (select y, z from tab2 where y != 4),
sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
select * from sub5 order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0
with sub1 as (select x, y from tab1 where x != 2),
@ -126,7 +126,7 @@ sub2 as (select y, z from tab2 where y != 4),
sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
select * from sub5 order by x SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
select * from sub5 order by x SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- Subqueries for IN allowed
@ -136,7 +136,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
explain description=0
with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)),
@ -145,7 +145,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
SETTINGS enable_parallel_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
--
-- Subqueries for IN are not allowed
@ -155,7 +155,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
explain description=0
with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)),
@ -164,6 +164,6 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
sub4 as (select z, a from tab3 where z != 8),
sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
select * from sub5 order by x
SETTINGS use_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
{%- endfor %}

16
tests/queries/0_stateless/02972_parallel_replicas_cte.sql
View File

@ -13,21 +13,21 @@ SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.
WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100)
SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
SETTINGS use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
-- Testing that it is disabled for enable_analyzer=0. With analyzer it will be supported (with correct result)
WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100)
SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
SETTINGS enable_analyzer = 0, use_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
SETTINGS enable_analyzer = 0, enable_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
-- Disabled for any value of use_parallel_replicas != 1, not just 2
-- Disabled for any value of enable_parallel_replicas != 1, not just 2
WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100)
SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
SETTINGS enable_analyzer = 0, use_parallel_replicas = 512, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
SETTINGS enable_analyzer = 0, enable_parallel_replicas = 512, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
-- Sanitizer
SELECT count() FROM pr_2 JOIN numbers(10) as pr_1 ON pr_2.a = pr_1.number
SETTINGS use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
-- Parallel replicas detection should work inside subqueries
SELECT *
@ -36,7 +36,7 @@ FROM
WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100)
SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
)
SETTINGS use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
-- Subquery + subquery
SELECT count()
@ -49,7 +49,7 @@ FROM
SELECT count() as c FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
)
)
SETTINGS use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
CREATE TABLE numbers_1e3
(
@ -74,7 +74,7 @@ WITH
)
SELECT count()
FROM cte2
SETTINGS use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
SETTINGS enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
DROP TABLE IF EXISTS numbers_1e6;
DROP TABLE IF EXISTS pr_1;

2
tests/queries/0_stateless/02981_insert_select_resize_to_max_insert_threads.sh
View File

@ -49,7 +49,7 @@ ${CLICKHOUSE_CLIENT} --max_insert_threads "$max_insert_threads" --parallel_distr
""" | grep -v EmptySink | grep -c Sink
echo "inserting into a remote table from remote (reading with parallel replicas) with concurrency max_insert_threads"
${CLICKHOUSE_CLIENT} --max_insert_threads "$max_insert_threads" --use_parallel_replicas 2 --cluster_for_parallel_replicas 'parallel_replicas' --max_parallel_replicas 3 -q """
${CLICKHOUSE_CLIENT} --max_insert_threads "$max_insert_threads" --enable_parallel_replicas 2 --cluster_for_parallel_replicas 'parallel_replicas' --max_parallel_replicas 3 -q """
EXPLAIN PIPELINE
INSERT INTO t3_dist
SELECT * FROM t4_pr;

2
tests/queries/0_stateless/02982_parallel_replicas_unexpected_cluster.sql
View File

@ -2,7 +2,7 @@ DROP TABLE IF EXISTS test_unexpected_cluster;
CREATE TABLE test_unexpected_cluster (n UInt64) ENGINE=MergeTree() ORDER BY tuple();
INSERT INTO test_unexpected_cluster SELECT * FROM numbers(10);
SET use_parallel_replicas=2, max_parallel_replicas=2, cluster_for_parallel_replicas='test_cluster_two_shards', parallel_replicas_for_non_replicated_merge_tree=1;
SET enable_parallel_replicas=2, max_parallel_replicas=2, cluster_for_parallel_replicas='test_cluster_two_shards', parallel_replicas_for_non_replicated_merge_tree=1;
SELECT count() FROM test_unexpected_cluster WHERE NOT ignore(*); -- { serverError UNEXPECTED_CLUSTER }
DROP TABLE test_unexpected_cluster;

2
tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql
View File

@ -21,7 +21,7 @@ AS SELECT *
FROM numbers(10);
SET enable_analyzer = 1;
SET use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=0;
SET enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=0;
EXPLAIN SYNTAX
WITH

2
tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql
View File

@ -21,7 +21,7 @@ SELECT
FROM numbers(130000)
SETTINGS max_insert_block_size = 200000;
SET max_block_size = 1048576, max_threads = 1, use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=10000;
SET max_block_size = 1048576, max_threads = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=10000;
EXPLAIN ESTIMATE
SELECT count()

2
tests/queries/0_stateless/03093_reading_bug_with_parallel_replicas.sql
View File

@ -3,7 +3,7 @@ set max_threads = 16;
set use_hedged_requests = 0;
set max_parallel_replicas = 3;
set cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost';
set use_parallel_replicas = 1;
set enable_parallel_replicas = 1;
set parallel_replicas_for_non_replicated_merge_tree = 1;
set allow_aggregate_partitions_independently = 1;

2
tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql
View File

@ -1,7 +1,7 @@
DROP TABLE IF EXISTS mv_table;
DROP TABLE IF EXISTS null_table;
SET cluster_for_parallel_replicas='parallel_replicas', max_parallel_replicas=4, allow_experimental_parallel_reading_from_replicas=1;
SET cluster_for_parallel_replicas='parallel_replicas', max_parallel_replicas=4, enable_parallel_replicas=1;
SET enable_analyzer=1;
CREATE TABLE null_table (str String) ENGINE = Null;

3
tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql
View File

@ -26,5 +26,4 @@ ENGINE = MergeTree ORDER BY (date, pull_request_number, commit_sha, check_name,
insert into checks select * from generateRandom() limit 1;
select trimLeft(explain) from (explain SELECT count(1) FROM checks WHERE test_name IS NOT NULL) where explain like '%ReadFromPreparedSource%' SETTINGS enable_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 0;
select trimLeft(explain) from (explain SELECT count(1) FROM checks WHERE test_name IS NOT NULL) where explain like '%ReadFromPreparedSource%' SETTINGS enable_analyzer = 1, enable_parallel_replicas = 0;

152
tests/queries/0_stateless/03164_parallel_replicas_range_filter_min_max.sql
View File

@ -4,86 +4,126 @@ CREATE TABLE range_filter_custom_range_test (k UInt64) ENGINE=MergeTree ORDER BY
INSERT INTO range_filter_custom_range_test SELECT number + 5 from numbers(10);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 5, parallel_replicas_custom_key_range_upper = 15
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 5,
parallel_replicas_custom_key_range_upper = 15
);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 4, parallel_replicas_custom_key_range_upper = 14
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 4,
parallel_replicas_custom_key_range_upper = 14
);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 6, parallel_replicas_custom_key_range_upper = 17
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 6,
parallel_replicas_custom_key_range_upper = 17
);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 15
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 0,
parallel_replicas_custom_key_range_upper = 15
);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 15, parallel_replicas_custom_key_range_upper = 25
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 15,
parallel_replicas_custom_key_range_upper = 25
);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 5
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 0,
parallel_replicas_custom_key_range_upper = 5
);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 500, parallel_replicas_custom_key_range_upper = 10000
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 500,
parallel_replicas_custom_key_range_upper = 10000
);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 3, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 10, parallel_replicas_custom_key_range_upper = 13
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 3,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 10,
parallel_replicas_custom_key_range_upper = 13
);
DROP TABLE range_filter_custom_range_test;
@ -94,14 +134,19 @@ CREATE TABLE range_filter_custom_range_test_2 (k UInt64) ENGINE=MergeTree ORDER
INSERT INTO range_filter_custom_range_test_2 SELECT number from numbers(13);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(parallel_replicas, currentDatabase(), range_filter_custom_range_test_2)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 12, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 13
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 12,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 0,
parallel_replicas_custom_key_range_upper = 13
);
DROP TABLE range_filter_custom_range_test_2;
@ -112,14 +157,19 @@ CREATE TABLE range_filter_custom_range_test_3 (k UInt64) ENGINE=MergeTree ORDER
INSERT INTO range_filter_custom_range_test_3 SELECT number from numbers(4);
SELECT count()
FROM
SELECT count()
FROM
(
SELECT *
FROM cluster(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), range_filter_custom_range_test_3)
SETTINGS prefer_localhost_replica = 0, max_parallel_replicas = 12, distributed_group_by_no_merge = 0,
parallel_replicas_custom_key = 'k', parallel_replicas_custom_key_filter_type = 'range',
parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 4
SETTINGS prefer_localhost_replica = 0,
max_parallel_replicas = 12,
distributed_group_by_no_merge = 0,
enable_parallel_replicas = 1,
parallel_replicas_custom_key = 'k',
parallel_replicas_mode = 'custom_key_range',
parallel_replicas_custom_key_range_lower = 0,
parallel_replicas_custom_key_range_upper = 4
);
DROP TABLE range_filter_custom_range_test_3;
DROP TABLE range_filter_custom_range_test_3;

1
tests/queries/0_stateless/03168_loop_engine_with_parallel_replicas.sql
View File

@ -6,6 +6,7 @@ CREATE TABLE 03147_db.t (n Int8) ENGINE=MergeTree ORDER BY n;
INSERT INTO 03147_db.t SELECT * FROM numbers(10);
USE 03147_db;
-- We use the old setting here just to make sure we preserve it as an alias.
SET allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 100;
SELECT * FROM loop(03147_db.t) LIMIT 15 FORMAT Null;

2
tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh
View File

@ -17,7 +17,7 @@ $CLICKHOUSE_CLIENT -q "
"
$CLICKHOUSE_CLIENT -q "
SET enable_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 10, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
SET enable_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 10, enable_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
SELECT
id,

0
tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.reference Normal file
View File

34
tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql Normal file
View File

@ -0,0 +1,34 @@
-- Tags: no-s3-storage
-- no-s3-storage: the test checks that there is no crash when we set parallel_replicas_single_task_marks_count_multiplier to 0 and we have some custom heuristic for
-- finding its optimal value in case if the table is stored on remote disk.
DROP TABLE IF EXISTS 03215_parallel_replicas;
CREATE TABLE 03215_parallel_replicas
(
`k` Int16,
`v` Nullable(UInt8)
)
ENGINE = MergeTree
ORDER BY k
SETTINGS index_granularity = 10;
INSERT INTO 03215_parallel_replicas SELECT
number,
number
FROM numbers(1000);
INSERT INTO 03215_parallel_replicas SELECT
number,
number
FROM numbers(1000, 1000);
INSERT INTO 03215_parallel_replicas SELECT
number,
number
FROM numbers(2000, 1000);
SET parallel_distributed_insert_select = 2, prefer_localhost_replica = false, enable_parallel_replicas = 1, max_parallel_replicas = 65535, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0., parallel_replicas_for_non_replicated_merge_tree = true;
SELECT max(k) IGNORE NULLS FROM 03215_parallel_replicas WITH TOTALS SETTINGS enable_parallel_replicas = 1, max_parallel_replicas = 65535, prefer_localhost_replica = 0, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0; -- { serverError 36 }
DROP TABLE IF EXISTS 03215_parallel_replicas;

6
tests/queries/1_stateful/00177_memory_bound_merging.sh
View File

@ -31,7 +31,7 @@ test1() {
GROUP BY CounterID, URL, EventDate
ORDER BY URL, EventDate
LIMIT 5 OFFSET 10
SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3"
SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3"
check_replicas_read_in_order $query_id
}
@ -48,7 +48,7 @@ test2() {
GROUP BY URL, EventDate
ORDER BY URL, EventDate
LIMIT 5 OFFSET 10
SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, query_plan_aggregation_in_order = 1"
SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, query_plan_aggregation_in_order = 1"
check_replicas_read_in_order $query_id
}
@ -64,7 +64,7 @@ test3() {
FROM test.hits
WHERE CounterID = 1704509 AND UserID = 4322253409885123546
GROUP BY URL, EventDate
SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, use_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, parallel_replicas_local_plan=1
SETTINGS optimize_aggregation_in_order = 1, enable_memory_bound_merging_of_aggregation_results = 1, enable_parallel_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_parallel_replicas = 3, parallel_replicas_local_plan=1
)
WHERE explain LIKE '%Aggr%Transform%' OR explain LIKE '%InOrder%'"
}