Merge remote-tracking branch 'origin/master' into musysroot

docker/test/util/process_functional_tests_result.py

@@ -116,7 +116,7 @@ def process_test_log(log_path, broken_tests):
             test[0],
             test[1],
             test[2],
-            "".join(test[3])[:4096].replace("\t", "\\t").replace("\n", "\\n"),
+            "".join(test[3])[:8192].replace("\t", "\\t").replace("\n", "\\n"),
         ]
         for test in test_results
     ]

src/Databases/DatabaseReplicated.cpp

@@ -21,6 +21,7 @@
 #include <Databases/DatabaseReplicatedWorker.h>
 #include <Databases/DDLDependencyVisitor.h>
 #include <Databases/TablesDependencyGraph.h>
+#include <Databases/enableAllExperimentalSettings.h>
 #include <Interpreters/Cluster.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
@@ -1142,39 +1143,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep
 
         /// We will execute some CREATE queries for recovery (not ATTACH queries),
         /// so we need to allow experimental features that can be used in a CREATE query
-        query_context->setSetting("allow_experimental_inverted_index", 1);
-        query_context->setSetting("allow_experimental_full_text_index", 1);
-        query_context->setSetting("allow_experimental_codecs", 1);
-        query_context->setSetting("allow_experimental_live_view", 1);
-        query_context->setSetting("allow_experimental_window_view", 1);
-        query_context->setSetting("allow_experimental_funnel_functions", 1);
-        query_context->setSetting("allow_experimental_nlp_functions", 1);
-        query_context->setSetting("allow_experimental_hash_functions", 1);
-        query_context->setSetting("allow_experimental_object_type", 1);
-        query_context->setSetting("allow_experimental_variant_type", 1);
-        query_context->setSetting("allow_experimental_dynamic_type", 1);
-        query_context->setSetting("allow_experimental_json_type", 1);
-        query_context->setSetting("allow_experimental_vector_similarity_index", 1);
-        query_context->setSetting("allow_experimental_bigint_types", 1);
-        query_context->setSetting("allow_experimental_window_functions", 1);
-        query_context->setSetting("allow_experimental_geo_types", 1);
-        query_context->setSetting("allow_experimental_map_type", 1);
-        query_context->setSetting("allow_deprecated_error_prone_window_functions", 1);
-
-        query_context->setSetting("allow_suspicious_low_cardinality_types", 1);
-        query_context->setSetting("allow_suspicious_fixed_string_types", 1);
-        query_context->setSetting("allow_suspicious_indices", 1);
-        query_context->setSetting("allow_suspicious_codecs", 1);
-        query_context->setSetting("allow_hyperscan", 1);
-        query_context->setSetting("allow_simdjson", 1);
-        query_context->setSetting("allow_deprecated_syntax_for_merge_tree", 1);
-        query_context->setSetting("allow_suspicious_primary_key", 1);
-        query_context->setSetting("allow_suspicious_ttl_expressions", 1);
-        query_context->setSetting("allow_suspicious_variant_types", 1);
-        query_context->setSetting("enable_deflate_qpl_codec", 1);
-        query_context->setSetting("enable_zstd_qat_codec", 1);
-        query_context->setSetting("allow_create_index_without_type", 1);
-        query_context->setSetting("allow_experimental_s3queue", 1);
+        enableAllExperimentalSettings(query_context);
 
         auto txn = std::make_shared<ZooKeeperMetadataTransaction>(current_zookeeper, zookeeper_path, false, "");
         query_context->initZooKeeperMetadataTransaction(txn);

src/Databases/enableAllExperimentalSettings.cpp

@@ -0,0 +1,49 @@
+#include <Interpreters/Context.h>
+
+namespace DB
+{
+
+/*
+ * Enables all settings that allow the use of experimental, deprecated, or potentially unsafe features
+ * in a CREATE query. This function is used in DatabaseReplicated::recoverLostReplica() to create tables
+ * when the original settings used to create the table are not available.
+ */
+
+void enableAllExperimentalSettings(ContextMutablePtr context)
+{
+    context->setSetting("allow_experimental_inverted_index", 1);
+    context->setSetting("allow_experimental_full_text_index", 1);
+    context->setSetting("allow_experimental_codecs", 1);
+    context->setSetting("allow_experimental_live_view", 1);
+    context->setSetting("allow_experimental_window_view", 1);
+    context->setSetting("allow_experimental_funnel_functions", 1);
+    context->setSetting("allow_experimental_nlp_functions", 1);
+    context->setSetting("allow_experimental_hash_functions", 1);
+    context->setSetting("allow_experimental_object_type", 1);
+    context->setSetting("allow_experimental_variant_type", 1);
+    context->setSetting("allow_experimental_dynamic_type", 1);
+    context->setSetting("allow_experimental_json_type", 1);
+    context->setSetting("allow_experimental_vector_similarity_index", 1);
+    context->setSetting("allow_experimental_bigint_types", 1);
+    context->setSetting("allow_experimental_window_functions", 1);
+    context->setSetting("allow_experimental_geo_types", 1);
+    context->setSetting("allow_experimental_map_type", 1);
+    context->setSetting("allow_deprecated_error_prone_window_functions", 1);
+
+    context->setSetting("allow_suspicious_low_cardinality_types", 1);
+    context->setSetting("allow_suspicious_fixed_string_types", 1);
+    context->setSetting("allow_suspicious_indices", 1);
+    context->setSetting("allow_suspicious_codecs", 1);
+    context->setSetting("allow_hyperscan", 1);
+    context->setSetting("allow_simdjson", 1);
+    context->setSetting("allow_deprecated_syntax_for_merge_tree", 1);
+    context->setSetting("allow_suspicious_primary_key", 1);
+    context->setSetting("allow_suspicious_ttl_expressions", 1);
+    context->setSetting("allow_suspicious_variant_types", 1);
+    context->setSetting("enable_deflate_qpl_codec", 1);
+    context->setSetting("enable_zstd_qat_codec", 1);
+    context->setSetting("allow_create_index_without_type", 1);
+    context->setSetting("allow_experimental_s3queue", 1);
+}
+
+}

src/Databases/enableAllExperimentalSettings.h

@@ -0,0 +1,15 @@
+#pragma once
+#include <Interpreters/Context_fwd.h>
+
+namespace DB
+{
+
+/*
+ * Enables all settings that allow the use of experimental, deprecated, or potentially unsafe features
+ * in a CREATE query. This function is used in DatabaseReplicated::recoverLostReplica() to create tables
+ * when the original settings used to create the table are not available.
+ */
+
+void enableAllExperimentalSettings(ContextMutablePtr context);
+
+}

utils/check-style/check-style

@@ -467,3 +467,7 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' |
     grep -vP $EXCLUDE_DIRS |
     xargs grep -F -i 'ErrorCodes::LOGICAL_ERROR, "Logical error:' &&
     echo "If an exception has LOGICAL_ERROR code, there is no need to include the text 'Logical error' in the exception message, because then the phrase 'Logical error' will be printed twice."
+
+PATTERN="allow_";
+DIFF=$(comm -3 <(grep -o "\b$PATTERN\w*\b" $ROOT_PATH/src/Core/Settings.h | sort -u) <(grep -o -h "\b$PATTERN\w*\b" $ROOT_PATH/src/Databases/enableAllExperimentalSettings.cpp $ROOT_PATH/utils/check-style/experimental_settings_ignore.txt | sort -u));
+[ -n "$DIFF" ] && echo "$DIFF" && echo "^^ Detected 'allow_*' settings that might need to be included in src/Databases/enableAllExperimentalSettings.cpp" && echo "Alternatively, consider adding an exception to utils/check-style/experimental_settings_ignore.txt"

utils/check-style/experimental_settings_ignore.txt

@@ -0,0 +1,48 @@
+allow_aggregate_partitions_independently
+allow_archive_path_syntax
+allow_asynchronous_read_from_io_pool_for_merge_tree
+allow_changing_replica_until_first_data_packet
+allow_custom_error_code_in_throwif
+allow_ddl
+allow_deprecated_database_ordinary
+allow_deprecated_snowflake_conversion_functions
+allow_distributed_ddl
+allow_drop_detached
+allow_execute_multiif_columnar
+allow_experimental_alter_materialized_view_structure
+allow_experimental_analyzer
+allow_experimental_annoy_index
+allow_experimental_database_atomic
+allow_experimental_database_materialized_mysql
+allow_experimental_database_materialized_postgresql
+allow_experimental_database_replicated
+allow_experimental_join_condition
+allow_experimental_kafka_offsets_storage_in_keeper
+allow_experimental_lightweight_delete
+allow_experimental_materialized_postgresql_table
+allow_experimental_parallel_reading_from_replicas
+allow_experimental_projection_optimization
+allow_experimental_query_cache
+allow_experimental_query_deduplication
+allow_experimental_refreshable_materialized_view
+allow_experimental_shared_merge_tree
+allow_experimental_statistic
+allow_experimental_statistics
+allow_experimental_time_series_table
+allow_experimental_undrop_table_query
+allow_experimental_usearch_index
+allow_get_client_http_header
+allow_introspection_functions
+allow_materialized_view_with_bad_select
+allow_named_collection_override_by_default
+allow_non_metadata_alters
+allow_nonconst_timezone_arguments
+allow_nondeterministic_mutations
+allow_nondeterministic_optimize_skip_unused_shards
+allow_prefetched_read_pool_for_local_filesystem
+allow_prefetched_read_pool_for_remote_filesystem
+allow_push_predicate_when_subquery_contains_with
+allow_settings_after_format_in_insert
+allow_statistic_optimize
+allow_statistics_optimize
+allow_unrestricted_reads_from_keeper