Merge remote-tracking branch 'origin' into changelog-24.2

.github/workflows/backport_branches.yml

@@ -67,8 +67,6 @@ jobs:
       test_name: Compatibility check (amd64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
   CompatibilityCheckAarch64:
     needs: [RunConfig, BuilderDebAarch64]
     if: ${{ !failure() && !cancelled() }}
@@ -77,8 +75,6 @@ jobs:
       test_name: Compatibility check (aarch64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################

.github/workflows/master.yml

@@ -73,8 +73,6 @@ jobs:
       test_name: Compatibility check (amd64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
   CompatibilityCheckAarch64:
     needs: [RunConfig, BuilderDebAarch64]
     if: ${{ !failure() && !cancelled() }}
@@ -83,8 +81,6 @@ jobs:
       test_name: Compatibility check (aarch64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################

.github/workflows/pull_request.yml

@@ -117,8 +117,6 @@ jobs:
       test_name: Compatibility check (amd64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
   CompatibilityCheckAarch64:
     needs: [RunConfig, BuilderDebAarch64]
     if: ${{ !failure() && !cancelled() }}
@@ -127,8 +125,6 @@ jobs:
       test_name: Compatibility check (aarch64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################

.github/workflows/release_branches.yml

@@ -68,8 +68,6 @@ jobs:
       test_name: Compatibility check (amd64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (amd64)" --check-glibc --check-distributions
   CompatibilityCheckAarch64:
     needs: [RunConfig, BuilderDebAarch64]
     if: ${{ !failure() && !cancelled() }}
@@ -78,8 +76,6 @@ jobs:
       test_name: Compatibility check (aarch64)
       runner_type: style-checker
       data: ${{ needs.RunConfig.outputs.data }}
-      run_command: |
-        python3 compatibility_check.py --check-name "Compatibility check (aarch64)" --check-glibc
 #########################################################################################
 #################################### ORDINARY BUILDS ####################################
 #########################################################################################

.gitmessage

@@ -16,6 +16,7 @@
 #ci_set_reduced
 #ci_set_arm
 #ci_set_integration
+#ci_set_analyzer
 
 ## To run specified job in CI:
 #job_<JOB NAME>

docs/en/operations/system-tables/query_thread_log.md

@@ -21,7 +21,7 @@ Columns:
 - `hostname` ([LowCardinality(String)](../../sql-reference/data-types/string.md)) — Hostname of the server executing the query.
 - `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the thread has finished execution of the query.
 - `event_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query.
-- `event_time_microsecinds` ([DateTime](../../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query with microseconds precision.
+- `event_time_microseconds` ([DateTime](../../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query with microseconds precision.
 - `query_start_time` ([DateTime](../../sql-reference/data-types/datetime.md)) — Start time of query execution.
 - `query_start_time_microseconds` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — Start time of query execution with microsecond precision.
 - `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution.
@@ -32,8 +32,7 @@ Columns:
 - `memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — The difference between the amount of allocated and freed memory in context of this thread.
 - `peak_memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — The maximum difference between the amount of allocated and freed memory in context of this thread.
 - `thread_name` ([String](../../sql-reference/data-types/string.md)) — Name of the thread.
-- `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Internal thread ID.
-- `thread_id` ([Int32](../../sql-reference/data-types/int-uint.md)) — thread ID.
+- `thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — OS thread ID.
 - `master_thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — OS initial ID of initial thread.
 - `query` ([String](../../sql-reference/data-types/string.md)) — Query string.
 - `is_initial_query` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Query type. Possible values:

src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.cpp

@@ -0,0 +1,186 @@
+#include <Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.h>
+
+#include <AggregateFunctions/AggregateFunctionFactory.h>
+
+#include <Analyzer/ArrayJoinNode.h>
+#include <Analyzer/ColumnNode.h>
+#include <Analyzer/FunctionNode.h>
+#include <Analyzer/InDepthQueryTreeVisitor.h>
+#include <Analyzer/QueryNode.h>
+#include <Analyzer/TableNode.h>
+#include <Analyzer/UnionNode.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+namespace
+{
+
+/// Try to eliminate min/max/any/anyLast.
+class EliminateFunctionVisitor : public InDepthQueryTreeVisitorWithContext<EliminateFunctionVisitor>
+{
+public:
+    using Base = InDepthQueryTreeVisitorWithContext<EliminateFunctionVisitor>;
+    using Base::Base;
+
+    using GroupByKeysStack = std::vector<QueryTreeNodePtrWithHashSet>;
+
+    void enterImpl(QueryTreeNodePtr & node)
+    {
+        if (!getSettings().optimize_aggregators_of_group_by_keys)
+            return;
+
+        /// Collect group by keys.
+        auto * query_node = node->as<QueryNode>();
+        if (!query_node)
+            return;
+
+        if (!query_node->hasGroupBy())
+        {
+            group_by_keys_stack.push_back({});
+        }
+        else if (query_node->isGroupByWithTotals() || query_node->isGroupByWithCube() || query_node->isGroupByWithRollup())
+        {
+            /// Keep aggregator if group by is with totals/cube/rollup.
+            group_by_keys_stack.push_back({});
+        }
+        else
+        {
+            QueryTreeNodePtrWithHashSet group_by_keys;
+            for (auto & group_key : query_node->getGroupBy().getNodes())
+            {
+                /// For grouping sets case collect only keys that are presented in every set.
+                if (auto * list = group_key->as<ListNode>())
+                {
+                    QueryTreeNodePtrWithHashSet common_keys_set;
+                    for (auto & group_elem : list->getNodes())
+                    {
+                        if (group_by_keys.contains(group_elem))
+                            common_keys_set.insert(group_elem);
+                    }
+                    group_by_keys = std::move(common_keys_set);
+                }
+                else
+                {
+                    group_by_keys.insert(group_key);
+                }
+            }
+            group_by_keys_stack.push_back(std::move(group_by_keys));
+        }
+    }
+
+    /// Now we visit all nodes in QueryNode, we should remove group_by_keys from stack.
+    void leaveImpl(QueryTreeNodePtr & node)
+    {
+        if (!getSettings().optimize_aggregators_of_group_by_keys)
+            return;
+
+        if (node->getNodeType() == QueryTreeNodeType::FUNCTION)
+        {
+            if (aggregationCanBeEliminated(node, group_by_keys_stack.back()))
+                node = node->as<FunctionNode>()->getArguments().getNodes()[0];
+        }
+        else if (node->getNodeType() == QueryTreeNodeType::QUERY)
+        {
+            group_by_keys_stack.pop_back();
+        }
+    }
+
+    static bool needChildVisit(VisitQueryTreeNodeType & parent [[maybe_unused]], VisitQueryTreeNodeType & child)
+    {
+        /// Skip ArrayJoin.
+        return !child->as<ArrayJoinNode>();
+    }
+
+private:
+
+    struct NodeWithInfo
+    {
+        QueryTreeNodePtr node;
+        bool parents_are_only_deterministic = false;
+    };
+
+    bool aggregationCanBeEliminated(QueryTreeNodePtr & node, const QueryTreeNodePtrWithHashSet & group_by_keys)
+    {
+        if (group_by_keys.empty())
+            return false;
+
+        auto * function = node->as<FunctionNode>();
+        if (!function || !function->isAggregateFunction())
+            return false;
+
+        if (!(function->getFunctionName() == "min"
+                || function->getFunctionName() == "max"
+                || function->getFunctionName() == "any"
+                || function->getFunctionName() == "anyLast"))
+            return false;
+
+        std::vector<NodeWithInfo> candidates;
+        auto & function_arguments = function->getArguments().getNodes();
+        if (function_arguments.size() != 1)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected a single argument of function '{}' but received {}", function->getFunctionName(), function_arguments.size());
+
+        if (!function->getResultType()->equals(*function_arguments[0]->getResultType()))
+            return false;
+
+        candidates.push_back({ function_arguments[0], true });
+
+        /// Using DFS we traverse function tree and try to find if it uses other keys as function arguments.
+        while (!candidates.empty())
+        {
+            auto [candidate, parents_are_only_deterministic] = candidates.back();
+            candidates.pop_back();
+
+            bool found = group_by_keys.contains(candidate);
+
+            switch (candidate->getNodeType())
+            {
+                case QueryTreeNodeType::FUNCTION:
+                {
+                    auto * func = candidate->as<FunctionNode>();
+                    auto & arguments = func->getArguments().getNodes();
+                    if (arguments.empty())
+                        return false;
+
+                    if (!found)
+                    {
+                        bool is_deterministic_function = parents_are_only_deterministic &&
+                            func->getFunctionOrThrow()->isDeterministicInScopeOfQuery();
+                        for (auto it = arguments.rbegin(); it != arguments.rend(); ++it)
+                            candidates.push_back({ *it, is_deterministic_function });
+                    }
+                    break;
+                }
+                case QueryTreeNodeType::COLUMN:
+                    if (!found)
+                        return false;
+                    break;
+                case QueryTreeNodeType::CONSTANT:
+                    if (!parents_are_only_deterministic)
+                        return false;
+                    break;
+                default:
+                    return false;
+            }
+        }
+
+        return true;
+    }
+
+    GroupByKeysStack group_by_keys_stack;
+};
+
+}
+
+void AggregateFunctionOfGroupByKeysPass::run(QueryTreeNodePtr & query_tree_node, ContextPtr context)
+{
+    EliminateFunctionVisitor eliminator(context);
+    eliminator.visit(query_tree_node);
+}
+
+};

src/Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include <Analyzer/IQueryTreePass.h>
+
+namespace DB
+{
+
+/** Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section.
+  *
+  * Example: SELECT max(column) FROM table GROUP BY column;
+  * Result: SELECT column FROM table GROUP BY column;
+  */
+class AggregateFunctionOfGroupByKeysPass final : public IQueryTreePass
+{
+public:
+    String getName() override { return "AggregateFunctionOfGroupByKeys"; }
+
+    String getDescription() override
+    {
+        return "Eliminates min/max/any/anyLast aggregators of GROUP BY keys in SELECT section.";
+    }
+
+    void run(QueryTreeNodePtr & query_tree_node, ContextPtr context) override;
+
+};
+
+}
+

src/Analyzer/Passes/OptimizeGroupByFunctionKeysPass.cpp

@@ -92,7 +92,7 @@ private:
                     if (!found)
                     {
                         bool is_deterministic_function = parents_are_only_deterministic &&
-                            function->getFunctionOrThrow()->isDeterministicInScopeOfQuery();
+                            func->getFunctionOrThrow()->isDeterministicInScopeOfQuery();
                         for (auto it = arguments.rbegin(); it != arguments.rend(); ++it)
                             candidates.push_back({ *it, is_deterministic_function });
                     }

src/Analyzer/QueryTreePassManager.cpp

@@ -46,6 +46,7 @@
 #include <Analyzer/Passes/CrossToInnerJoinPass.h>
 #include <Analyzer/Passes/ShardNumColumnToFunctionPass.h>
 #include <Analyzer/Passes/ConvertQueryToCNFPass.h>
+#include <Analyzer/Passes/AggregateFunctionOfGroupByKeysPass.h>
 #include <Analyzer/Passes/OptimizeDateOrDateTimeConverterWithPreimagePass.h>
 
 
@@ -164,7 +165,6 @@ private:
 
 /** ClickHouse query tree pass manager.
   *
-  * TODO: Support setting optimize_aggregators_of_group_by_keys.
   * TODO: Support setting optimize_monotonous_functions_in_order_by.
   * TODO: Add optimizations based on function semantics. Example: SELECT * FROM test_table WHERE id != id. (id is not nullable column).
   */
@@ -264,6 +264,9 @@ void addQueryTreePasses(QueryTreePassManager & manager)
     manager.addPass(std::make_unique<RewriteArrayExistsToHasPass>());
     manager.addPass(std::make_unique<NormalizeCountVariantsPass>());
 
+    /// should before AggregateFunctionsArithmericOperationsPass
+    manager.addPass(std::make_unique<AggregateFunctionOfGroupByKeysPass>());
+
     manager.addPass(std::make_unique<AggregateFunctionsArithmericOperationsPass>());
     manager.addPass(std::make_unique<UniqInjectiveFunctionsEliminationPass>());
     manager.addPass(std::make_unique<OptimizeGroupByFunctionKeysPass>());

src/Common/CgroupsMemoryUsageObserver.cpp

@@ -203,7 +203,7 @@ uint64_t CgroupsMemoryUsageObserver::File::readMemoryUsage() const
     ReadBufferFromFileDescriptor buf(fd);
     buf.rewind();
 
-    uint64_t mem_usage;
+    uint64_t mem_usage = 0;
 
     switch (version)
     {
@@ -214,6 +214,8 @@ uint64_t CgroupsMemoryUsageObserver::File::readMemoryUsage() const
             ///   rss 15
             ///   [...]
             std::string key;
+            bool found_rss = false;
+
             while (!buf.eof())
             {
                 readStringUntilWhitespace(key, buf);
@@ -221,14 +223,20 @@ uint64_t CgroupsMemoryUsageObserver::File::readMemoryUsage() const
                 {
                     std::string dummy;
                     readStringUntilNewlineInto(dummy, buf);
+                    buf.ignore();
                     continue;
                 }
+
                 assertChar(' ', buf);
                 readIntText(mem_usage, buf);
-                assertChar('\n', buf);
+                found_rss = true;
                 break;
             }
-            throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find 'rss' in '{}'", file_name);
+
+            if (!found_rss)
+                throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find 'rss' in '{}'", file_name);
+
+            break;
         }
         case CgroupsVersion::V2:
         {

src/Coordination/Changelog.cpp

@@ -58,9 +58,10 @@ void moveChangelogBetweenDisks(
     const std::string & path_to,
     const KeeperContextPtr & keeper_context)
 {
+    auto path_from = description->path;
     moveFileBetweenDisks(
         disk_from,
-        description->path,
+        path_from,
         disk_to,
         path_to,
         [&]

src/Dictionaries/HashedArrayDictionary.cpp

@@ -975,7 +975,6 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::loadData()
 {
     if (!source_ptr->hasUpdateField())
     {
-
         std::optional<DictionaryParallelLoaderType> parallel_loader;
         if constexpr (sharded)
             parallel_loader.emplace(*this);
@@ -988,6 +987,7 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::loadData()
 
         size_t total_rows = 0;
         size_t total_blocks = 0;
+        String dictionary_name = getFullName();
 
         Block block;
         while (true)
@@ -1007,7 +1007,7 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::loadData()
 
             if (parallel_loader)
             {
-                parallel_loader->addBlock(block);
+                parallel_loader->addBlock(std::move(block));
             }
             else
             {
@@ -1020,10 +1020,12 @@ void HashedArrayDictionary<dictionary_key_type, sharded>::loadData()
         if (parallel_loader)
             parallel_loader->finish();
 
-        LOG_DEBUG(getLogger("HashedArrayDictionary"),
-            "Finished {}reading {} blocks with {} rows from pipeline in {:.2f} sec and inserted into hashtable in {:.2f} sec",
+        LOG_DEBUG(log,
+            "Finished {}reading {} blocks with {} rows to dictionary {} from pipeline in {:.2f} sec and inserted into hashtable in {:.2f} sec",
             configuration.use_async_executor ? "asynchronous " : "",
-            total_blocks, total_rows, pull_time_microseconds / 1000000.0, process_time_microseconds / 1000000.0);
+            total_blocks, total_rows,
+            dictionary_name,
+            pull_time_microseconds / 1000000.0, process_time_microseconds / 1000000.0);
     }
     else
     {
@@ -1167,6 +1169,9 @@ void registerDictionaryArrayHashed(DictionaryFactory & factory)
 
         HashedArrayDictionaryStorageConfiguration configuration{require_nonempty, dict_lifetime, static_cast<size_t>(shards)};
 
+        if (source_ptr->hasUpdateField() && shards > 1)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARDS parameter does not supports for updatable source (UPDATE_FIELD)", full_name);
+
         ContextMutablePtr context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix);
         const auto & settings = context->getSettingsRef();
 

src/Dictionaries/HashedDictionary.h

@@ -374,9 +374,10 @@ HashedDictionary<dictionary_key_type, sparse, sharded>::~HashedDictionary()
         }
     }
 
-    LOG_TRACE(log, "Destroying {} non empty hash tables (using {} threads)", hash_tables_count, pool.getMaxThreads());
+    String dictionary_name = getFullName();
+    LOG_TRACE(log, "Destroying {} non empty hash tables for dictionary {} (using {} threads) ", hash_tables_count, dictionary_name, pool.getMaxThreads());
     pool.wait();
-    LOG_TRACE(log, "Hash tables destroyed");
+    LOG_TRACE(log, "Hash tables for dictionary {} destroyed", dictionary_name);
 }
 
 template <DictionaryKeyType dictionary_key_type, bool sparse, bool sharded>
@@ -833,6 +834,7 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::createAttributes()
     attributes.reserve(size);
 
     HashTableGrowerWithPrecalculationAndMaxLoadFactor grower(configuration.max_load_factor);
+    String dictionary_name = getFullName();
 
     for (const auto & dictionary_attribute : dict_struct.attributes)
     {
@@ -862,9 +864,9 @@ void HashedDictionary<dictionary_key_type, sparse, sharded>::createAttributes()
             }
 
             if constexpr (IsBuiltinHashTable<typename CollectionsHolder<ValueType>::value_type>)
-                LOG_TRACE(log, "Using builtin hash table for {} attribute", dictionary_attribute.name);
+                LOG_TRACE(log, "Using builtin hash table for {} attribute of {}", dictionary_attribute.name, dictionary_name);
             else
-                LOG_TRACE(log, "Using sparsehash for {} attribute", dictionary_attribute.name);
+                LOG_TRACE(log, "Using sparsehash for {} attribute of {}", dictionary_attribute.name, dictionary_name);
         };
 
         callOnDictionaryAttributeType(dictionary_attribute.underlying_type, type_call);

src/Dictionaries/HashedDictionaryParallelLoader.h

@@ -46,12 +46,13 @@ class HashedDictionaryParallelLoader : public boost::noncopyable
 public:
     explicit HashedDictionaryParallelLoader(DictionaryType & dictionary_)
         : dictionary(dictionary_)
+        , dictionary_name(dictionary.getFullName())
         , shards(dictionary.configuration.shards)
         , pool(CurrentMetrics::HashedDictionaryThreads, CurrentMetrics::HashedDictionaryThreadsActive, CurrentMetrics::HashedDictionaryThreadsScheduled, shards)
         , shards_queues(shards)
     {
         UInt64 backlog = dictionary.configuration.shard_load_queue_backlog;
-        LOG_TRACE(dictionary.log, "Will load the dictionary using {} threads (with {} backlog)", shards, backlog);
+        LOG_TRACE(dictionary.log, "Will load the {} dictionary using {} threads (with {} backlog)", dictionary_name, shards, backlog);
 
         shards_slots.resize(shards);
         iota(shards_slots.data(), shards_slots.size(), UInt64(0));
@@ -82,6 +83,7 @@ public:
     {
         IColumn::Selector selector = createShardSelector(block, shards_slots);
         Blocks shards_blocks = splitBlock(selector, block);
+        block.clear();
 
         for (size_t shard = 0; shard < shards; ++shard)
         {
@@ -98,7 +100,7 @@ public:
         Stopwatch watch;
         pool.wait();
         UInt64 elapsed_ms = watch.elapsedMilliseconds();
-        LOG_TRACE(dictionary.log, "Processing the tail took {}ms", elapsed_ms);
+        LOG_TRACE(dictionary.log, "Processing the tail of dictionary {} took {}ms", dictionary_name, elapsed_ms);
     }
 
     ~HashedDictionaryParallelLoader()
@@ -119,6 +121,7 @@ public:
 
 private:
     DictionaryType & dictionary;
+    String dictionary_name;
     const size_t shards;
     ThreadPool pool;
     std::vector<std::optional<ConcurrentBoundedQueue<Block>>> shards_queues;

src/Dictionaries/registerHashedDictionary.cpp

@@ -80,7 +80,7 @@ void registerDictionaryHashed(DictionaryFactory & factory)
         };
 
         if (source_ptr->hasUpdateField() && shards > 1)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,"{}: SHARDS parameter does not supports for updatable source (UPDATE_FIELD)", full_name);
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: SHARDS parameter does not supports for updatable source (UPDATE_FIELD)", full_name);
 
         if (dictionary_key_type == DictionaryKeyType::Simple)
         {

src/Interpreters/ExpressionActions.cpp

@@ -566,7 +566,7 @@ namespace
     };
 }
 
-static void executeAction(const ExpressionActions::Action & action, ExecutionContext & execution_context, bool dry_run)
+static void executeAction(const ExpressionActions::Action & action, ExecutionContext & execution_context, bool dry_run, bool allow_duplicates_in_input)
 {
     auto & inputs = execution_context.inputs;
     auto & columns = execution_context.columns;
@@ -697,14 +697,19 @@ static void executeAction(const ExpressionActions::Action & action, ExecutionCon
                                     action.node->result_name);
             }
             else
-                columns[action.result_position] = std::move(inputs[pos]);
+            {
+                if (allow_duplicates_in_input)
+                    columns[action.result_position] = inputs[pos];
+                else
+                    columns[action.result_position] = std::move(inputs[pos]);
+            }
 
             break;
         }
     }
 }
 
-void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run) const
+void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run, bool allow_duplicates_in_input) const
 {
     ExecutionContext execution_context
     {
@@ -725,7 +730,8 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run)
                 if (execution_context.inputs_pos[input_pos] < 0)
                 {
                     execution_context.inputs_pos[input_pos] = pos;
-                    break;
+                    if (!allow_duplicates_in_input)
+                        break;
                 }
             }
         }
@@ -737,12 +743,8 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run)
     {
         try
         {
-            executeAction(action, execution_context, dry_run);
+            executeAction(action, execution_context, dry_run, allow_duplicates_in_input);
             checkLimits(execution_context.columns);
-
-            //std::cerr << "Action: " << action.toString() << std::endl;
-            //for (const auto & col : execution_context.columns)
-            //    std::cerr << col.dumpStructure() << std::endl;
         }
         catch (Exception & e)
         {
@@ -755,6 +757,12 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run)
     {
         block.clear();
     }
+    else if (allow_duplicates_in_input)
+    {
+        /// This case is the same as when the input is projected
+        /// since we do not need any input columns.
+        block.clear();
+    }
     else
     {
         ::sort(execution_context.inputs_pos.rbegin(), execution_context.inputs_pos.rend());
@@ -777,11 +785,11 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run)
     num_rows = execution_context.num_rows;
 }
 
-void ExpressionActions::execute(Block & block, bool dry_run) const
+void ExpressionActions::execute(Block & block, bool dry_run, bool allow_duplicates_in_input) const
 {
     size_t num_rows = block.rows();
 
-    execute(block, num_rows, dry_run);
+    execute(block, num_rows, dry_run, allow_duplicates_in_input);
 
     if (!block)
         block.insert({DataTypeUInt8().createColumnConst(num_rows, 0), std::make_shared<DataTypeUInt8>(), "_dummy"});

src/Interpreters/ExpressionActions.h

@@ -98,9 +98,15 @@ public:
     const NamesAndTypesList & getRequiredColumnsWithTypes() const { return required_columns; }
 
     /// Execute the expression on the block. The block must contain all the columns returned by getRequiredColumns.
-    void execute(Block & block, size_t & num_rows, bool dry_run = false) const;
+    ///
+    /// @param allow_duplicates_in_input - actions are allowed to have
+    /// duplicated input (that will refer into the block). This is needed for
+    /// preliminary query filtering (filterBlockWithDAG()), because they just
+    /// pass available virtual columns, which cannot be moved in case they are
+    /// used multiple times.
+    void execute(Block & block, size_t & num_rows, bool dry_run = false, bool allow_duplicates_in_input = false) const;
     /// The same, but without `num_rows`. If result block is empty, adds `_dummy` column to keep block size.
-    void execute(Block & block, bool dry_run = false) const;
+    void execute(Block & block, bool dry_run = false, bool allow_duplicates_in_input = false) const;
 
     bool hasArrayJoin() const;
     void assertDeterministic() const;

src/Interpreters/convertFieldToType.cpp

@@ -252,8 +252,21 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
 
         if (which_type.isDateTime64() && src.getType() == Field::Types::Decimal64)
         {
-            /// Already in needed type.
-            return src;
+            const auto & from_type = src.get<Decimal64>();
+            const auto & to_type = static_cast<const DataTypeDateTime64 &>(type);
+
+            const auto scale_from = from_type.getScale();
+            const auto scale_to = to_type.getScale();
+            const auto scale_multiplier_diff = scale_from > scale_to ? from_type.getScaleMultiplier() / to_type.getScaleMultiplier() : to_type.getScaleMultiplier() / from_type.getScaleMultiplier();
+
+            if (scale_multiplier_diff == 1) /// Already in needed type.
+                return src;
+
+            /// in case if we need to make DateTime64(a) from DateTime64(b), a != b, we need to convert datetime value to the right scale
+            const UInt64 value = scale_from > scale_to ? from_type.getValue().value / scale_multiplier_diff : from_type.getValue().value * scale_multiplier_diff;
+            return DecimalField(
+                DecimalUtils::decimalFromComponentsWithMultiplier<DateTime64>(value, 0, 1),
+                scale_to);
         }
 
         /// For toDate('xxx') in 1::Int64, we CAST `src` to UInt64, which may

src/Storages/MergeTree/MergeTreeRangeReader.cpp

@@ -448,21 +448,16 @@ static ColumnPtr andFilters(ColumnPtr c1, ColumnPtr c2)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of filters don't match: {} and {}",
             c1->size(), c2->size());
 
-    // TODO: use proper vectorized implementation of AND?
     auto res = ColumnUInt8::create(c1->size());
     auto & res_data = res->getData();
     const auto & c1_data = typeid_cast<const ColumnUInt8&>(*c1).getData();
     const auto & c2_data = typeid_cast<const ColumnUInt8&>(*c2).getData();
     const size_t size = c1->size();
-    const size_t step = 16;
-    size_t i = 0;
-    /// NOTE: '&&' must be used instead of '&' for 'AND' operation because UInt8 columns might contain any non-zero
-    /// value for true and we cannot bitwise AND them to get the correct result.
-    for (; i + step < size; i += step)
-        for (size_t j = 0; j < step; ++j)
-            res_data[i+j] = (c1_data[i+j] && c2_data[i+j]);
-    for (; i < size; ++i)
-        res_data[i] = (c1_data[i] && c2_data[i]);
+    /// The double NOT operators (!!) convert the non-zeros to the bool value of true (0x01) and zeros to false (0x00).
+    /// After casting them to UInt8, '&' could replace '&&' for the 'AND' operation implementation and at the same
+    /// time enable the auto vectorization.
+    for (size_t i = 0; i < size; ++i)
+        res_data[i] = (static_cast<UInt8>(!!c1_data[i]) & static_cast<UInt8>(!!c2_data[i]));
     return res;
 }
 

src/Storages/MergeTree/tests/gtest_combine_filters.cpp

@@ -138,6 +138,57 @@ bool testCombineColumns(size_t size)
     return true;
 }
 
+/* To ensure the vectorized DB::andFilters works as its scalar implementation, this test validates the AND (&&)
+ * of any combinations of the UInt8 values.
+ */
+bool testAndFilters(size_t size)
+{
+    auto generateFastIncrementColumn = [](size_t len)->ColumnPtr
+    {
+        auto filter = ColumnUInt8::create(len);
+        auto & filter_data = filter->getData();
+
+        for (size_t i = 0; i < len; ++i)
+            filter_data[i] = static_cast<UInt8>(i & 0xFF);
+
+        return filter;
+    };
+
+    auto generateSlowIncrementColumn = [](size_t len)->ColumnPtr
+    {
+        auto filter = ColumnUInt8::create(len);
+        auto & filter_data = filter->getData();
+
+        for (size_t i = 0; i < len; ++i)
+            filter_data[i] = static_cast<UInt8>((i >> 8) & 0xFF);
+
+        return filter;
+    };
+
+    auto first_filter = generateFastIncrementColumn(size);
+    auto second_filter = generateSlowIncrementColumn(size);
+
+    auto result = andFilters(first_filter, second_filter);
+
+    const auto & first_filter_data = typeid_cast<const ColumnUInt8 *>(first_filter.get())->getData();
+    const auto & second_filter_data = typeid_cast<const ColumnUInt8 *>(second_filter.get())->getData();
+    const auto & result_data = typeid_cast<const ColumnUInt8 *>(result.get())->getData();
+
+    if (result->size() != size)
+    {
+        return false;
+    }
+
+    for (size_t i = 0; i < size; i++)
+    {
+        UInt8 expected = first_filter_data[i] && second_filter_data[i];
+        if (result_data[i] != expected)
+            return false;
+    }
+
+    return true;
+}
+
 TEST(MergeTree, CombineFilters)
 {
     /// Tests with only 0/1 and fixed intervals.
@@ -159,3 +210,18 @@ TEST(MergeTree, CombineFilters)
     EXPECT_TRUE(testCombineColumns(2000));
     EXPECT_TRUE(testCombineColumns(200000));
 }
+
+TEST(MergeTree, AndFilters)
+{
+    EXPECT_TRUE(testAndFilters(1));
+    EXPECT_TRUE(testAndFilters(2));
+    EXPECT_TRUE(testAndFilters(15));
+    EXPECT_TRUE(testAndFilters(16));
+    EXPECT_TRUE(testAndFilters(200));
+    EXPECT_TRUE(testAndFilters(201));
+    EXPECT_TRUE(testAndFilters(2000));
+    EXPECT_TRUE(testAndFilters(65535));
+    EXPECT_TRUE(testAndFilters(65536));
+    EXPECT_TRUE(testAndFilters(65537));
+    EXPECT_TRUE(testAndFilters(200000));
+}

src/Storages/StorageS3.cpp

@@ -548,7 +548,15 @@ StorageS3Source::KeyWithInfoPtr StorageS3Source::ReadTaskIterator::next(size_t)
     if (current_index >= buffer.size())
         return std::make_shared<KeyWithInfo>(callback());
 
-    return buffer[current_index];
+    while (current_index < buffer.size())
+    {
+        if (const auto & key_info = buffer[current_index]; key_info && !key_info->key.empty())
+            return buffer[current_index];
+
+        current_index = index.fetch_add(1, std::memory_order_relaxed);
+    }
+
+    return nullptr;
 }
 
 size_t StorageS3Source::ReadTaskIterator::estimatedKeysCount()

src/Storages/VirtualColumnUtils.cpp

@@ -223,7 +223,7 @@ bool prepareFilterBlockWithQuery(const ASTPtr & query, ContextPtr context, Block
 
         auto expression_actions = std::make_shared<ExpressionActions>(actions);
         auto block_with_constants = block;
-        expression_actions->execute(block_with_constants);
+        expression_actions->execute(block_with_constants, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true);
         return block_with_constants.has(expr_column_name) && isColumnConst(*block_with_constants.getByName(expr_column_name).column);
     };
 
@@ -266,7 +266,7 @@ void filterBlockWithDAG(ActionsDAGPtr dag, Block & block, ContextPtr context)
     auto actions = std::make_shared<ExpressionActions>(dag);
     makeSets(actions, context);
     Block block_with_filter = block;
-    actions->execute(block_with_filter);
+    actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true);
 
     /// Filter the block.
     String filter_column_name = dag->getOutputs().at(0)->result_name;
@@ -313,7 +313,7 @@ void filterBlockWithQuery(const ASTPtr & query, Block & block, ContextPtr contex
     makeSets(actions, context);
 
     Block block_with_filter = block;
-    actions->execute(block_with_filter);
+    actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true);
 
     /// Filter the block.
     String filter_column_name = expression_ast->getColumnName();

tests/ci/ci.py

@@ -953,10 +953,18 @@ def _mark_success_action(
     # FIXME: find generic design for propagating and handling job status (e.g. stop using statuses in GH api)
     #   now job ca be build job w/o status data, any other job that exit with 0 with or w/o status data
     if CI_CONFIG.is_build_job(job):
-        # there is no status for build jobs
-        # create dummy success to mark it as done
+        # there is no CommitStatus for build jobs
+        # create dummy status relying on JobReport
         # FIXME: consider creating commit status for build jobs too, to treat everything the same way
-        CommitStatusData(SUCCESS, "dummy description", "dummy_url").dump_status()
+        job_report = JobReport.load() if JobReport.exist() else None
+        if job_report and job_report.status == SUCCESS:
+            CommitStatusData(
+                SUCCESS,
+                "dummy description",
+                "dummy_url",
+                pr_num=pr_info.number,
+                sha=pr_info.sha,
+            ).dump_status()
 
     job_status = None
     if CommitStatusData.exist():

tests/ci/ci_config.py

@@ -21,6 +21,7 @@ class Labels(metaclass=WithIter):
     CI_SET_REDUCED = "ci_set_reduced"
     CI_SET_ARM = "ci_set_arm"
     CI_SET_INTEGRATION = "ci_set_integration"
+    CI_SET_ANALYZER = "ci_set_analyzer"
 
     libFuzzer = "libFuzzer"
 
@@ -398,6 +399,10 @@ bugfix_validate_check = DigestConfig(
     ],
 )
 # common test params
+compatibility_test_common_params = {
+    "digest": compatibility_check_digest,
+    "run_command": "compatibility_check.py",
+}
 statless_test_common_params = {
     "digest": stateless_check_digest,
     "run_command": 'functional_test_check.py "$CHECK_NAME" $KILL_TIMEOUT',
@@ -647,6 +652,16 @@ CI_CONFIG = CIConfig(
                 JobNames.INTEGRATION_TEST,
             ]
         ),
+        Labels.CI_SET_ANALYZER: LabelConfig(
+            run_jobs=[
+                JobNames.STYLE_CHECK,
+                JobNames.FAST_TEST,
+                Build.PACKAGE_RELEASE,
+                Build.PACKAGE_ASAN,
+                JobNames.STATELESS_TEST_ANALYZER_S3_REPLICATED_RELEASE,
+                JobNames.INTEGRATION_TEST_ASAN_ANALYZER,
+            ]
+        ),
         Labels.CI_SET_REDUCED: LabelConfig(
             run_jobs=[
                 job
@@ -1038,13 +1053,13 @@ CI_CONFIG = CIConfig(
         JobNames.COMPATIBILITY_TEST: TestConfig(
             Build.PACKAGE_RELEASE,
             job_config=JobConfig(
-                required_on_release_branch=True, digest=compatibility_check_digest
+                required_on_release_branch=True, **compatibility_test_common_params  # type: ignore
             ),
         ),
         JobNames.COMPATIBILITY_TEST_ARM: TestConfig(
             Build.PACKAGE_AARCH64,
             job_config=JobConfig(
-                required_on_release_branch=True, digest=compatibility_check_digest
+                required_on_release_branch=True, **compatibility_test_common_params  # type: ignore
             ),
         ),
         JobNames.UNIT_TEST: TestConfig(

tests/ci/compatibility_check.py

@@ -2,6 +2,7 @@
 
 import argparse
 import logging
+import os
 import subprocess
 import sys
 from pathlib import Path
@@ -122,11 +123,7 @@ def get_run_commands_distributions(
 
 def parse_args():
     parser = argparse.ArgumentParser("Check compatibility with old distributions")
-    parser.add_argument("--check-name", required=True)
-    parser.add_argument("--check-glibc", action="store_true")
-    parser.add_argument(
-        "--check-distributions", action="store_true"
-    )  # currently hardcoded to x86, don't enable for ARM
+    parser.add_argument("--check-name", required=False)
     return parser.parse_args()
 
 
@@ -134,6 +131,13 @@ def main():
     logging.basicConfig(level=logging.INFO)
 
     args = parse_args()
+    check_name = args.check_name or os.getenv("CHECK_NAME")
+    assert check_name
+    check_glibc = True
+    # currently hardcoded to x86, don't enable for ARM
+    check_distributions = (
+        "aarch64" not in check_name.lower() and "arm64" not in check_name.lower()
+    )
 
     stopwatch = Stopwatch()
 
@@ -150,7 +154,7 @@ def main():
             "clickhouse-common-static_" in url or "clickhouse-server_" in url
         )
 
-    download_builds_filter(args.check_name, reports_path, packages_path, url_filter)
+    download_builds_filter(check_name, reports_path, packages_path, url_filter)
 
     for package in packages_path.iterdir():
         if package.suffix == ".deb":
@@ -166,11 +170,11 @@ def main():
 
     run_commands = []
 
-    if args.check_glibc:
+    if check_glibc:
         check_glibc_commands = get_run_commands_glibc(packages_path, result_path)
         run_commands.extend(check_glibc_commands)
 
-    if args.check_distributions:
+    if check_distributions:
         centos_image = pull_image(get_docker_image(IMAGE_CENTOS))
         ubuntu_image = pull_image(get_docker_image(IMAGE_UBUNTU))
         check_distributions_commands = get_run_commands_distributions(
@@ -195,9 +199,9 @@ def main():
 
     # See https://sourceware.org/glibc/wiki/Glibc%20Timeline
     max_glibc_version = ""
-    if "amd64" in args.check_name:
+    if "amd64" in check_name:
         max_glibc_version = "2.4"
-    elif "aarch64" in args.check_name:
+    elif "aarch64" in check_name:
         max_glibc_version = "2.18"  # because of build with newer sysroot?
     else:
         raise Exception("Can't determine max glibc version")
@@ -205,8 +209,8 @@ def main():
     state, description, test_results, additional_logs = process_result(
         result_path,
         server_log_path,
-        args.check_glibc,
-        args.check_distributions,
+        check_glibc,
+        check_distributions,
         max_glibc_version,
     )
 

tests/ci/docker_images_check.py

@@ -25,7 +25,6 @@ from stopwatch import Stopwatch
 from tee_popen import TeePopen
 from upload_result_helper import upload_results
 
-NAME = "Push to Dockerhub"
 TEMP_PATH = Path(RUNNER_TEMP) / "docker_images_check"
 TEMP_PATH.mkdir(parents=True, exist_ok=True)
 
@@ -177,6 +176,9 @@ def main():
     stopwatch = Stopwatch()
 
     args = parse_args()
+
+    NAME = f"Push to Dockerhub {args.suffix}"
+
     if args.push:
         logging.info("login to docker hub")
         docker_login()

tests/ci/run_check.py

@@ -37,7 +37,8 @@ CAN_BE_TESTED_LABEL = "can be tested"
 FEATURE_LABEL = "pr-feature"
 SUBMODULE_CHANGED_LABEL = "submodule changed"
 PR_CHECK = "PR Check"
-AUTO_BACKPORT_LABELS = ["pr-ci", "pr-bugfix"]
+# pr-bugfix autoport can lead to issues in releases, let's do ci fixes only
+AUTO_BACKPORT_LABELS = ["pr-ci"]
 
 
 def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):

tests/ci/worker/dockerhub_proxy_template.sh

@@ -1,19 +1,7 @@
 #!/usr/bin/env bash
 set -xeuo pipefail
 
-# Add cloudflare DNS as a fallback
-# Get default gateway interface
-IFACE=$(ip --json route list | jq '.[]|select(.dst == "default").dev' --raw-output)
-# `Link 2 (eth0): 172.31.0.2`
-ETH_DNS=$(resolvectl dns "$IFACE") || :
-CLOUDFLARE_NS=1.1.1.1
-if [[ "$ETH_DNS" ]] && [[ "${ETH_DNS#*: }" != *"$CLOUDFLARE_NS"* ]]; then
-  # Cut the leading legend
-  ETH_DNS=${ETH_DNS#*: }
-  # shellcheck disable=SC2206
-  new_dns=(${ETH_DNS} "$CLOUDFLARE_NS")
-  resolvectl dns "$IFACE" "${new_dns[@]}"
-fi
+bash /usr/local/share/scripts/init-network.sh
 
 # tune sysctl for network performance
 cat > /etc/sysctl.d/10-network-memory.conf << EOF

tests/ci/worker/init_runner.sh

@@ -60,19 +60,7 @@ export RUNNER_URL="https://github.com/${RUNNER_ORG}"
 INSTANCE_ID=$(ec2metadata --instance-id)
 export INSTANCE_ID
 
-# Add cloudflare DNS as a fallback
-# Get default gateway interface
-IFACE=$(ip --json route list | jq '.[]|select(.dst == "default").dev' --raw-output)
-# `Link 2 (eth0): 172.31.0.2`
-ETH_DNS=$(resolvectl dns "$IFACE") || :
-CLOUDFLARE_NS=1.1.1.1
-if [[ "$ETH_DNS" ]] && [[ "${ETH_DNS#*: }" != *"$CLOUDFLARE_NS"* ]]; then
-  # Cut the leading legend
-  ETH_DNS=${ETH_DNS#*: }
-  # shellcheck disable=SC2206
-  new_dns=(${ETH_DNS} "$CLOUDFLARE_NS")
-  resolvectl dns "$IFACE" "${new_dns[@]}"
-fi
+bash /usr/local/share/scripts/init-network.sh
 
 # combine labels
 RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:runner-type'].Value" --output text)

tests/ci/worker/prepare-ci-ami.sh

@@ -9,7 +9,7 @@ set -xeuo pipefail
 
 echo "Running prepare script"
 export DEBIAN_FRONTEND=noninteractive
-export RUNNER_VERSION=2.311.0
+export RUNNER_VERSION=2.313.0
 export RUNNER_HOME=/home/ubuntu/actions-runner
 
 deb_arch() {
@@ -138,6 +138,49 @@ dpkg -i /tmp/amazon-cloudwatch-agent.deb
 aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
 systemctl enable amazon-cloudwatch-agent.service
 
+
+echo "Install tailscale"
+# Build get-authkey for tailscale
+docker run --rm -v /usr/local/bin/:/host-local-bin -i golang:alpine sh -ex <<'EOF'
+  CGO_ENABLED=0 go install -tags tag:svc-core-ci-github tailscale.com/cmd/get-authkey@main
+  mv /go/bin/get-authkey /host-local-bin
+EOF
+
+# install tailscale
+curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/$(lsb_release -cs).noarmor.gpg" > /usr/share/keyrings/tailscale-archive-keyring.gpg
+curl -fsSL "https://pkgs.tailscale.com/stable/ubuntu/$(lsb_release -cs).tailscale-keyring.list" > /etc/apt/sources.list.d/tailscale.list
+apt-get update
+apt-get install tailscale --yes --no-install-recommends
+
+
+# Create a common script for the instances
+mkdir /usr/local/share/scripts -p
+cat > /usr/local/share/scripts/init-network.sh << 'EOF'
+#!/usr/bin/env bash
+
+# Add cloudflare DNS as a fallback
+# Get default gateway interface
+IFACE=$(ip --json route list | jq '.[]|select(.dst == "default").dev' --raw-output)
+# `Link 2 (eth0): 172.31.0.2`
+ETH_DNS=$(resolvectl dns "$IFACE") || :
+CLOUDFLARE_NS=1.1.1.1
+if [[ "$ETH_DNS" ]] && [[ "${ETH_DNS#*: }" != *"$CLOUDFLARE_NS"* ]]; then
+  # Cut the leading legend
+  ETH_DNS=${ETH_DNS#*: }
+  # shellcheck disable=SC2206
+  new_dns=(${ETH_DNS} "$CLOUDFLARE_NS")
+  resolvectl dns "$IFACE" "${new_dns[@]}"
+fi
+
+# Setup tailscale, the very first action
+TS_API_CLIENT_ID=$(aws ssm get-parameter --region us-east-1 --name /tailscale/api-client-id --query 'Parameter.Value' --output text --with-decryption)
+TS_API_CLIENT_SECRET=$(aws ssm get-parameter --region us-east-1 --name /tailscale/api-client-secret --query 'Parameter.Value' --output text --with-decryption)
+export TS_API_CLIENT_ID TS_API_CLIENT_SECRET
+TS_AUTHKEY=$(get-authkey -tags tag:svc-core-ci-github -reusable -ephemeral)
+tailscale up --ssh --auth-key="$TS_AUTHKEY" --hostname="ci-runner-$INSTANCE_ID"
+EOF
+
+
 # The following line is used in aws TOE check.
 touch /var/tmp/clickhouse-ci-ami.success
 # END OF THE SCRIPT

tests/queries/0_stateless/02346_non_negative_derivative.reference

@@ -1,63 +1,63 @@
 1
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	340.00000000000006
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	0.20550000000000002
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	0.0000010200000000000004
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	6.165000000000001e-10
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	0.00136
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	8.22e-7
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	1.7000000000000004
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	0.0010275000000000002
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	2040.0000000000005
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	1.233
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	142800.00000000003
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	86.31
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	9792000.000000002
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	5918.400000000001
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	264384000.00000003
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	159796.80000000002
 1979-12-12 21:21:23.000	1.54	0
-1979-12-12 21:21:21.127	3.7	0
-2299-12-31 23:37:36.788	1.1	0
-2299-12-31 23:37:36.789	2.34	0
+1979-12-12 21:21:21.123	1.1	0
+1979-12-12 21:21:21.123	2.34	0
+1979-12-12 21:21:21.127	3.7	2056320000.0000002
 1979-12-12 21:21:21.129	2.1	0
 1979-12-12 21:21:22.000	1.3345	0
 1979-12-12 21:21:23.000	1.54	1242864

tests/queries/0_stateless/02346_non_negative_derivative.sql

@@ -18,7 +18,7 @@ SELECT (
            SELECT
                ts,
                metric,
-               nonNegativeDerivative(metric, ts) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv
+               nonNegativeDerivative(metric, ts) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv
            FROM nnd
            LIMIT 5, 1
        ) = (
@@ -29,37 +29,37 @@ SELECT (
            FROM nnd
            LIMIT 5, 1
        );
-SELECT ts, metric, nonNegativeDerivative(metric, ts) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS deriv FROM nnd;
 -- Nanosecond
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN 2 PRECEDING AND 2 FOLLOWING) AS deriv FROM nnd;
 -- Microsecond
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 4 MICROSECOND) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 4 MICROSECOND) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS deriv FROM nnd;
 -- Millisecond
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 5 MILLISECOND) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 5 MILLISECOND) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN 1 PRECEDING AND 1 FOLLOWING) AS deriv FROM nnd;
 -- Second
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 6 SECOND) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 6 SECOND) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd;
 -- Minute
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 7 MINUTE) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 7 MINUTE) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND 2 FOLLOWING) AS deriv FROM nnd;
 -- Hour
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 8 HOUR) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 8 HOUR) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd;
 -- Day
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 9 DAY) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN 3 PRECEDING AND 3 FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 9 DAY) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN 3 PRECEDING AND 3 FOLLOWING) AS deriv FROM nnd;
 -- Week
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 10 WEEK) OVER (PARTITION BY id>3 ORDER BY ts ASC Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd;
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 10 WEEK) OVER (PARTITION BY id>3 ORDER BY ts, metric ASC Rows BETWEEN 1 PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd;
 
 -- shall not work for month, quarter, year (intervals with floating number of seconds)
 -- Month
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 11 MONTH) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 11 MONTH) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 -- Quarter
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 12 QUARTER) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 12 QUARTER) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 -- Year
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 13 YEAR) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 13 YEAR) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 
 -- test against wrong arguments/types
-SELECT ts, metric, nonNegativeDerivative(metric, 1, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
-SELECT ts, metric, nonNegativeDerivative('string not datetime', ts, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND, id) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
-SELECT ts, metric, nonNegativeDerivative(metric) OVER (PARTITION BY metric ORDER BY ts ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
+SELECT ts, metric, nonNegativeDerivative(metric, 1, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
+SELECT ts, metric, nonNegativeDerivative('string not datetime', ts, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND, id) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
+SELECT ts, metric, nonNegativeDerivative(metric) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
 
 -- cleanup
 DROP TABLE IF EXISTS nnd;

tests/queries/0_stateless/02783_parsedatetimebesteffort_syslog.reference

@@ -1,37 +0,0 @@
-The reference time point is 2023-06-30 23:59:30
-───────────────────────────────────────────────
-The argument is before the reference time point
-───────────────────────────────────────────────
-Row 1:
-──────
-syslog_arg:    Jun 30 23:58:30
-res:           2023-06-30 23:58:30
-res_null:      2023-06-30 23:58:30
-res_zero:      2023-06-30 23:58:30
-res_us:        2023-06-30 23:58:30
-res_us_null:   2023-06-30 23:58:30
-res_us_zero:   2023-06-30 23:58:30
-res64:         2023-06-30 23:58:30.000
-res64_null:    2023-06-30 23:58:30.000
-res64_zero:    2023-06-30 23:58:30.000
-res64_us:      2023-06-30 23:58:30.000
-res64_us_null: 2023-06-30 23:58:30.000
-res64_us_zero: 2023-06-30 23:58:30.000
-──────────────────────────────────────────────
-The argument is after the reference time point
-──────────────────────────────────────────────
-Row 1:
-──────
-syslog_arg:    Jul  1 00:00:30
-res:           2022-07-01 00:00:30
-res_null:      2022-07-01 00:00:30
-res_zero:      2022-07-01 00:00:30
-res_us:        2022-07-01 00:00:30
-res_us_null:   2022-07-01 00:00:30
-res_us_zero:   2022-07-01 00:00:30
-res64:         2022-07-01 00:00:30.000
-res64_null:    2022-07-01 00:00:30.000
-res64_zero:    2022-07-01 00:00:30.000
-res64_us:      2022-07-01 00:00:30.000
-res64_us_null: 2022-07-01 00:00:30.000
-res64_us_zero: 2022-07-01 00:00:30.000

tests/queries/0_stateless/02783_parsedatetimebesteffort_syslog.sql

@@ -1,54 +0,0 @@
-SET session_timezone = 'UTC';
-
-SELECT 'The reference time point is 2023-06-30 23:59:30';
-SELECT '───────────────────────────────────────────────';
-SELECT 'The argument is before the reference time point';
-SELECT '───────────────────────────────────────────────';
-
-WITH
-    toDateTime('2023-06-30 23:59:30') AS dt_ref,
-    now() AS dt_now, 
-    date_sub(MINUTE, 1, dt_now) as dt_before,
-    dateDiff(SECOND, dt_ref, dt_now) AS time_shift,
-    formatDateTime(dt_before, '%b %e %T') AS syslog_before
-SELECT
-    formatDateTime(dt_before - time_shift, '%b %e %T') AS syslog_arg,
-    parseDateTimeBestEffort(syslog_before) - time_shift AS res,
-    parseDateTimeBestEffortOrNull(syslog_before) - time_shift AS res_null,
-    parseDateTimeBestEffortOrZero(syslog_before) - time_shift AS res_zero,
-    parseDateTimeBestEffortUS(syslog_before) - time_shift AS res_us,
-    parseDateTimeBestEffortUSOrNull(syslog_before) - time_shift AS res_us_null,
-    parseDateTimeBestEffortUSOrZero(syslog_before) - time_shift AS res_us_zero,
-    parseDateTime64BestEffort(syslog_before) - time_shift AS res64,
-    parseDateTime64BestEffortOrNull(syslog_before) - time_shift AS res64_null,
-    parseDateTime64BestEffortOrZero(syslog_before) - time_shift AS res64_zero,
-    parseDateTime64BestEffortUS(syslog_before) - time_shift AS res64_us,
-    parseDateTime64BestEffortUSOrNull(syslog_before) - time_shift AS res64_us_null,
-    parseDateTime64BestEffortUSOrZero(syslog_before) - time_shift AS res64_us_zero
-FORMAT Vertical;
-
-SELECT '──────────────────────────────────────────────';
-SELECT 'The argument is after the reference time point';
-SELECT '──────────────────────────────────────────────';
-
-WITH
-    toDateTime('2023-06-30 23:59:30') AS dt_ref,
-    now() AS dt_now, 
-    date_add(MINUTE, 1, dt_now) as dt_after,
-    dateDiff(SECOND, dt_ref, dt_now) AS time_shift,
-    formatDateTime(dt_after, '%b %e %T') AS syslog_after
-SELECT
-    formatDateTime(dt_after - time_shift, '%b %e %T') AS syslog_arg,
-    parseDateTimeBestEffort(syslog_after) - time_shift AS res,
-    parseDateTimeBestEffortOrNull(syslog_after) - time_shift AS res_null,
-    parseDateTimeBestEffortOrZero(syslog_after) - time_shift AS res_zero,
-    parseDateTimeBestEffortUS(syslog_after) - time_shift AS res_us,
-    parseDateTimeBestEffortUSOrNull(syslog_after) - time_shift AS res_us_null,
-    parseDateTimeBestEffortUSOrZero(syslog_after) - time_shift AS res_us_zero,
-    parseDateTime64BestEffort(syslog_after) - time_shift AS res64,
-    parseDateTime64BestEffortOrNull(syslog_after) - time_shift AS res64_null,
-    parseDateTime64BestEffortOrZero(syslog_after) - time_shift AS res64_zero,
-    parseDateTime64BestEffortUS(syslog_after) - time_shift AS res64_us,
-    parseDateTime64BestEffortUSOrNull(syslog_after) - time_shift AS res64_us_null,
-    parseDateTime64BestEffortUSOrZero(syslog_after) - time_shift AS res64_us_zero
-FORMAT Vertical;

tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.reference

@@ -0,0 +1,526 @@
+set optimize_aggregators_of_group_by_keys = 1
+0	0
+0	1
+0	2
+1	0
+1	1
+1	2
+0	0
+0	1
+0	2
+1	0
+1	1
+1	2
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+1
+2
+2
+3
+3
+4
+4
+4
+5
+6
+6
+6
+8
+8
+9
+10
+12
+12
+12
+15
+16
+18
+20
+24
+0
+0
+QUERY id: 0
+  PROJECTION COLUMNS
+    a UInt8
+    b UInt8
+  PROJECTION
+    LIST id: 1, nodes: 2
+      FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 3, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 8, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 10, nodes: 1
+          CONSTANT id: 11, constant_value: UInt64_10000000, constant_value_type: UInt32
+  GROUP BY
+    LIST id: 12, nodes: 2
+      FUNCTION id: 13, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 14, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 15, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 16, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 17, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
+  ORDER BY
+    LIST id: 19, nodes: 2
+      SORT id: 20, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
+            ARGUMENTS
+              LIST id: 3, nodes: 2
+                COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+                CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
+      SORT id: 21, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
+            ARGUMENTS
+              LIST id: 8, nodes: 2
+                COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+                CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
+QUERY id: 0
+  PROJECTION COLUMNS
+    a UInt8
+    b UInt8
+  PROJECTION
+    LIST id: 1, nodes: 2
+      FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 3, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 8, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 10, nodes: 1
+          CONSTANT id: 11, constant_value: UInt64_10000000, constant_value_type: UInt32
+  GROUP BY
+    LIST id: 12, nodes: 2
+      FUNCTION id: 13, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 14, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 15, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 16, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 17, nodes: 2
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+            CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
+  ORDER BY
+    LIST id: 19, nodes: 2
+      SORT id: 20, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 2, function_name: modulo, function_type: ordinary, result_type: UInt8
+            ARGUMENTS
+              LIST id: 3, nodes: 2
+                COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+                CONSTANT id: 6, constant_value: UInt64_2, constant_value_type: UInt8
+      SORT id: 21, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 7, function_name: modulo, function_type: ordinary, result_type: UInt8
+            ARGUMENTS
+              LIST id: 8, nodes: 2
+                COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+                CONSTANT id: 9, constant_value: UInt64_3, constant_value_type: UInt8
+QUERY id: 0
+  PROJECTION COLUMNS
+    a UInt16
+  PROJECTION
+    LIST id: 1, nodes: 1
+      FUNCTION id: 2, function_name: multiply, function_type: ordinary, result_type: UInt16
+        ARGUMENTS
+          LIST id: 3, nodes: 2
+            FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
+              ARGUMENTS
+                LIST id: 5, nodes: 2
+                  COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                  CONSTANT id: 8, constant_value: UInt64_5, constant_value_type: UInt8
+            FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: UInt8
+              ARGUMENTS
+                LIST id: 10, nodes: 2
+                  COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                  CONSTANT id: 11, constant_value: UInt64_7, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 12, nodes: 1
+          CONSTANT id: 13, constant_value: UInt64_10000000, constant_value_type: UInt32
+  GROUP BY
+    LIST id: 14, nodes: 2
+      FUNCTION id: 15, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 16, nodes: 2
+            COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+            CONSTANT id: 17, constant_value: UInt64_7, constant_value_type: UInt8
+      FUNCTION id: 18, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 19, nodes: 2
+            COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+            CONSTANT id: 20, constant_value: UInt64_5, constant_value_type: UInt8
+  ORDER BY
+    LIST id: 21, nodes: 1
+      SORT id: 22, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 2, function_name: multiply, function_type: ordinary, result_type: UInt16
+            ARGUMENTS
+              LIST id: 3, nodes: 2
+                FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 5, nodes: 2
+                      COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                      CONSTANT id: 8, constant_value: UInt64_5, constant_value_type: UInt8
+                FUNCTION id: 9, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 10, nodes: 2
+                      COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                      CONSTANT id: 11, constant_value: UInt64_7, constant_value_type: UInt8
+QUERY id: 0
+  PROJECTION COLUMNS
+    foo UInt64
+  PROJECTION
+    LIST id: 1, nodes: 1
+      COLUMN id: 2, column_name: foo, result_type: UInt64, source_id: 3
+  JOIN TREE
+    QUERY id: 3, alias: __table1, is_subquery: 1
+      PROJECTION COLUMNS
+        foo UInt64
+      PROJECTION
+        LIST id: 4, nodes: 1
+          COLUMN id: 5, column_name: number, result_type: UInt64, source_id: 6
+      JOIN TREE
+        TABLE_FUNCTION id: 6, alias: __table2, table_function_name: numbers
+          ARGUMENTS
+            LIST id: 7, nodes: 1
+              CONSTANT id: 8, constant_value: UInt64_1, constant_value_type: UInt8
+      GROUP BY
+        LIST id: 9, nodes: 1
+          COLUMN id: 5, column_name: number, result_type: UInt64, source_id: 6
+QUERY id: 0
+  PROJECTION COLUMNS
+    min(number) OVER (PARTITION BY modulo(number, 2)) UInt64
+  PROJECTION
+    LIST id: 1, nodes: 1
+      FUNCTION id: 2, function_name: min, function_type: window, result_type: UInt64
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+        WINDOW
+          WINDOW id: 6, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current
+            PARTITION BY
+              LIST id: 7, nodes: 1
+                FUNCTION id: 8, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 9, nodes: 2
+                      COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+                      CONSTANT id: 10, constant_value: UInt64_2, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 11, nodes: 1
+          CONSTANT id: 12, constant_value: UInt64_3, constant_value_type: UInt8
+  GROUP BY
+    LIST id: 13, nodes: 1
+      COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+set optimize_aggregators_of_group_by_keys = 0
+0	0
+0	1
+0	2
+1	0
+1	1
+1	2
+0	0
+0	1
+0	2
+1	0
+1	1
+1	2
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+0
+1
+2
+2
+3
+3
+4
+4
+4
+5
+6
+6
+6
+8
+8
+9
+10
+12
+12
+12
+15
+16
+18
+20
+24
+0
+QUERY id: 0
+  PROJECTION COLUMNS
+    a UInt8
+    b UInt8
+  PROJECTION
+    LIST id: 1, nodes: 2
+      FUNCTION id: 2, function_name: min, function_type: aggregate, result_type: UInt8
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
+              ARGUMENTS
+                LIST id: 5, nodes: 2
+                  COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                  CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 9, function_name: max, function_type: aggregate, result_type: UInt8
+        ARGUMENTS
+          LIST id: 10, nodes: 1
+            FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
+              ARGUMENTS
+                LIST id: 12, nodes: 2
+                  COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                  CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 14, nodes: 1
+          CONSTANT id: 15, constant_value: UInt64_10000000, constant_value_type: UInt32
+  GROUP BY
+    LIST id: 16, nodes: 2
+      FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 18, nodes: 2
+            COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+            CONSTANT id: 19, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 20, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 21, nodes: 2
+            COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+            CONSTANT id: 22, constant_value: UInt64_3, constant_value_type: UInt8
+  ORDER BY
+    LIST id: 23, nodes: 2
+      SORT id: 24, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 2, function_name: min, function_type: aggregate, result_type: UInt8
+            ARGUMENTS
+              LIST id: 3, nodes: 1
+                FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 5, nodes: 2
+                      COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                      CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
+      SORT id: 25, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 9, function_name: max, function_type: aggregate, result_type: UInt8
+            ARGUMENTS
+              LIST id: 10, nodes: 1
+                FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 12, nodes: 2
+                      COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                      CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
+QUERY id: 0
+  PROJECTION COLUMNS
+    a UInt8
+    b UInt8
+  PROJECTION
+    LIST id: 1, nodes: 2
+      FUNCTION id: 2, function_name: any, function_type: aggregate, result_type: UInt8
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
+              ARGUMENTS
+                LIST id: 5, nodes: 2
+                  COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                  CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 9, function_name: anyLast, function_type: aggregate, result_type: UInt8
+        ARGUMENTS
+          LIST id: 10, nodes: 1
+            FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
+              ARGUMENTS
+                LIST id: 12, nodes: 2
+                  COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                  CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 14, nodes: 1
+          CONSTANT id: 15, constant_value: UInt64_10000000, constant_value_type: UInt32
+  GROUP BY
+    LIST id: 16, nodes: 2
+      FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 18, nodes: 2
+            COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+            CONSTANT id: 19, constant_value: UInt64_2, constant_value_type: UInt8
+      FUNCTION id: 20, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 21, nodes: 2
+            COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+            CONSTANT id: 22, constant_value: UInt64_3, constant_value_type: UInt8
+  ORDER BY
+    LIST id: 23, nodes: 2
+      SORT id: 24, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 2, function_name: any, function_type: aggregate, result_type: UInt8
+            ARGUMENTS
+              LIST id: 3, nodes: 1
+                FUNCTION id: 4, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 5, nodes: 2
+                      COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                      CONSTANT id: 8, constant_value: UInt64_2, constant_value_type: UInt8
+      SORT id: 25, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 9, function_name: anyLast, function_type: aggregate, result_type: UInt8
+            ARGUMENTS
+              LIST id: 10, nodes: 1
+                FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 12, nodes: 2
+                      COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
+                      CONSTANT id: 13, constant_value: UInt64_3, constant_value_type: UInt8
+QUERY id: 0
+  PROJECTION COLUMNS
+    a UInt16
+  PROJECTION
+    LIST id: 1, nodes: 1
+      FUNCTION id: 2, function_name: max, function_type: aggregate, result_type: UInt16
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            FUNCTION id: 4, function_name: multiply, function_type: ordinary, result_type: UInt16
+              ARGUMENTS
+                LIST id: 5, nodes: 2
+                  FUNCTION id: 6, function_name: modulo, function_type: ordinary, result_type: UInt8
+                    ARGUMENTS
+                      LIST id: 7, nodes: 2
+                        COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
+                        CONSTANT id: 10, constant_value: UInt64_5, constant_value_type: UInt8
+                  FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
+                    ARGUMENTS
+                      LIST id: 12, nodes: 2
+                        COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
+                        CONSTANT id: 13, constant_value: UInt64_7, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 9, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 14, nodes: 1
+          CONSTANT id: 15, constant_value: UInt64_10000000, constant_value_type: UInt32
+  GROUP BY
+    LIST id: 16, nodes: 2
+      FUNCTION id: 17, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 18, nodes: 2
+            COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
+            CONSTANT id: 19, constant_value: UInt64_7, constant_value_type: UInt8
+      FUNCTION id: 20, function_name: modulo, function_type: ordinary, result_type: UInt8
+        ARGUMENTS
+          LIST id: 21, nodes: 2
+            COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
+            CONSTANT id: 22, constant_value: UInt64_5, constant_value_type: UInt8
+  ORDER BY
+    LIST id: 23, nodes: 1
+      SORT id: 24, sort_direction: ASCENDING, with_fill: 0
+        EXPRESSION
+          FUNCTION id: 2, function_name: max, function_type: aggregate, result_type: UInt16
+            ARGUMENTS
+              LIST id: 3, nodes: 1
+                FUNCTION id: 4, function_name: multiply, function_type: ordinary, result_type: UInt16
+                  ARGUMENTS
+                    LIST id: 5, nodes: 2
+                      FUNCTION id: 6, function_name: modulo, function_type: ordinary, result_type: UInt8
+                        ARGUMENTS
+                          LIST id: 7, nodes: 2
+                            COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
+                            CONSTANT id: 10, constant_value: UInt64_5, constant_value_type: UInt8
+                      FUNCTION id: 11, function_name: modulo, function_type: ordinary, result_type: UInt8
+                        ARGUMENTS
+                          LIST id: 12, nodes: 2
+                            COLUMN id: 8, column_name: number, result_type: UInt64, source_id: 9
+                            CONSTANT id: 13, constant_value: UInt64_7, constant_value_type: UInt8
+QUERY id: 0
+  PROJECTION COLUMNS
+    foo UInt64
+  PROJECTION
+    LIST id: 1, nodes: 1
+      COLUMN id: 2, column_name: foo, result_type: UInt64, source_id: 3
+  JOIN TREE
+    QUERY id: 3, alias: __table1, is_subquery: 1
+      PROJECTION COLUMNS
+        foo UInt64
+      PROJECTION
+        LIST id: 4, nodes: 1
+          FUNCTION id: 5, function_name: anyLast, function_type: aggregate, result_type: UInt64
+            ARGUMENTS
+              LIST id: 6, nodes: 1
+                COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
+      JOIN TREE
+        TABLE_FUNCTION id: 8, alias: __table2, table_function_name: numbers
+          ARGUMENTS
+            LIST id: 9, nodes: 1
+              CONSTANT id: 10, constant_value: UInt64_1, constant_value_type: UInt8
+      GROUP BY
+        LIST id: 11, nodes: 1
+          COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
+QUERY id: 0
+  PROJECTION COLUMNS
+    min(number) OVER (PARTITION BY modulo(number, 2)) UInt64
+  PROJECTION
+    LIST id: 1, nodes: 1
+      FUNCTION id: 2, function_name: min, function_type: window, result_type: UInt64
+        ARGUMENTS
+          LIST id: 3, nodes: 1
+            COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+        WINDOW
+          WINDOW id: 6, frame_type: RANGE, frame_begin_type: unbounded preceding, frame_end_type: current
+            PARTITION BY
+              LIST id: 7, nodes: 1
+                FUNCTION id: 8, function_name: modulo, function_type: ordinary, result_type: UInt8
+                  ARGUMENTS
+                    LIST id: 9, nodes: 2
+                      COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5
+                      CONSTANT id: 10, constant_value: UInt64_2, constant_value_type: UInt8
+  JOIN TREE
+    TABLE_FUNCTION id: 5, alias: __table1, table_function_name: numbers
+      ARGUMENTS
+        LIST id: 11, nodes: 1
+          CONSTANT id: 12, constant_value: UInt64_3, constant_value_type: UInt8
+  GROUP BY
+    LIST id: 13, nodes: 1
+      COLUMN id: 4, column_name: number, result_type: UInt64, source_id: 5

tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.sql

@@ -0,0 +1,39 @@
+set allow_experimental_analyzer = 1;
+set optimize_move_functions_out_of_any = 0;
+
+SELECT 'set optimize_aggregators_of_group_by_keys = 1';
+set optimize_aggregators_of_group_by_keys = 1;
+
+SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
+SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
+SELECT anyLast(number) FROM numbers(1) GROUP BY number;
+
+EXPLAIN QUERY TREE SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+EXPLAIN QUERY TREE SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+EXPLAIN QUERY TREE SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
+EXPLAIN QUERY TREE SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
+
+EXPLAIN QUERY TREE
+SELECT min(number) OVER (PARTITION BY number % 2)
+FROM numbers(3)
+GROUP BY number;
+
+SELECT 'set optimize_aggregators_of_group_by_keys = 0';
+set optimize_aggregators_of_group_by_keys = 0;
+
+SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
+SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
+
+EXPLAIN QUERY TREE SELECT min(number % 2) AS a, max(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+EXPLAIN QUERY TREE SELECT any(number % 2) AS a, anyLast(number % 3) AS b FROM numbers(10000000) GROUP BY number % 2, number % 3 ORDER BY a, b;
+EXPLAIN QUERY TREE SELECT max((number % 5) * (number % 7)) AS a FROM numbers(10000000) GROUP BY number % 7, number % 5 ORDER BY a;
+EXPLAIN QUERY TREE SELECT foo FROM (SELECT anyLast(number) AS foo FROM numbers(1) GROUP BY number);
+
+EXPLAIN QUERY TREE
+SELECT min(number) OVER (PARTITION BY number % 2)
+FROM numbers(3)
+GROUP BY number;

tests/queries/0_stateless/02995_preliminary_filters_duplicated_columns.reference

@@ -0,0 +1,2 @@
+0
+0

tests/queries/0_stateless/02995_preliminary_filters_duplicated_columns.sql

@@ -0,0 +1,6 @@
+-- It is special because actions cannot be reused for SimpleAggregateFunction (see https://github.com/ClickHouse/ClickHouse/pull/54436)
+drop table if exists data;
+create table data (key Int) engine=AggregatingMergeTree() order by tuple();
+insert into data values (0);
+select * from data final prewhere indexHint(_partition_id = 'all') or indexHint(_partition_id = 'all');
+select * from data final prewhere indexHint(_partition_id = 'all') or indexHint(_partition_id = 'all') or indexHint(_partition_id = 'all');

tests/queries/0_stateless/02995_preliminary_filters_duplicated_columns_SimpleAggregateFunction.reference

@@ -0,0 +1 @@
+0

tests/queries/0_stateless/02995_preliminary_filters_duplicated_columns_SimpleAggregateFunction.sql

@@ -0,0 +1,5 @@
+-- It is special because actions cannot be reused for SimpleAggregateFunction (see https://github.com/ClickHouse/ClickHouse/pull/54436)
+drop table if exists data;
+create table data (key SimpleAggregateFunction(max, Int)) engine=AggregatingMergeTree() order by tuple();
+insert into data values (0);
+select * from data final prewhere indexHint(_partition_id = 'all') and key >= -1 where key >= 0;

tests/queries/0_stateless/02997_fix_datetime64_scale_conversion.reference

@@ -0,0 +1,100 @@
+2023-01-01 00:00:00
+2023-01-01 00:00:00
+2023-01-01 01:01:01
+2023-01-01 01:01:01
+2023-01-02 02:02:02
+2023-01-02 02:02:02
+2023-01-03 03:03:03
+2023-01-03 03:03:03
+2023-01-04 04:04:04
+2023-01-04 04:04:04
+2023-01-05 05:05:05
+2023-01-05 05:05:05
+2023-01-06 06:06:06
+2023-01-06 06:06:06
+2023-01-07 07:07:07
+2023-01-07 07:07:07
+2023-01-08 08:08:08
+2023-01-08 08:08:08
+2023-01-09 09:09:09
+2023-01-09 09:09:09
+2023-01-01 00:00:00.00
+2023-01-01 00:00:00.00
+2023-01-01 01:01:01.00
+2023-01-01 01:01:01.10
+2023-01-02 02:02:02.00
+2023-01-02 02:02:02.12
+2023-01-03 03:03:03.00
+2023-01-03 03:03:03.12
+2023-01-04 04:04:04.00
+2023-01-04 04:04:04.12
+2023-01-05 05:05:05.00
+2023-01-05 05:05:05.12
+2023-01-06 06:06:06.00
+2023-01-06 06:06:06.12
+2023-01-07 07:07:07.00
+2023-01-07 07:07:07.12
+2023-01-08 08:08:08.00
+2023-01-08 08:08:08.12
+2023-01-09 09:09:09.00
+2023-01-09 09:09:09.12
+2023-01-01 00:00:00.000
+2023-01-01 00:00:00.000
+2023-01-01 01:01:01.000
+2023-01-01 01:01:01.100
+2023-01-02 02:02:02.000
+2023-01-02 02:02:02.120
+2023-01-03 03:03:03.000
+2023-01-03 03:03:03.123
+2023-01-04 04:04:04.000
+2023-01-04 04:04:04.123
+2023-01-05 05:05:05.000
+2023-01-05 05:05:05.123
+2023-01-06 06:06:06.000
+2023-01-06 06:06:06.123
+2023-01-07 07:07:07.000
+2023-01-07 07:07:07.123
+2023-01-08 08:08:08.000
+2023-01-08 08:08:08.123
+2023-01-09 09:09:09.000
+2023-01-09 09:09:09.123
+2023-01-01 00:00:00.000000
+2023-01-01 00:00:00.000000
+2023-01-01 01:01:01.000000
+2023-01-01 01:01:01.100000
+2023-01-02 02:02:02.000000
+2023-01-02 02:02:02.120000
+2023-01-03 03:03:03.000000
+2023-01-03 03:03:03.123000
+2023-01-04 04:04:04.000000
+2023-01-04 04:04:04.123400
+2023-01-05 05:05:05.000000
+2023-01-05 05:05:05.123450
+2023-01-06 06:06:06.000000
+2023-01-06 06:06:06.123456
+2023-01-07 07:07:07.000000
+2023-01-07 07:07:07.123456
+2023-01-08 08:08:08.000000
+2023-01-08 08:08:08.123456
+2023-01-09 09:09:09.000000
+2023-01-09 09:09:09.123456
+2023-01-01 00:00:00.000000
+2023-01-01 00:00:00.000000
+2023-01-01 01:01:01.000000
+2023-01-01 01:01:01.100000
+2023-01-02 02:02:02.000000
+2023-01-02 02:02:02.120000
+2023-01-03 03:03:03.000000
+2023-01-03 03:03:03.123000
+2023-01-04 04:04:04.000000
+2023-01-04 04:04:04.123400
+2023-01-05 05:05:05.000000
+2023-01-05 05:05:05.123450
+2023-01-06 06:06:06.000000
+2023-01-06 06:06:06.123456
+2023-01-07 07:07:07.000000
+2023-01-07 07:07:07.123456
+2023-01-08 08:08:08.000000
+2023-01-08 08:08:08.123456
+2023-01-09 09:09:09.000000
+2023-01-09 09:09:09.123456

tests/queries/0_stateless/02997_fix_datetime64_scale_conversion.sql

@@ -0,0 +1,124 @@
+DROP TABLE IF EXISTS test_0;
+CREATE TABLE IF NOT EXISTS test_0 (a DateTime64(0)) engine = MergeTree order by a;
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-01 00:00:00', 0));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-01 00:00:00.123456789', 0));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-01 01:01:01', 1));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-01 01:01:01.123456789', 1));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-02 02:02:02', 2));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-02 02:02:02.123456789', 2));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-03 03:03:03', 3));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-03 03:03:03.123456789', 3));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-04 04:04:04', 4));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-04 04:04:04.123456789', 4));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-05 05:05:05', 5));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-05 05:05:05.123456789', 5));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-06 06:06:06', 6));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-06 06:06:06.123456789', 6));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-07 07:07:07', 7));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-07 07:07:07.123456789', 7));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-08 08:08:08', 8));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-08 08:08:08.123456789', 8));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-09 09:09:09', 9));
+INSERT INTO test_0 VALUES (toDateTime64('2023-01-09 09:09:09.123456789', 9));
+SELECT * FROM test_0 ORDER BY a;
+DROP TABLE test_0;
+
+DROP TABLE IF EXISTS test_2;
+CREATE TABLE IF NOT EXISTS test_2 (a DateTime64(2)) engine = MergeTree order by a;
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-01 00:00:00', 0));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-01 00:00:00.123456789', 0));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-01 01:01:01', 1));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-01 01:01:01.123456789', 1));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-02 02:02:02', 2));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-02 02:02:02.123456789', 2));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-03 03:03:03', 3));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-03 03:03:03.123456789', 3));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-04 04:04:04', 4));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-04 04:04:04.123456789', 4));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-05 05:05:05', 5));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-05 05:05:05.123456789', 5));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-06 06:06:06', 6));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-06 06:06:06.123456789', 6));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-07 07:07:07', 7));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-07 07:07:07.123456789', 7));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-08 08:08:08', 8));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-08 08:08:08.123456789', 8));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-09 09:09:09', 9));
+INSERT INTO test_2 VALUES (toDateTime64('2023-01-09 09:09:09.123456789', 9));
+SELECT * FROM test_2 ORDER BY a;
+DROP TABLE test_2;
+
+DROP TABLE IF EXISTS test_3;
+CREATE TABLE IF NOT EXISTS test_3 (a DateTime64(3)) engine = MergeTree order by a;
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-01 00:00:00', 0));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-01 00:00:00.123456789', 0));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-01 01:01:01', 1));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-01 01:01:01.123456789', 1));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-02 02:02:02', 2));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-02 02:02:02.123456789', 2));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-03 03:03:03', 3));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-03 03:03:03.123456789', 3));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-04 04:04:04', 4));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-04 04:04:04.123456789', 4));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-05 05:05:05', 5));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-05 05:05:05.123456789', 5));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-06 06:06:06', 6));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-06 06:06:06.123456789', 6));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-07 07:07:07', 7));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-07 07:07:07.123456789', 7));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-08 08:08:08', 8));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-08 08:08:08.123456789', 8));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-09 09:09:09', 9));
+INSERT INTO test_3 VALUES (toDateTime64('2023-01-09 09:09:09.123456789', 9));
+SELECT * FROM test_3 ORDER BY a;
+DROP TABLE test_3;
+
+DROP TABLE IF EXISTS test_6;
+CREATE TABLE IF NOT EXISTS test_6 (a DateTime64(6)) engine = MergeTree order by a;
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-01 00:00:00', 0));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-01 00:00:00.123456789', 0));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-01 01:01:01', 1));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-01 01:01:01.123456789', 1));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-02 02:02:02', 2));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-02 02:02:02.123456789', 2));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-03 03:03:03', 3));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-03 03:03:03.123456789', 3));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-04 04:04:04', 4));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-04 04:04:04.123456789', 4));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-05 05:05:05', 5));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-05 05:05:05.123456789', 5));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-06 06:06:06', 6));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-06 06:06:06.123456789', 6));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-07 07:07:07', 7));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-07 07:07:07.123456789', 7));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-08 08:08:08', 8));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-08 08:08:08.123456789', 8));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-09 09:09:09', 9));
+INSERT INTO test_6 VALUES (toDateTime64('2023-01-09 09:09:09.123456789', 9));
+SELECT * FROM test_6 ORDER BY a;
+DROP TABLE test_6;
+
+DROP TABLE IF EXISTS test_9;
+CREATE TABLE IF NOT EXISTS test_9 (a DateTime64(6)) engine = MergeTree order by a;
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-01 00:00:00', 0));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-01 00:00:00.123456789', 0));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-01 01:01:01', 1));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-01 01:01:01.123456789', 1));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-02 02:02:02', 2));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-02 02:02:02.123456789', 2));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-03 03:03:03', 3));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-03 03:03:03.123456789', 3));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-04 04:04:04', 4));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-04 04:04:04.123456789', 4));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-05 05:05:05', 5));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-05 05:05:05.123456789', 5));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-06 06:06:06', 6));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-06 06:06:06.123456789', 6));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-07 07:07:07', 7));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-07 07:07:07.123456789', 7));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-08 08:08:08', 8));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-08 08:08:08.123456789', 8));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-09 09:09:09', 9));
+INSERT INTO test_9 VALUES (toDateTime64('2023-01-09 09:09:09.123456789', 9));
+SELECT * FROM test_9 ORDER BY a;
+DROP TABLE test_9;

tests/queries/0_stateless/02998_attach_partition_not_allowed_if_structure_differs_due_to_materialized_column.sql

@@ -0,0 +1,21 @@
+CREATE TABLE attach_partition_t7 (
+                                     a UInt32,
+                                     b UInt32
+)
+    ENGINE = MergeTree
+PARTITION BY a ORDER BY a;
+
+ALTER TABLE attach_partition_t7
+    ADD COLUMN mat_column
+        UInt32 MATERIALIZED a+b;
+
+insert into attach_partition_t7 values (1, 2);
+
+CREATE TABLE attach_partition_t8 (
+                                     a UInt32,
+                                     b UInt32
+)
+    ENGINE = MergeTree
+PARTITION BY a ORDER BY a;
+
+ALTER TABLE attach_partition_t8 ATTACH PARTITION ID '1' FROM attach_partition_t7; -- {serverError INCOMPATIBLE_COLUMNS};

utils/check-style/check-style

@@ -50,11 +50,6 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/n
 # Broken symlinks
 find -L $ROOT_PATH -type l 2>/dev/null | grep -v contrib && echo "^ Broken symlinks found"
 
-# Double whitespaces
-find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
-    grep -vP $EXCLUDE_DIRS |
-    while read i; do $ROOT_PATH/utils/check-style/double-whitespaces.pl < $i || echo -e "^ File $i contains double whitespaces\n"; done
-
 # Unused/Undefined/Duplicates ErrorCodes/ProfileEvents/CurrentMetrics
 declare -A EXTERN_TYPES
 EXTERN_TYPES[ErrorCodes]=int

utils/check-style/check-whitespaces

@@ -2,8 +2,9 @@
 
 ROOT_PATH=$(git rev-parse --show-toplevel)
 EXCLUDE_DIRS='build/|integration/|widechar_width/|glibc-compatibility/|memcpy/|consistent-hashing/|Parsers/New'
+NPROC=$(($(nproc) + 3))
 
 # Double whitespaces
 find $ROOT_PATH/{src,base,programs,utils} -name '*.h' -or -name '*.cpp' 2>/dev/null |
     grep -vP $EXCLUDE_DIRS |
-    while read i; do $ROOT_PATH/utils/check-style/double-whitespaces.pl < $i || echo -e "^ File $i contains double whitespaces\n"; done
+    xargs -P "$NPROC" -n 20 "${ROOT_PATH}/utils/check-style/double-whitespaces.pl"

utils/check-style/double-whitespaces.pl

@@ -5,27 +5,31 @@ use strict;
 # Find double whitespace such as "a,  b, c" that looks very ugly and annoying.
 # But skip double whitespaces if they are used as an alignment - by comparing to surrounding lines.
 
-my @array;
-
-while (<>)
-{
-    push @array, $_;
-}
-
 my $ret = 0;
 
-for (my $i = 1; $i < $#array; ++$i)
+foreach my $file (@ARGV)
 {
-    if ($array[$i] =~ ',( {2,3})[^ /]')
-    {
-        # https://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
+    my @array;
 
-        if ((substr($array[$i - 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) # whitespaces are not part of alignment
-         && (substr($array[$i + 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/)
-         && $array[$i] !~ /(-?\d+\w*,\s+){3,}/) # this is not a number table like { 10, -1,  2 }
+    open (FH,'<',$file);
+    while (<FH>)
+    {
+        push @array, $_;
+    }
+
+    for (my $i = 1; $i < $#array; ++$i)
+    {
+        if ($array[$i] =~ ',( {2,3})[^ /]')
         {
-            print(($i + 1) . ":" . $array[$i]);
-            $ret = 1;
+            # https://stackoverflow.com/questions/87380/how-can-i-find-the-location-of-a-regex-match-in-perl
+
+            if ((substr($array[$i - 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/) # whitespaces are not part of alignment
+             && (substr($array[$i + 1], $+[1] - 1, 2) !~ /^[ -][^ ]$/)
+             && $array[$i] !~ /(-?\d+\w*,\s+){3,}/) # this is not a number table like { 10, -1,  2 }
+            {
+                print($file . ":" . ($i + 1) . $array[$i]);
+                $ret = 1;
+            }
         }
     }
 }