Merge remote-tracking branch 'origin/master' into igor/use_only_input_order_info

src/Core/SortDescription.h

@@ -48,6 +48,8 @@ struct SortColumnDescription
     bool with_fill;
     FillColumnDescription fill_description;
 
+    SortColumnDescription() = default;
+
     explicit SortColumnDescription(
         const std::string & column_name_,
         int direction_ = 1,

src/Interpreters/ProcessList.cpp

@@ -372,6 +372,12 @@ CancellationCode QueryStatus::cancelQuery(bool)
 
 void QueryStatus::addPipelineExecutor(PipelineExecutor * e)
 {
+    /// In case of asynchronous distributed queries it is possible to call
+    /// addPipelineExecutor() from the cancelQuery() context, and this will
+    /// lead to deadlock.
+    if (is_killed.load())
+        throw Exception("Query was cancelled", ErrorCodes::QUERY_WAS_CANCELLED);
+
     std::lock_guard lock(executors_mutex);
     assert(std::find(executors.begin(), executors.end(), e) == executors.end());
     executors.push_back(e);

src/Processors/QueryPlan/ReadFromMergeTree.cpp

@@ -142,7 +142,12 @@ ReadFromMergeTree::ReadFromMergeTree(
         if (!sort_description.empty())
         {
             if (query_info.getInputOrderInfo())
+            {
                 output_stream->sort_scope = DataStream::SortScope::Stream;
+                const size_t used_prefix_of_sorting_key_size = query_info.getInputOrderInfo()->used_prefix_of_sorting_key_size;
+                if (sort_description.size() > used_prefix_of_sorting_key_size)
+                    sort_description.resize(used_prefix_of_sorting_key_size);
+            }
             else
                 output_stream->sort_scope = DataStream::SortScope::Chunk;
         }

tests/ci/docker_images_check.py

@@ -164,7 +164,7 @@ def gen_versions(
     # The order is important, PR number is used as cache during the build
     versions = [str(pr_info.number), pr_commit_version]
     result_version = pr_commit_version
-    if pr_info.number == 0:
+    if pr_info.number == 0 and pr_info.base_name == "master":
         # First get the latest for cache
         versions.insert(0, "latest")
 

tests/ci/docker_test.py

@@ -99,6 +99,11 @@ class TestDockerImageCheck(unittest.TestCase):
 
     def test_gen_version(self):
         pr_info = PRInfo(PRInfo.default_event.copy())
+        pr_info.base_name = "anything-else"
+        versions, result_version = di.gen_versions(pr_info, None)
+        self.assertEqual(versions, ["0", "0-HEAD"])
+        self.assertEqual(result_version, "0-HEAD")
+        pr_info.base_name = "master"
         versions, result_version = di.gen_versions(pr_info, None)
         self.assertEqual(versions, ["latest", "0", "0-HEAD"])
         self.assertEqual(result_version, "0-HEAD")

tests/clickhouse-test

@@ -1808,6 +1808,9 @@ def main(args):
         args, "system", "processes", "is_all_data_sent"
     )
 
+    if args.s3_storage and (BuildFlags.THREAD in args.build_flags or BuildFlags.DEBUG in args.build_flags):
+        args.no_random_settings = True
+
     if args.skip:
         args.skip = set(args.skip)
 

tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.reference

@@ -21,7 +21,6 @@ PartialSortingTransform
 -- ExpressionStep preserves sort mode
 -- QUERY: set optimize_read_in_order=1;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a FROM optimize_sorting ORDER BY a
 Sorting (Global): a ASC
-Sorting
 Sorting (Global): a ASC
 Sorting (Stream): a ASC
 Sorting (Stream): a ASC
@@ -66,7 +65,6 @@ Sorting (Global): a ASC
 Sorting (Sorting for ORDER BY)
 Sorting (Global): a ASC
 Sorting (None)
-Sorting
 Sorting (Global): a ASC
 Sorting (Stream): a ASC
 Sorting (Stream): a ASC
@@ -87,3 +85,9 @@ Sorting (Sorting for ORDER BY)
 Sorting (Global): plus(a, 1) ASC
 Sorting (Chunk): a ASC
 Sorting (Chunk): a ASC
+-- check that correct sorting info is provided in case of only prefix of sorting key is in ORDER BY clause but all sorting key columns returned by query
+-- QUERY: set optimize_read_in_order=1;EXPLAIN PLAN sorting=1 SELECT a, b FROM optimize_sorting ORDER BY a
+Sorting (Global): a ASC
+Sorting (Global): a ASC
+Sorting (Stream): a ASC
+Sorting (Stream): a ASC

tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh

@@ -8,7 +8,7 @@ DISABLE_OPTIMIZATION="set optimize_sorting_by_input_stream_properties=0;set max_
 ENABLE_OPTIMIZATION="set optimize_sorting_by_input_stream_properties=1;set max_threads=1"
 MAKE_OUTPUT_STABLE="set optimize_read_in_order=1"
 GREP_SORTING="grep 'PartialSortingTransform\|LimitsCheckingTransform\|MergeSortingTransform\|MergingSortedTransform'"
-GREP_SORTMODE="grep 'Sorting'"
+GREP_SORTMODE="grep 'Sorting ('"
 TRIM_LEADING_SPACES="sed -e 's/^[ \t]*//'"
 FIND_SORTING="$GREP_SORTING | $TRIM_LEADING_SPACES"
 FIND_SORTMODE="$GREP_SORTMODE | $TRIM_LEADING_SPACES"
@@ -72,4 +72,7 @@ explain_sortmode "$MAKE_OUTPUT_STABLE;EXPLAIN PLAN actions=1, header=1, sorting=
 echo "-- actions chain breaks sorting order: input(column a)->sipHash64(column a)->alias(sipHash64(column a), a)->plus(alias a, 1)"
 explain_sortmode "$MAKE_OUTPUT_STABLE;EXPLAIN PLAN actions=1, header=1, sorting=1 SELECT a, z FROM (SELECT sipHash64(a) AS a, a + 1 AS z FROM (SELECT a FROM optimize_sorting ORDER BY a + 1)) ORDER BY a + 1"
 
+echo "-- check that correct sorting info is provided in case of only prefix of sorting key is in ORDER BY clause but all sorting key columns returned by query"
+explain_sortmode "$MAKE_OUTPUT_STABLE;EXPLAIN PLAN sorting=1 SELECT a, b FROM optimize_sorting ORDER BY a"
+
 $CLICKHOUSE_CLIENT -q "drop table if exists optimize_sorting sync"