Merge 8953babcd0 into e0f8b8d351

Fix ephemeral column comment

docker/server/README.md

@@ -16,16 +16,18 @@ ClickHouse works 100-1000x faster than traditional database management systems,
 
 For more information and documentation see https://clickhouse.com/.
 
-<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
 ## Versions
 
 -	The `latest` tag points to the latest release of the latest stable branch.
 -	Branch tags like `22.2` point to the latest release of the corresponding branch.
--	Full version tags like `22.2.3.5` point to the corresponding release.
+-	Full version tags like `22.2.3` and `22.2.3.5` point to the corresponding release.
+<!-- docker-official-library:off -->
+<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
 -	The tag `head` is built from the latest commit to the default branch.
 -	Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
 
 <!-- REMOVE UNTIL HERE -->
+<!-- docker-official-library:on -->
 ### Compatibility
 
 -	The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.

docker/server/README.src/content.md

@@ -10,16 +10,18 @@ ClickHouse works 100-1000x faster than traditional database management systems,
 
 For more information and documentation see https://clickhouse.com/.
 
-<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
 ## Versions
 
 -	The `latest` tag points to the latest release of the latest stable branch.
 -	Branch tags like `22.2` point to the latest release of the corresponding branch.
--	Full version tags like `22.2.3.5` point to the corresponding release.
+-	Full version tags like `22.2.3` and `22.2.3.5` point to the corresponding release.
+<!-- docker-official-library:off -->
+<!-- This is not related to the docker official library, remove it before commit to https://github.com/docker-library/docs -->
 -	The tag `head` is built from the latest commit to the default branch.
 -	Each tag has optional `-alpine` suffix to reflect that it's built on top of `alpine`.
 
 <!-- REMOVE UNTIL HERE -->
+<!-- docker-official-library:on -->
 ### Compatibility
 
 -	The amd64 image requires support for [SSE3 instructions](https://en.wikipedia.org/wiki/SSE3). Virtually all x86 CPUs after 2005 support SSE3.

docs/changelogs/v24.8.1.2684-lts.md

@@ -522,4 +522,3 @@ sidebar_label: 2024
 * Backported in [#68518](https://github.com/ClickHouse/ClickHouse/issues/68518): Minor update in Dynamic/JSON serializations. [#68459](https://github.com/ClickHouse/ClickHouse/pull/68459) ([Kruglov Pavel](https://github.com/Avogar)).
 * Backported in [#68558](https://github.com/ClickHouse/ClickHouse/issues/68558): CI: Minor release workflow fix. [#68536](https://github.com/ClickHouse/ClickHouse/pull/68536) ([Max K.](https://github.com/maxknv)).
 * Backported in [#68576](https://github.com/ClickHouse/ClickHouse/issues/68576): CI: Tidy build timeout from 2h to 3h. [#68567](https://github.com/ClickHouse/ClickHouse/pull/68567) ([Max K.](https://github.com/maxknv)).
-

docs/changelogs/v24.9.1.3278-stable.md

@@ -497,4 +497,3 @@ sidebar_label: 2024
 * Backported in [#69899](https://github.com/ClickHouse/ClickHouse/issues/69899): Revert "Merge pull request [#69032](https://github.com/ClickHouse/ClickHouse/issues/69032) from alexon1234/include_real_time_execution_in_http_header". [#69885](https://github.com/ClickHouse/ClickHouse/pull/69885) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Backported in [#69931](https://github.com/ClickHouse/ClickHouse/issues/69931): RIPE is an acronym and thus should be capital. RIPE stands for **R**ACE **I**ntegrity **P**rimitives **E**valuation and RACE stands for **R**esearch and Development in **A**dvanced **C**ommunications **T**echnologies in **E**urope. [#69901](https://github.com/ClickHouse/ClickHouse/pull/69901) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Backported in [#70034](https://github.com/ClickHouse/ClickHouse/issues/70034): Revert "Add RIPEMD160 function". [#70005](https://github.com/ClickHouse/ClickHouse/pull/70005) ([Robert Schulze](https://github.com/rschu1ze)).
-

docs/en/operations/settings/merge-tree-settings.md

@@ -1091,7 +1091,7 @@ Default value: 0 bytes.
 
 Note that if both `min_free_disk_bytes_to_perform_insert` and `min_free_disk_ratio_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory.
 
-## min_free_disk_ratio_to_perform_insert 
+## min_free_disk_ratio_to_perform_insert
 
 The minimum free to total disk space ratio to perform an `INSERT`. Must be a floating point value between 0 and 1. Note that this setting:
 - takes into account the `keep_free_space_bytes` setting.
@@ -1106,6 +1106,31 @@ Default value: 0.0
 
 Note that if both `min_free_disk_ratio_to_perform_insert` and `min_free_disk_bytes_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory.
 
+## allow_experimental_reverse_key
+
+Enables support for descending sort order in MergeTree sorting keys. This setting is particularly useful for time series analysis and Top-N queries, allowing data to be stored in reverse chronological order to optimize query performance.
+
+With `allow_experimental_reverse_key` enabled, you can define descending sort orders within the `ORDER BY` clause of a MergeTree table. This enables the use of more efficient `ReadInOrder` optimizations instead of `ReadInReverseOrder` for descending queries.
+
+**Example**
+
+```sql
+CREATE TABLE example
+(
+    time DateTime,
+    key Int32,
+    value String
+) ENGINE = MergeTree
+ORDER BY (time DESC, key)  -- Descending order on 'time' field
+SETTINGS allow_experimental_reverse_key = 1;
+
+SELECT * FROM example WHERE key = 'xxx' ORDER BY time DESC LIMIT 10;
+```
+
+By using `ORDER BY time DESC` in the query, `ReadInOrder` is applied.
+
+**Default Value:** false
+
 ## cache_populated_by_fetch
 
 A Cloud only setting. 
@@ -1114,4 +1139,4 @@ When `cache_populated_by_fetch` is disabled (the default setting), new data part
 
 If enabled, `cache_populated_by_fetch` will instead cause all nodes to load new data parts from storage into their cache without requiring a query to trigger such an action. 
 
-Default value: 0.
+Default value: 0.

docs/en/sql-reference/functions/tuple-map-functions.md

@@ -936,4 +936,4 @@ SELECT mapPartialReverseSort((k, v) -> v, 2, map('k1', 3, 'k2', 1, 'k3', 2));
 ┌─mapPartialReverseSort(lambda(tuple(k, v), v), 2, map('k1', 3, 'k2', 1, 'k3', 2))─┐
 │ {'k1':3,'k3':2,'k2':1}                                                           │
 └──────────────────────────────────────────────────────────────────────────────────┘
-```
+```

programs/server/dashboard.html

@@ -476,7 +476,7 @@
             <input id="edit" type="button" value="✎" style="display: none;">
             <input id="add" type="button" value="Add chart" style="display: none;">
             <input id="reload" type="button" value="Reload">
-            <span id="search-span" class="nowrap" style="display: none;"><input id="search" type="button" value="🔎" title="Run query to obtain list of charts from ClickHouse"><input id="search-query" name="search" type="text" spellcheck="false"></span>
+            <span id="search-span" class="nowrap" style="display: none;"><input id="search" type="button" value="🔎" title="Run query to obtain list of charts from ClickHouse. Either select dashboard name or write your own query"><input id="search-query" name="search" list="search-options" type="text" spellcheck="false"><datalist id="search-options"></datalist></span>
             <div id="chart-params"></div>
         </div>
     </form>
@@ -532,9 +532,15 @@ const errorMessages = [
     }
 ]
 
+/// Dashboard selector
+const dashboardSearchQuery = (dashboard_name) => `SELECT title, query FROM system.dashboards WHERE dashboard = '${dashboard_name}'`;
+let dashboard_queries = {
+    "Overview": dashboardSearchQuery("Overview"),
+};
+const default_dashboard = 'Overview';
 
 /// Query to fill `queries` list for the dashboard
-let search_query = `SELECT title, query FROM system.dashboards WHERE dashboard = 'Overview'`;
+let search_query = dashboardSearchQuery(default_dashboard);
 let customized = false;
 let queries = [];
 
@@ -1439,7 +1445,7 @@ async function reloadAll(do_search) {
     try {
         updateParams();
         if (do_search) {
-            search_query = document.getElementById('search-query').value;
+            search_query = toSearchQuery(document.getElementById('search-query').value);
             queries = [];
             refreshCustomized(false);
         }
@@ -1504,7 +1510,7 @@ function updateFromState() {
     document.getElementById('url').value = host;
     document.getElementById('user').value = user;
     document.getElementById('password').value = password;
-    document.getElementById('search-query').value = search_query;
+    document.getElementById('search-query').value = fromSearchQuery(search_query);
     refreshCustomized();
 }
 
@@ -1543,6 +1549,44 @@ if (window.location.hash) {
     } catch {}
 }
 
+function fromSearchQuery(query) {
+    for (const dashboard_name in dashboard_queries) {
+        if (query == dashboard_queries[dashboard_name])
+            return dashboard_name;
+    }
+    return query;
+}
+
+function toSearchQuery(value) {
+    if (value in dashboard_queries)
+        return dashboard_queries[value];
+    else
+        return value;
+}
+
+async function populateSearchOptions() {
+    let {reply, error} = await doFetch("SELECT dashboard FROM system.dashboards GROUP BY dashboard ORDER BY ALL");
+    if (error) {
+        throw new Error(error);
+    }
+    let data = reply.data;
+    if (data.dashboard.length == 0) {
+        console.log("Unable to fetch dashboards list");
+        return;
+    }
+    dashboard_queries = {};
+    for (let i = 0; i < data.dashboard.length; i++) {
+        const dashboard = data.dashboard[i];
+        dashboard_queries[dashboard] = dashboardSearchQuery(dashboard);
+    }
+    const searchOptions = document.getElementById('search-options');
+    for (const dashboard in dashboard_queries) {
+        const opt = document.createElement('option');
+        opt.value = dashboard;
+        searchOptions.appendChild(opt);
+    }
+}
+
 async function start() {
     try {
         updateFromState();
@@ -1558,6 +1602,7 @@ async function start() {
         } else {
             drawAll();
         }
+        await populateSearchOptions();
     } catch (e) {
         showError(e.message);
     }

src/Analyzer/Resolve/IdentifierResolver.cpp

@@ -528,7 +528,7 @@ QueryTreeNodePtr IdentifierResolver::tryResolveIdentifierFromCompoundExpression(
   *
   * Resolve strategy:
   * 1. Try to bind identifier to scope argument name to node map.
-  * 2. If identifier is binded but expression context and node type are incompatible return nullptr.
+  * 2. If identifier is bound but expression context and node type are incompatible return nullptr.
   *
   * It is important to support edge cases, where we lookup for table or function node, but argument has same name.
   * Example: WITH (x -> x + 1) AS func, (func -> func(1) + func) AS lambda SELECT lambda(1);

src/Client/ReplxxLineReader.cpp

@@ -362,7 +362,7 @@ ReplxxLineReader::ReplxxLineReader(
     if (highlighter)
         rx.set_highlighter_callback(highlighter);
 
-    /// By default C-p/C-n binded to COMPLETE_NEXT/COMPLETE_PREV,
+    /// By default C-p/C-n bound to COMPLETE_NEXT/COMPLETE_PREV,
     /// bind C-p/C-n to history-previous/history-next like readline.
     rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); });
     rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); });
@@ -384,9 +384,9 @@ ReplxxLineReader::ReplxxLineReader(
     rx.bind_key(Replxx::KEY::control('J'), commit_action);
     rx.bind_key(Replxx::KEY::ENTER, commit_action);
 
-    /// By default COMPLETE_NEXT/COMPLETE_PREV was binded to C-p/C-n, re-bind
+    /// By default COMPLETE_NEXT/COMPLETE_PREV was bound to C-p/C-n, re-bind
     /// to M-P/M-N (that was used for HISTORY_COMMON_PREFIX_SEARCH before, but
-    /// it also binded to M-p/M-n).
+    /// it also bound to M-p/M-n).
     rx.bind_key(Replxx::KEY::meta('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_NEXT, code); });
     rx.bind_key(Replxx::KEY::meta('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMPLETE_PREVIOUS, code); });
     /// By default M-BACKSPACE is KILL_TO_WHITESPACE_ON_LEFT, while in readline it is backward-kill-word

src/Interpreters/MutationsInterpreter.cpp

@@ -1449,6 +1449,7 @@ size_t MutationsInterpreter::evaluateCommandsSize()
 std::optional<SortDescription> MutationsInterpreter::getStorageSortDescriptionIfPossible(const Block & header) const
 {
     Names sort_columns = metadata_snapshot->getSortingKeyColumns();
+    std::vector<bool> reverse_flags = metadata_snapshot->getSortingKeyReverseFlags();
     SortDescription sort_description;
     size_t sort_columns_size = sort_columns.size();
     sort_description.reserve(sort_columns_size);
@@ -1456,9 +1457,16 @@ std::optional<SortDescription> MutationsInterpreter::getStorageSortDescriptionIf
     for (size_t i = 0; i < sort_columns_size; ++i)
     {
         if (header.has(sort_columns[i]))
-            sort_description.emplace_back(sort_columns[i], 1, 1);
+        {
+            if (!reverse_flags.empty() && reverse_flags[i])
+                sort_description.emplace_back(sort_columns[i], -1, 1);
+            else
+                sort_description.emplace_back(sort_columns[i], 1, 1);
+        }
         else
+        {
             return {};
+        }
     }
 
     return sort_description;

src/Parsers/ASTOrderByElement.cpp

@@ -62,4 +62,18 @@ void ASTOrderByElement::formatImpl(const FormatSettings & settings, FormatState
     }
 }
 
+void ASTStorageOrderByElement::updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const
+{
+    hash_state.update(direction);
+    IAST::updateTreeHashImpl(hash_state, ignore_aliases);
+}
+
+void ASTStorageOrderByElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
+{
+    children.front()->formatImpl(settings, state, frame);
+
+    if (direction == -1)
+        settings.ostr << (settings.hilite ? hilite_keyword : "") << " DESC" << (settings.hilite ? hilite_none : "");
+}
+
 }

src/Parsers/ASTOrderByElement.h

@@ -85,4 +85,23 @@ private:
     std::unordered_map<Child, size_t> positions;
 };
 
+class ASTStorageOrderByElement : public IAST
+{
+public:
+    int direction = 1; /// 1 for ASC, -1 for DESC
+
+    ASTPtr clone() const override
+    {
+        auto clone = std::make_shared<ASTStorageOrderByElement>(*this);
+        clone->cloneChildren();
+        return clone;
+    }
+
+    String getID(char) const override { return "StorageOrderByElement"; }
+    void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override;
+
+protected:
+    void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
+};
+
 }

src/Parsers/ExpressionElementParsers.cpp

@@ -2163,6 +2163,39 @@ bool ParserWithOptionalAlias::parseImpl(Pos & pos, ASTPtr & node, Expected & exp
     return true;
 }
 
+bool ParserStorageOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ParserExpression elem_p;
+    ParserKeyword ascending(Keyword::ASCENDING);
+    ParserKeyword descending(Keyword::DESCENDING);
+    ParserKeyword asc(Keyword::ASC);
+    ParserKeyword desc(Keyword::DESC);
+
+    ASTPtr expr_elem;
+    if (!elem_p.parse(pos, expr_elem, expected))
+        return false;
+
+    if (!allow_order)
+    {
+        node = std::move(expr_elem);
+        return true;
+    }
+
+    int direction = 1;
+
+    if (descending.ignore(pos, expected) || desc.ignore(pos, expected))
+        direction = -1;
+    else
+        ascending.ignore(pos, expected) || asc.ignore(pos, expected);
+
+    auto storage_elem = std::make_shared<ASTStorageOrderByElement>();
+    storage_elem->children.push_back(std::move(expr_elem));
+    storage_elem->direction = direction;
+
+    node = std::move(storage_elem);
+    return true;
+}
+
 
 bool ParserOrderByElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {

src/Parsers/ExpressionElementParsers.h

@@ -432,6 +432,19 @@ protected:
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 };
 
+/** Element of storage ORDER BY expression - same as expression element, but in addition, ASC[ENDING] | DESC[ENDING] could be specified
+  */
+class ParserStorageOrderByElement : public IParserBase
+{
+public:
+    explicit ParserStorageOrderByElement(bool allow_order_) : allow_order(allow_order_) {}
+
+protected:
+    bool allow_order;
+
+    const char * getName() const override { return "element of storage ORDER BY expression"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
 
 /** Element of ORDER BY expression - same as expression element, but in addition, ASC[ENDING] | DESC[ENDING] could be specified
   *  and optionally, NULLS LAST|FIRST

src/Parsers/ExpressionListParsers.cpp

@@ -325,6 +325,12 @@ bool ParserNotEmptyExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected
     return nested_parser.parse(pos, node, expected) && !node->children.empty();
 }
 
+bool ParserStorageOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    return ParserList(std::make_unique<ParserStorageOrderByElement>(allow_order), std::make_unique<ParserToken>(TokenType::Comma), false)
+        .parse(pos, node, expected);
+}
+
 bool ParserOrderByExpressionList::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
     return ParserList(std::make_unique<ParserOrderByElement>(), std::make_unique<ParserToken>(TokenType::Comma), false)

src/Parsers/ExpressionListParsers.h

@@ -249,6 +249,17 @@ protected:
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 };
 
+class ParserStorageOrderByExpressionList : public IParserBase
+{
+public:
+    explicit ParserStorageOrderByExpressionList(bool allow_order_) : allow_order(allow_order_) {}
+
+protected:
+    bool allow_order;
+
+    const char * getName() const override { return "storage order by expression"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
 
 class ParserOrderByExpressionList : public IParserBase
 {

src/Parsers/ParserCreateQuery.cpp

@@ -26,6 +26,7 @@
 #include <Parsers/ParserViewTargets.h>
 #include <Common/typeid_cast.h>
 #include <Parsers/ASTColumnDeclaration.h>
+#include <Parsers/ASTOrderByElement.h>
 
 
 namespace DB
@@ -494,6 +495,47 @@ bool ParserTablePropertiesDeclarationList::parseImpl(Pos & pos, ASTPtr & node, E
     return true;
 }
 
+bool ParserStorageOrderByClause::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ParserStorageOrderByExpressionList order_list_p(allow_order);
+    ParserStorageOrderByElement order_elem_p(allow_order);
+    ParserToken s_lparen(TokenType::OpeningRoundBracket);
+    ParserToken s_rparen(TokenType::ClosingRoundBracket);
+
+    ASTPtr order_by;
+
+    /// Check possible ASC|DESC suffix for single key
+    if (order_elem_p.parse(pos, order_by, expected))
+    {
+        /// This is needed because 'order by (x, y)' is parsed as tuple.
+        /// We can remove ASTStorageOrderByElement if no ASC|DESC suffix was specified.
+        if (const auto * elem = order_by->as<ASTStorageOrderByElement>(); elem && elem->direction > 0)
+            order_by = elem->children.front();
+
+        node = order_by;
+        return true;
+    }
+
+    /// Check possible ASC|DESC suffix for a list of keys
+    if (pos->type == TokenType::BareWord && std::string_view(pos->begin, pos->size()) == "tuple")
+        ++pos;
+
+    if (!s_lparen.ignore(pos, expected))
+        return false;
+
+    if (!order_list_p.parse(pos, order_by, expected))
+        order_by = std::make_shared<ASTExpressionList>();
+
+    if (!s_rparen.ignore(pos, expected))
+        return false;
+
+    auto tuple_function = std::make_shared<ASTFunction>();
+    tuple_function->name = "tuple";
+    tuple_function->arguments = std::move(order_by);
+
+    node = std::move(tuple_function);
+    return true;
+}
 
 bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
@@ -508,6 +550,7 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 
     ParserIdentifierWithOptionalParameters ident_with_optional_params_p;
     ParserExpression expression_p;
+    ParserStorageOrderByClause order_by_p(/*allow_order_*/ true);
     ParserSetQuery settings_p(/* parse_only_internals_ = */ true);
     ParserTTLExpressionList parser_ttl_list;
     ParserStringLiteral string_literal_parser;
@@ -556,7 +599,7 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 
         if (!order_by && s_order_by.ignore(pos, expected))
         {
-            if (expression_p.parse(pos, order_by, expected))
+            if (order_by_p.parse(pos, order_by, expected))
             {
                 storage_like = true;
                 continue;

src/Parsers/ParserCreateQuery.h

@@ -88,6 +88,17 @@ protected:
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
 };
 
+class ParserStorageOrderByClause : public IParserBase
+{
+public:
+    explicit ParserStorageOrderByClause(bool allow_order_) : allow_order(allow_order_) {}
+
+protected:
+    bool allow_order;
+
+    const char * getName() const override { return "storage order by clause"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
 
 template <typename NameParser>
 class IParserColumnDeclaration : public IParserBase
@@ -237,6 +248,7 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
             null_modifier.emplace(true);
     }
 
+    bool is_comment = false;
     /// Collate is also allowed after NULL/NOT NULL
     if (!collation_expression && s_collate.ignore(pos, expected)
         && !collation_parser.parse(pos, collation_expression, expected))
@@ -254,7 +266,9 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
     else if (s_ephemeral.ignore(pos, expected))
     {
         default_specifier = s_ephemeral.getName();
-        if (!expr_parser.parse(pos, default_expression, expected) && type)
+        if (s_comment.ignore(pos, expected))
+            is_comment = true;
+        if ((is_comment || !expr_parser.parse(pos, default_expression, expected)) && type)
         {
             ephemeral_default = true;
 
@@ -289,19 +303,22 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
     if (require_type && !type && !default_expression)
         return false; /// reject column name without type
 
-    if ((type || default_expression) && allow_null_modifiers && !null_modifier.has_value())
+    if (!is_comment)
     {
-        if (s_not.ignore(pos, expected))
+        if ((type || default_expression) && allow_null_modifiers && !null_modifier.has_value())
         {
-            if (!s_null.ignore(pos, expected))
-                return false;
-            null_modifier.emplace(false);
+            if (s_not.ignore(pos, expected))
+            {
+                if (!s_null.ignore(pos, expected))
+                    return false;
+                null_modifier.emplace(false);
+            }
+            else if (s_null.ignore(pos, expected))
+                null_modifier.emplace(true);
         }
-        else if (s_null.ignore(pos, expected))
-            null_modifier.emplace(true);
     }
 
-    if (s_comment.ignore(pos, expected))
+    if (is_comment || s_comment.ignore(pos, expected))
     {
         /// should be followed by a string literal
         if (!string_literal_parser.parse(pos, comment_expression, expected))

src/Parsers/ParserProjectionSelectQuery.cpp

@@ -22,7 +22,7 @@ bool ParserProjectionSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
 
     ParserNotEmptyExpressionList exp_list_for_with_clause(false);
     ParserNotEmptyExpressionList exp_list_for_select_clause(true); /// Allows aliases without AS keyword.
-    ParserExpression order_expression_p;
+    ParserStorageOrderByExpressionList order_list_p(/*allow_order_*/ false);
 
     ASTPtr with_expression_list;
     ASTPtr select_expression_list;
@@ -59,7 +59,7 @@ bool ParserProjectionSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     if (s_order_by.ignore(pos, expected))
     {
         ASTPtr expr_list;
-        if (!ParserList(std::make_unique<ParserExpression>(), std::make_unique<ParserToken>(TokenType::Comma)).parse(pos, expr_list, expected))
+        if (!order_list_p.parse(pos, expr_list, expected))
             return false;
 
         if (expr_list->children.size() == 1)

src/Planner/PlannerActionsVisitor.cpp

@@ -926,6 +926,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
 PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitFunction(const QueryTreeNodePtr & node)
 {
     const auto & function_node = node->as<FunctionNode &>();
+
     if (function_node.getFunctionName() == "indexHint")
         return visitIndexHintFunction(node);
 

src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp

@@ -391,8 +391,8 @@ SortingInputOrder buildInputOrderFromSortDescription(
     }
 
     /// This is a result direction we will read from MergeTree
-    ///  1 - in order,
-    /// -1 - in reverse order,
+    ///  1 - in same order of keys,
+    /// -1 - in reverse order of keys,
     ///  0 - usual read, don't apply optimization
     ///
     /// So far, 0 means any direction is possible. It is ok for constant prefix.
@@ -415,6 +415,7 @@ SortingInputOrder buildInputOrderFromSortDescription(
     while (next_description_column < description.size() && next_sort_key < sorting_key.column_names.size())
     {
         const auto & sorting_key_column = sorting_key.column_names[next_sort_key];
+        int reverse_indicator = (!sorting_key.reverse_flags.empty() && sorting_key.reverse_flags[next_sort_key]) ? -1 : 1;
         const auto & sort_column_description = description[next_description_column];
 
         /// If required order depend on collation, it cannot be matched with primary key order.
@@ -425,7 +426,7 @@ SortingInputOrder buildInputOrderFromSortDescription(
         /// Since sorting key columns are always sorted with NULLS LAST, reading in order
         /// supported only for ASC NULLS LAST ("in order"), and DESC NULLS FIRST ("reverse")
         const auto column_is_nullable = sorting_key.data_types[next_sort_key]->isNullable();
-        if (column_is_nullable && sort_column_description.nulls_direction != 1)
+        if (column_is_nullable && sort_column_description.nulls_direction != sort_column_description.direction)
             break;
 
         /// Direction for current sort key.
@@ -448,8 +449,7 @@ SortingInputOrder buildInputOrderFromSortDescription(
             if (sort_column_description.column_name != sorting_key_column)
                 break;
 
-            current_direction = sort_column_description.direction;
-
+            current_direction = sort_column_description.direction * reverse_indicator;
 
             //std::cerr << "====== (no dag) Found direct match" << std::endl;
 
@@ -477,7 +477,7 @@ SortingInputOrder buildInputOrderFromSortDescription(
                 ///          'SELECT x, y FROM table WHERE x = 42 ORDER BY x + 1, y + 1'
                 /// Here, 'x + 1' would be a fixed point. But it is reasonable to read-in-order.
 
-                current_direction = sort_column_description.direction;
+                current_direction = sort_column_description.direction * reverse_indicator;
                 if (match.monotonicity)
                 {
                     current_direction *= match.monotonicity->direction;

src/Processors/QueryPlan/ReadFromMergeTree.cpp

@@ -225,10 +225,15 @@ static bool checkAllPartsOnRemoteFS(const RangesInDataParts & parts)
 
 /// build sort description for output stream
 static SortDescription getSortDescriptionForOutputHeader(
-    const Header & output_header, const Names & sorting_key_columns, const int sort_direction, InputOrderInfoPtr input_order_info, PrewhereInfoPtr prewhere_info, bool enable_vertical_final)
+    const Header & output_header,
+    const Names & sorting_key_columns,
+    const int sort_direction,
+    InputOrderInfoPtr input_order_info,
+    PrewhereInfoPtr prewhere_info,
+    bool enable_vertical_final)
 {
     /// Updating sort description can be done after PREWHERE actions are applied to the header.
-    /// Aftert PREWHERE actions are applied, column names in header can differ from storage column names due to aliases
+    /// After PREWHERE actions are applied, column names in header can differ from storage column names due to aliases
     /// To mitigate it, we're trying to build original header and use it to deduce sorting description
     /// TODO: this approach is fragile, it'd be more robust to update sorting description for the whole plan during plan optimization
     Block original_header = output_header.cloneEmpty();
@@ -1418,6 +1423,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
             continue;
 
         Names sort_columns = storage_snapshot->metadata->getSortingKeyColumns();
+        std::vector<bool> reverse_flags = storage_snapshot->metadata->getSortingKeyReverseFlags();
         SortDescription sort_description;
         sort_description.compile_sort_description = settings[Setting::compile_sort_description];
         sort_description.min_count_to_compile_sort_description = settings[Setting::min_count_to_compile_sort_description];
@@ -1428,7 +1434,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
         Names partition_key_columns = storage_snapshot->metadata->getPartitionKey().column_names;
 
         for (size_t i = 0; i < sort_columns_size; ++i)
-            sort_description.emplace_back(sort_columns[i], 1, 1);
+        {
+            if (!reverse_flags.empty() && reverse_flags[i])
+                sort_description.emplace_back(sort_columns[i], -1, 1);
+            else
+                sort_description.emplace_back(sort_columns[i], 1, 1);
+        }
 
         for (auto & pipe : pipes)
             addMergingFinal(

src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp

@@ -411,7 +411,7 @@ ReadFromSystemNumbersStep::ReadFromSystemNumbersStep(
         context_)
     , column_names{column_names_}
     , storage{std::move(storage_)}
-    , key_expression{KeyDescription::parse(column_names[0], storage_snapshot->metadata->columns, context).expression}
+    , key_expression{KeyDescription::parse(column_names[0], storage_snapshot->metadata->columns, context, false).expression}
     , max_block_size{max_block_size_}
     , num_streams{num_streams_}
     , limit_length_and_offset(InterpreterSelectQuery::getLimitLengthAndOffset(query_info.query->as<ASTSelectQuery &>(), context))

src/Storages/KeyDescription.cpp

@@ -9,7 +9,8 @@
 #include <Storages/extractKeyExpressionList.h>
 #include <Common/quoteString.h>
 #include <Interpreters/FunctionNameNormalizer.h>
-#include <Parsers/ExpressionListParsers.h>
+#include <Parsers/ASTOrderByElement.h>
+#include <Parsers/ParserCreateQuery.h>
 #include <Parsers/parseQuery.h>
 
 
@@ -27,6 +28,7 @@ KeyDescription::KeyDescription(const KeyDescription & other)
     , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr)
     , sample_block(other.sample_block)
     , column_names(other.column_names)
+    , reverse_flags(other.reverse_flags)
     , data_types(other.data_types)
     , additional_column(other.additional_column)
 {
@@ -49,7 +51,6 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other)
     else
         expression_list_ast.reset();
 
-
     if (other.expression)
         expression = other.expression->clone();
     else
@@ -57,6 +58,7 @@ KeyDescription & KeyDescription::operator=(const KeyDescription & other)
 
     sample_block = other.sample_block;
     column_names = other.column_names;
+    reverse_flags = other.reverse_flags;
     data_types = other.data_types;
 
     /// additional_column is constant property It should never be lost.
@@ -122,18 +124,38 @@ KeyDescription KeyDescription::getSortingKeyFromAST(
 {
     KeyDescription result;
     result.definition_ast = definition_ast;
-    result.expression_list_ast = extractKeyExpressionList(definition_ast);
+    auto key_expression_list = extractKeyExpressionList(definition_ast);
+
+    result.expression_list_ast = std::make_shared<ASTExpressionList>();
+    for (const auto & child : key_expression_list->children)
+    {
+        auto real_key = child;
+        if (auto * elem = child->as<ASTStorageOrderByElement>())
+        {
+            real_key = elem->children.front();
+            result.reverse_flags.emplace_back(elem->direction < 0);
+        }
+
+        result.expression_list_ast->children.push_back(real_key);
+        result.column_names.emplace_back(real_key->getColumnName());
+    }
 
     if (additional_column)
     {
         result.additional_column = additional_column;
         ASTPtr column_identifier = std::make_shared<ASTIdentifier>(*additional_column);
+        result.column_names.emplace_back(column_identifier->getColumnName());
         result.expression_list_ast->children.push_back(column_identifier);
+
+        if (!result.reverse_flags.empty())
+            result.reverse_flags.emplace_back(false);
     }
 
-    const auto & children = result.expression_list_ast->children;
-    for (const auto & child : children)
-        result.column_names.emplace_back(child->getColumnName());
+    if (!result.reverse_flags.empty() && result.reverse_flags.size() != result.expression_list_ast->children.size())
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "The size of reverse_flags ({}) does not match the size of KeyDescription {}",
+            result.reverse_flags.size(), result.expression_list_ast->children.size());
 
     {
         auto expr = result.expression_list_ast->clone();
@@ -168,6 +190,24 @@ KeyDescription KeyDescription::getSortingKeyFromAST(
     return result;
 }
 
+ASTPtr KeyDescription::getOriginalExpressionList() const
+{
+    if (!expression_list_ast || reverse_flags.empty())
+        return expression_list_ast;
+
+    auto expr_list = std::make_shared<ASTExpressionList>();
+    size_t size = expression_list_ast->children.size();
+    for (size_t i = 0; i < size; ++i)
+    {
+        auto column_ast = std::make_shared<ASTStorageOrderByElement>();
+        column_ast->children.push_back(expression_list_ast->children[i]);
+        column_ast->direction = (!reverse_flags.empty() && reverse_flags[i]) ? -1 : 1;
+        expr_list->children.push_back(std::move(column_ast));
+    }
+
+    return expr_list;
+}
+
 KeyDescription KeyDescription::buildEmptyKey()
 {
     KeyDescription result;
@@ -176,13 +216,13 @@ KeyDescription KeyDescription::buildEmptyKey()
     return result;
 }
 
-KeyDescription KeyDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context)
+KeyDescription KeyDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context, bool allow_order)
 {
     KeyDescription result;
     if (str.empty())
         return result;
 
-    ParserExpression parser;
+    ParserStorageOrderByClause parser(allow_order);
     ASTPtr ast = parseQuery(parser, "(" + str + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
     FunctionNameNormalizer::visit(ast.get());
 

src/Storages/KeyDescription.h

@@ -14,7 +14,7 @@ struct KeyDescription
     /// primary key in merge tree can be part of sorting key)
     ASTPtr definition_ast;
 
-    /// ASTExpressionList with key fields, example: (x, toStartOfMonth(date))).
+    /// ASTExpressionList with key fields, example: (x DESC, toStartOfMonth(date))).
     ASTPtr expression_list_ast;
 
     /// Expression from expression_list_ast created by ExpressionAnalyzer. Useful,
@@ -27,6 +27,9 @@ struct KeyDescription
     /// Column names in key definition, example: x, toStartOfMonth(date), a * b.
     Names column_names;
 
+    /// Indicator of key column being sorted reversely, example: x DESC, y -> {1, 0}.
+    std::vector<bool> reverse_flags;
+
     /// Types from sample block ordered in columns order.
     DataTypes data_types;
 
@@ -67,6 +70,8 @@ struct KeyDescription
         const ColumnsDescription & columns,
         ContextPtr context);
 
+    ASTPtr getOriginalExpressionList() const;
+
     KeyDescription() = default;
 
     /// We need custom copy constructors because we don't want
@@ -78,7 +83,7 @@ struct KeyDescription
     static bool moduloToModuloLegacyRecursive(ASTPtr node_expr);
 
     /// Parse description from string
-    static KeyDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context);
+    static KeyDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context, bool allow_order);
 };
 
 }

src/Storages/MergeTree/MergeTask.cpp

@@ -1763,6 +1763,7 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const
     /// Merge
     {
         Names sort_columns = global_ctx->metadata_snapshot->getSortingKeyColumns();
+        std::vector<bool> reverse_flags = global_ctx->metadata_snapshot->getSortingKeyReverseFlags();
         sort_description.compile_sort_description = global_ctx->data->getContext()->getSettingsRef()[Setting::compile_sort_description];
         sort_description.min_count_to_compile_sort_description = global_ctx->data->getContext()->getSettingsRef()[Setting::min_count_to_compile_sort_description];
 
@@ -1772,7 +1773,12 @@ void MergeTask::ExecuteAndFinalizeHorizontalPart::createMergedStream() const
         Names partition_key_columns = global_ctx->metadata_snapshot->getPartitionKey().column_names;
 
         for (size_t i = 0; i < sort_columns_size; ++i)
-            sort_description.emplace_back(sort_columns[i], 1, 1);
+        {
+            if (!reverse_flags.empty() && reverse_flags[i])
+                sort_description.emplace_back(sort_columns[i], -1, 1);
+            else
+                sort_description.emplace_back(sort_columns[i], 1, 1);
+        }
 
         const bool is_vertical_merge = (global_ctx->chosen_merge_algorithm == MergeAlgorithm::Vertical);
         /// If merge is vertical we cannot calculate it

src/Storages/MergeTree/MergeTreeData.cpp

@@ -185,6 +185,7 @@ namespace Setting
 
 namespace MergeTreeSetting
 {
+    extern const MergeTreeSettingsBool allow_experimental_reverse_key;
     extern const MergeTreeSettingsBool allow_nullable_key;
     extern const MergeTreeSettingsBool allow_remote_fs_zero_copy_replication;
     extern const MergeTreeSettingsBool allow_suspicious_indices;
@@ -460,6 +461,7 @@ MergeTreeData::MergeTreeData(
     bool sanity_checks = mode <= LoadingStrictnessLevel::CREATE;
 
     allow_nullable_key = !sanity_checks || (*settings)[MergeTreeSetting::allow_nullable_key];
+    allow_reverse_key = !sanity_checks || (*settings)[MergeTreeSetting::allow_experimental_reverse_key];
 
     /// Check sanity of MergeTreeSettings. Only when table is created.
     if (sanity_checks)
@@ -666,12 +668,28 @@ void MergeTreeData::checkProperties(
     const StorageInMemoryMetadata & old_metadata,
     bool attach,
     bool allow_empty_sorting_key,
+    bool allow_reverse_sorting_key,
     bool allow_nullable_key_,
     ContextPtr local_context) const
 {
     if (!new_metadata.sorting_key.definition_ast && !allow_empty_sorting_key)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "ORDER BY cannot be empty");
 
+    if (!allow_reverse_sorting_key)
+    {
+        size_t num_sorting_keys = new_metadata.sorting_key.column_names.size();
+        for (size_t i = 0; i < num_sorting_keys; ++i)
+        {
+            if (!new_metadata.sorting_key.reverse_flags.empty() && new_metadata.sorting_key.reverse_flags[i])
+            {
+                throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Sorting key {} is reversed, but merge tree setting `allow_experimental_reverse_key` is disabled",
+                    new_metadata.sorting_key.column_names[i]);
+            }
+        }
+    }
+
     KeyDescription new_sorting_key = new_metadata.sorting_key;
     KeyDescription new_primary_key = new_metadata.primary_key;
 
@@ -800,7 +818,14 @@ void MergeTreeData::checkProperties(
 
             /// We cannot alter a projection so far. So here we do not try to find a projection in old metadata.
             bool is_aggregate = projection.type == ProjectionDescription::Type::Aggregate;
-            checkProperties(*projection.metadata, *projection.metadata, attach, is_aggregate, true /* allow_nullable_key */, local_context);
+            checkProperties(
+                *projection.metadata,
+                *projection.metadata,
+                attach,
+                is_aggregate,
+                allow_reverse_key,
+                true /* allow_nullable_key */,
+                local_context);
             projections_names.insert(projection.name);
         }
     }
@@ -820,7 +845,14 @@ void MergeTreeData::setProperties(
     bool attach,
     ContextPtr local_context)
 {
-    checkProperties(new_metadata, old_metadata, attach, false, allow_nullable_key, local_context);
+    checkProperties(
+        new_metadata,
+        old_metadata,
+        attach,
+        false,
+        allow_reverse_key,
+        allow_nullable_key,
+        local_context);
     setInMemoryMetadata(new_metadata);
     setVirtuals(createVirtuals(new_metadata));
 }
@@ -3701,7 +3733,7 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
     }
 
     checkColumnFilenamesForCollision(new_metadata, /*throw_on_error=*/ true);
-    checkProperties(new_metadata, old_metadata, false, false, allow_nullable_key, local_context);
+    checkProperties(new_metadata, old_metadata, false, false, allow_reverse_key, allow_nullable_key, local_context);
     checkTTLExpressions(new_metadata, old_metadata);
 
     if (!columns_to_check_conversion.empty())

src/Storages/MergeTree/MergeTreeData.h

@@ -1334,6 +1334,7 @@ protected:
         const StorageInMemoryMetadata & old_metadata,
         bool attach,
         bool allow_empty_sorting_key,
+        bool allow_reverse_sorting_key,
         bool allow_nullable_key_,
         ContextPtr local_context) const;
 
@@ -1714,7 +1715,8 @@ private:
 
     virtual void startBackgroundMovesIfNeeded() = 0;
 
-    bool allow_nullable_key{};
+    bool allow_nullable_key = false;
+    bool allow_reverse_key = false;
 
     void addPartContributionToDataVolume(const DataPartPtr & part);
     void removePartContributionToDataVolume(const DataPartPtr & part);

src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp

@@ -1070,6 +1070,7 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
 
     const auto & primary_key = metadata_snapshot->getPrimaryKey();
     auto index_columns = std::make_shared<ColumnsWithTypeAndName>();
+    std::vector<bool> reverse_flags;
     const auto & key_indices = key_condition.getKeyIndices();
     DataTypes key_types;
     if (!key_indices.empty())
@@ -1079,9 +1080,15 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
         for (size_t i : key_indices)
         {
             if (i < index->size())
+            {
                 index_columns->emplace_back(index->at(i), primary_key.data_types[i], primary_key.column_names[i]);
+                reverse_flags.push_back(!primary_key.reverse_flags.empty() && primary_key.reverse_flags[i]);
+            }
             else
+            {
                 index_columns->emplace_back(); /// The column of the primary key was not loaded in memory - we'll skip it.
+                reverse_flags.push_back(false);
+            }
 
             key_types.emplace_back(primary_key.data_types[i]);
         }
@@ -1130,28 +1137,32 @@ MarkRanges MergeTreeDataSelectExecutor::markRangesFromPKRange(
             {
                 for (size_t i = 0; i < used_key_size; ++i)
                 {
+                    auto & left = reverse_flags[i] ? index_right[i] : index_left[i];
+                    auto & right = reverse_flags[i] ? index_left[i] : index_right[i];
                     if ((*index_columns)[i].column)
-                        create_field_ref(range.begin, i, index_left[i]);
+                        create_field_ref(range.begin, i, left);
                     else
-                        index_left[i] = NEGATIVE_INFINITY;
+                        left = NEGATIVE_INFINITY;
 
-                    index_right[i] = POSITIVE_INFINITY;
+                    right = POSITIVE_INFINITY;
                 }
             }
             else
             {
                 for (size_t i = 0; i < used_key_size; ++i)
                 {
+                    auto & left = reverse_flags[i] ? index_right[i] : index_left[i];
+                    auto & right = reverse_flags[i] ? index_left[i] : index_right[i];
                     if ((*index_columns)[i].column)
                     {
-                        create_field_ref(range.begin, i, index_left[i]);
-                        create_field_ref(range.end, i, index_right[i]);
+                        create_field_ref(range.begin, i, left);
+                        create_field_ref(range.end, i, right);
                     }
                     else
                     {
                         /// If the PK column was not loaded in memory - exclude it from the analysis.
-                        index_left[i] = NEGATIVE_INFINITY;
-                        index_right[i] = POSITIVE_INFINITY;
+                        left = NEGATIVE_INFINITY;
+                        right = POSITIVE_INFINITY;
                     }
                 }
             }

src/Storages/MergeTree/MergeTreeDataWriter.cpp

@@ -517,12 +517,18 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPartImpl(
         data.getSortingKeyAndSkipIndicesExpression(metadata_snapshot, indices)->execute(block);
 
     Names sort_columns = metadata_snapshot->getSortingKeyColumns();
+    std::vector<bool> reverse_flags = metadata_snapshot->getSortingKeyReverseFlags();
     SortDescription sort_description;
     size_t sort_columns_size = sort_columns.size();
     sort_description.reserve(sort_columns_size);
 
     for (size_t i = 0; i < sort_columns_size; ++i)
-        sort_description.emplace_back(sort_columns[i], 1, 1);
+    {
+        if (!reverse_flags.empty() && reverse_flags[i])
+            sort_description.emplace_back(sort_columns[i], -1, 1);
+        else
+            sort_description.emplace_back(sort_columns[i], 1, 1);
+    }
 
     ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterBlocks);
 
@@ -795,12 +801,18 @@ MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl(
         data.getSortingKeyAndSkipIndicesExpression(metadata_snapshot, {})->execute(block);
 
     Names sort_columns = metadata_snapshot->getSortingKeyColumns();
+    std::vector<bool> reverse_flags = metadata_snapshot->getSortingKeyReverseFlags();
     SortDescription sort_description;
     size_t sort_columns_size = sort_columns.size();
     sort_description.reserve(sort_columns_size);
 
     for (size_t i = 0; i < sort_columns_size; ++i)
-        sort_description.emplace_back(sort_columns[i], 1, 1);
+    {
+        if (!reverse_flags.empty() && reverse_flags[i])
+            sort_description.emplace_back(sort_columns[i], -1, 1);
+        else
+            sort_description.emplace_back(sort_columns[i], 1, 1);
+    }
 
     ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterBlocks);
 

src/Storages/MergeTree/MergeTreeSettings.cpp

@@ -201,6 +201,7 @@ namespace ErrorCodes
     DECLARE(String, storage_policy, "default", "Name of storage disk policy", 0) \
     DECLARE(String, disk, "", "Name of storage disk. Can be specified instead of storage policy.", 0) \
     DECLARE(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \
+    DECLARE(Bool, allow_experimental_reverse_key, false, "Allow descending sorting key in MergeTree tables (experimental feature).", 0) \
     DECLARE(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm.", 0) \
     DECLARE(Bool, assign_part_uuids, false, "Generate UUIDs for parts. Before enabling check that all replicas support new format.", 0) \
     DECLARE(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \

src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp

@@ -77,10 +77,9 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
     /// So rules in zookeeper metadata is following:
     /// - When we have only ORDER BY, than store it in "primary key:" row of /metadata
     /// - When we have both, than store PRIMARY KEY in "primary key:" row and ORDER BY in "sorting key:" row of /metadata
-
-    primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().expression_list_ast);
     if (metadata_snapshot->isPrimaryKeyDefined())
     {
+        primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().expression_list_ast);
         /// We don't use preparsed AST `sorting_key.expression_list_ast` because
         /// it contain version column for VersionedCollapsingMergeTree, which
         /// is not stored in ZooKeeper for compatibility reasons. So the best
@@ -89,6 +88,10 @@ ReplicatedMergeTreeTableMetadata::ReplicatedMergeTreeTableMetadata(const MergeTr
         /// used.
         sorting_key = formattedASTNormalized(extractKeyExpressionList(metadata_snapshot->getSortingKey().definition_ast));
     }
+    else
+    {
+        primary_key = formattedASTNormalized(metadata_snapshot->getPrimaryKey().getOriginalExpressionList());
+    }
 
     data_format_version = data.format_version;
 
@@ -301,7 +304,7 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat
 
     /// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes
     ///    in formatAST code.
-    String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context).expression_list_ast);
+    String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context, true).getOriginalExpressionList());
     if (primary_key != parsed_zk_primary_key)
         throw Exception(ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in primary key. "
             "Stored in ZooKeeper: {}, parsed from ZooKeeper: {}, local: {}",
@@ -313,7 +316,7 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat
                         "Stored in ZooKeeper: {}, local: {}", DB::toString(from_zk.data_format_version.toUnderType()),
                         DB::toString(data_format_version.toUnderType()));
 
-    String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context).expression_list_ast);
+    String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context, false).expression_list_ast);
     if (partition_key != parsed_zk_partition_key)
         throw Exception(ErrorCodes::METADATA_MISMATCH,
                         "Existing table metadata in ZooKeeper differs in partition key expression. "
@@ -326,7 +329,7 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl
 
     checkImmutableFieldsEquals(from_zk, columns, context);
 
-    String parsed_zk_sampling_expression = formattedAST(KeyDescription::parse(from_zk.sampling_expression, columns, context).definition_ast);
+    String parsed_zk_sampling_expression = formattedAST(KeyDescription::parse(from_zk.sampling_expression, columns, context, false).definition_ast);
     if (sampling_expression != parsed_zk_sampling_expression)
     {
         throw Exception(ErrorCodes::METADATA_MISMATCH, "Existing table metadata in ZooKeeper differs in sample expression. "
@@ -334,7 +337,7 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl
             from_zk.sampling_expression, parsed_zk_sampling_expression, sampling_expression);
     }
 
-    String parsed_zk_sorting_key = formattedAST(extractKeyExpressionList(KeyDescription::parse(from_zk.sorting_key, columns, context).definition_ast));
+    String parsed_zk_sorting_key = formattedAST(extractKeyExpressionList(KeyDescription::parse(from_zk.sorting_key, columns, context, true).definition_ast));
     if (sorting_key != parsed_zk_sorting_key)
     {
         throw Exception(ErrorCodes::METADATA_MISMATCH,
@@ -343,7 +346,7 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl
                         from_zk.sorting_key, parsed_zk_sorting_key, sorting_key);
     }
 
-    auto parsed_primary_key = KeyDescription::parse(primary_key, columns, context);
+    auto parsed_primary_key = KeyDescription::parse(primary_key, columns, context, true);
     String parsed_zk_ttl_table = formattedAST(TTLTableDescription::parse(from_zk.ttl_table, columns, context, parsed_primary_key).definition_ast);
     if (ttl_table != parsed_zk_ttl_table)
     {

src/Storages/StorageInMemoryMetadata.cpp

@@ -513,6 +513,13 @@ Names StorageInMemoryMetadata::getSortingKeyColumns() const
     return {};
 }
 
+std::vector<bool> StorageInMemoryMetadata::getSortingKeyReverseFlags() const
+{
+    if (hasSortingKey())
+        return sorting_key.reverse_flags;
+    return {};
+}
+
 const KeyDescription & StorageInMemoryMetadata::getSamplingKey() const
 {
     return sampling_key;

src/Storages/StorageInMemoryMetadata.h

@@ -222,6 +222,9 @@ struct StorageInMemoryMetadata
     /// Returns columns names in sorting key specified by user in ORDER BY
     /// expression. For example: 'a', 'x * y', 'toStartOfMonth(date)', etc.
     Names getSortingKeyColumns() const;
+    /// Returns reverse indicators of columns in sorting key specified by user in ORDER BY
+    /// expression. For example: ('a' DESC, 'x * y', 'toStartOfMonth(date)' DESC) -> {1, 0, 1}.
+    std::vector<bool> getSortingKeyReverseFlags() const;
 
     /// Returns column names that need to be read for FINAL to work.
     Names getColumnsRequiredForFinal() const { return getColumnsRequiredForSortingKey(); }

src/Storages/TTLDescription.cpp

@@ -275,7 +275,7 @@ TTLDescription TTLDescription::getTTLFromAST(
             for (size_t i = 0; i < ttl_element->group_by_key.size(); ++i)
             {
                 if (ttl_element->group_by_key[i]->getColumnName() != pk_columns[i])
-                    throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "TTL Expression GROUP BY key should be a prefix of primary key");
+                    throw Exception(ErrorCodes::BAD_TTL_EXPRESSION, "TTL Expression GROUP BY key should be a prefix of primary key {} {}", ttl_element->group_by_key[i]->getColumnName(), pk_columns[i]);
 
                 used_primary_key_columns_set.insert(pk_columns[i]);
             }

tests/ci/official_docker.py

@@ -299,8 +299,6 @@ class TagAttrs:
 
     # Only one latest can exist
     latest: ClickHouseVersion
-    # Only one can be a major one (the most fresh per a year)
-    majors: Dict[int, ClickHouseVersion]
     # Only one lts version can exist
     lts: Optional[ClickHouseVersion]
 
@@ -345,14 +343,6 @@ def ldf_tags(version: ClickHouseVersion, distro: str, tag_attrs: TagAttrs) -> st
             tags.append("lts")
         tags.append(f"lts-{distro}")
 
-    # If the tag `22`, `23`, `24` etc. should be included in the tags
-    with_major = tag_attrs.majors.get(version.major) in (None, version)
-    if with_major:
-        tag_attrs.majors[version.major] = version
-        if without_distro:
-            tags.append(f"{version.major}")
-        tags.append(f"{version.major}-{distro}")
-
     # Add all normal tags
     for tag in (
         f"{version.major}.{version.minor}",
@@ -384,7 +374,7 @@ def generate_ldf(args: argparse.Namespace) -> None:
         args.directory / git_runner(f"git -C {args.directory} rev-parse --show-cdup")
     ).absolute()
     lines = ldf_header(git, directory)
-    tag_attrs = TagAttrs(versions[-1], {}, None)
+    tag_attrs = TagAttrs(versions[-1], None)
 
     # We iterate from the most recent to the oldest version
     for version in reversed(versions):

tests/integration/test_server_reload/test.py

@@ -378,7 +378,7 @@ def test_reload_via_client(cluster, zk):
                         configure_from_zk(zk)
                     break
                 except QueryRuntimeException:
-                    logging.exception("The new socket is not binded yet")
+                    logging.exception("The new socket is not bound yet")
                     time.sleep(0.1)
 
     if exception:

tests/queries/0_stateless/03250_ephemeral_comment.sql

@@ -0,0 +1,11 @@
+drop table if exists test;
+CREATE TABLE test (
+    `start_s`  UInt32 EPHEMERAL COMMENT 'start UNIX time' ,
+    `start_us` UInt16 EPHEMERAL COMMENT 'start microseconds',
+    `finish_s`  UInt32 EPHEMERAL COMMENT 'finish UNIX time',
+    `finish_us` UInt16 EPHEMERAL COMMENT 'finish microseconds',
+    `captured` DateTime MATERIALIZED fromUnixTimestamp(start_s),
+    `duration` Decimal32(6) MATERIALIZED finish_s - start_s + (finish_us - start_us)/1000000
+)
+ENGINE Null;
+drop table if exists test;

tests/queries/0_stateless/03257_reverse_sorting_key.reference

@@ -0,0 +1,75 @@
+3
+8
+Sorting (Sorting for ORDER BY)
+Prefix sort description: __table1.i DESC
+Result sort description: __table1.i DESC
+(Expression)
+ExpressionTransform
+  (Limit)
+  Limit
+    (Sorting)
+      (Expression)
+      ExpressionTransform
+        (ReadFromMergeTree)
+        MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1
+99
+98
+97
+96
+95
+Sorting (Sorting for ORDER BY)
+Prefix sort description: __table1.i ASC
+Result sort description: __table1.i ASC
+(Expression)
+ExpressionTransform
+  (Limit)
+  Limit
+    (Sorting)
+      (Expression)
+      ExpressionTransform
+        (ReadFromMergeTree)
+        ReverseTransform
+          MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InReverseOrder) 0 → 1
+0
+1
+2
+3
+4
+3	1003
+6
+Sorting (Sorting for ORDER BY)
+Prefix sort description: __table1.i ASC, __table1.j DESC
+Result sort description: __table1.i ASC, __table1.j DESC
+(Expression)
+ExpressionTransform
+  (Limit)
+  Limit
+    (Sorting)
+      (Expression)
+      ExpressionTransform
+        (ReadFromMergeTree)
+        MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1
+0	1090
+0	1080
+0	1070
+0	1060
+0	1050
+Sorting (Sorting for ORDER BY)
+Prefix sort description: __table1.i ASC
+Result sort description: __table1.i ASC, __table1.j ASC
+(Expression)
+ExpressionTransform
+  (Limit)
+  Limit
+    (Sorting)
+    FinishSortingTransform
+      PartialSortingTransform
+        (Expression)
+        ExpressionTransform
+          (ReadFromMergeTree)
+          MergeTreeSelect(pool: ReadPoolInOrder, algorithm: InOrder) 0 → 1
+0	1000
+0	1010
+0	1020
+0	1030
+0	1040

tests/queries/0_stateless/03257_reverse_sorting_key.sql

@@ -0,0 +1,52 @@
+-- Tags: no-random-merge-tree-settings
+
+set optimize_read_in_order = 1;
+set read_in_order_two_level_merge_threshold=100;
+
+drop table if exists x1;
+
+drop table if exists x2;
+
+create table x1 (i Nullable(int)) engine MergeTree order by i desc settings allow_nullable_key = 1, index_granularity = 2, allow_experimental_reverse_key = 1;
+
+insert into x1 select * from numbers(100);
+
+optimize table x1 final;
+
+select * from x1 where i = 3;
+
+select count() from x1 where i between 3 and 10;
+
+select trimLeft(explain) from (explain actions=1 select * from x1 order by i desc limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1;
+explain pipeline select * from x1 order by i desc limit 5 settings max_threads=1;
+
+select * from x1 order by i desc limit 5;
+
+select trimLeft(explain) from (explain actions=1 select * from x1 order by i limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1;
+explain pipeline select * from x1 order by i limit 5 settings max_threads=1;
+
+select * from x1 order by i limit 5;
+
+create table x2 (i Nullable(int), j Nullable(int)) engine MergeTree order by (i, j desc) settings allow_nullable_key = 1, index_granularity = 2, allow_experimental_reverse_key = 1;
+
+insert into x2 select number % 10, number + 1000 from numbers(100);
+
+optimize table x2 final;
+
+select * from x2 where j = 1003;
+
+select count() from x2 where i between 3 and 10 and j between 1003 and 1008;
+
+select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j desc limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1;
+explain pipeline select * from x2 order by i, j desc limit 5 settings max_threads=1;
+
+select * from x2 order by i, j desc limit 5;
+
+select trimLeft(explain) from (explain actions=1 select * from x2 order by i, j limit 5) where explain ilike '%sort%' settings max_threads=1, enable_analyzer=1;
+explain pipeline select * from x2 order by i, j limit 5 settings max_threads=1;
+
+select * from x2 order by i, j limit 5;
+
+drop table x1;
+
+drop table x2;

tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.reference

@@ -0,0 +1,2 @@
+metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 2\nmode: 0\nsign column: \nprimary key: i DESC\ndata format version: 1\npartition key: \ngranularity bytes: 10000\nmerge parameters format version: 2\n
+metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 2\nmode: 0\nsign column: \nprimary key: i, j DESC\ndata format version: 1\npartition key: \ngranularity bytes: 10000\nmerge parameters format version: 2\n

tests/queries/0_stateless/03257_reverse_sorting_key_zookeeper.sql

@@ -0,0 +1,16 @@
+-- Tags: zookeeper, no-random-merge-tree-settings, no-replicated-database
+
+drop table if exists x1;
+drop table if exists x2;
+
+create table x1 (i Nullable(int)) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x1', 'r1') order by i desc settings allow_nullable_key = 1, index_granularity = 2, index_granularity_bytes = 10000, allow_experimental_reverse_key = 1;
+
+create table x2 (i Nullable(int), j Nullable(int)) engine ReplicatedMergeTree('/clickhouse/tables/{database}/x2', 'r1') order by (i, j desc) settings allow_nullable_key = 1, index_granularity = 2, index_granularity_bytes = 10000, allow_experimental_reverse_key = 1;
+
+set allow_unrestricted_reads_from_keeper = 'true';
+
+select value from system.zookeeper where path = '/clickhouse/tables/' || currentDatabase() || '/x1' and name = 'metadata';
+select value from system.zookeeper where path = '/clickhouse/tables/' || currentDatabase() || '/x2' and name = 'metadata';
+
+drop table x1;
+drop table x2;