From 7b96a7b798813900504e0517cbb7ed5f93f8c3c1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 14 Oct 2022 16:20:56 +0200
Subject: [PATCH] Move Tokens::operator and optimize the common path

---
 src/Parsers/TokenIterator.cpp | 19 +++++++++++++++++++
 src/Parsers/TokenIterator.h   | 17 +----------------
 2 files changed, 20 insertions(+), 16 deletions(-)

diff --git a/src/Parsers/TokenIterator.cpp b/src/Parsers/TokenIterator.cpp
index 08877e0b2fe..ecff8b0d482 100644
--- a/src/Parsers/TokenIterator.cpp
+++ b/src/Parsers/TokenIterator.cpp
@@ -4,6 +4,25 @@
 namespace DB
 {
 
+const Token & Tokens::operator[](size_t index)
+{
+    if (likely(index < data.size()))
+        return data[index];
+
+    while (data.empty() || !data.back().isEnd())
+    {
+        Token token = lexer.nextToken();
+
+        if (token.isSignificant())
+        {
+            data.emplace_back(std::move(token));
+            if (index < data.size())
+                return data[index];
+        }
+    }
+    return data.back();
+}
+
 UnmatchedParentheses checkUnmatchedParentheses(TokenIterator begin)
 {
     /// We have just two kind of parentheses: () and [].
diff --git a/src/Parsers/TokenIterator.h b/src/Parsers/TokenIterator.h
index b84bec57817..62570f49358 100644
--- a/src/Parsers/TokenIterator.h
+++ b/src/Parsers/TokenIterator.h
@@ -25,22 +25,7 @@ private:
 public:
     Tokens(const char * begin, const char * end, size_t max_query_size = 0) : lexer(begin, end, max_query_size) {}
 
-    const Token & operator[] (size_t index)
-    {
-        while (true)
-        {
-            if (index < data.size())
-                return data[index];
-
-            if (!data.empty() && data.back().isEnd())
-                return data.back();
-
-            Token token = lexer.nextToken();
-
-            if (token.isSignificant())
-                data.emplace_back(token);
-        }
-    }
+    const Token & operator[](size_t index);
 
     const Token & max()
     {