Merge 92cc90a711 into 71b651d4a8

docker/test/stateless/Dockerfile

@@ -78,6 +78,10 @@ RUN arch=${TARGETARCH:-amd64} \
     && curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o ./mc \
     && chmod +x ./mc ./minio
 
+RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
+    && tar -xvf hadoop-3.3.1.tar.gz \
+    && rm -rf hadoop-3.3.1.tar.gz
+
 ENV MINIO_ROOT_USER="clickhouse"
 ENV MINIO_ROOT_PASSWORD="clickhouse"
 ENV EXPORT_S3_STORAGE_POLICIES=1

docs/en/engines/database-engines/atomic.md

@@ -6,7 +6,7 @@ sidebar_position: 10
 
 # Atomic 
 
-It supports non-blocking [DROP TABLE](#drop-detach-table) and [RENAME TABLE](#rename-table) queries and atomic [EXCHANGE TABLES](#exchange-tables) queries. `Atomic` database engine is used by default. Note that on ClickHouse Cloud, the `Replicated` database engine is used by default.
+It supports non-blocking [DROP TABLE](#drop-detach-table) and [RENAME TABLE](#rename-table) queries and atomic [EXCHANGE TABLES](#exchange-tables) queries. `Atomic` database engine is used by default.
 
 ## Creating a Database {#creating-a-database}
 

src/Common/ProfileEvents.cpp

@@ -962,15 +962,6 @@ Counters::Counters(VariableContext level_, Counters * parent_)
     counters = counters_holder.get();
 }
 
-Counters::Counters(Counters && src) noexcept
-    : counters(std::exchange(src.counters, nullptr))
-    , counters_holder(std::move(src.counters_holder))
-    , parent(src.parent.exchange(nullptr))
-    , trace_profile_events(src.trace_profile_events)
-    , level(src.level)
-{
-}
-
 void Counters::resetCounters()
 {
     if (counters)
@@ -982,7 +973,7 @@ void Counters::resetCounters()
 
 void Counters::reset()
 {
-    setParent(nullptr);
+    parent = nullptr;
     resetCounters();
 }
 

src/Common/ProfileEvents.h

@@ -60,7 +60,7 @@ namespace ProfileEvents
         Counter * counters = nullptr;
         std::unique_ptr<Counter[]> counters_holder;
         /// Used to propagate increments
-        std::atomic<Counters *> parent = {};
+        Counters * parent = nullptr;
         bool trace_profile_events = false;
 
     public:
@@ -74,8 +74,6 @@ namespace ProfileEvents
         explicit Counters(Counter * allocated_counters) noexcept
             : counters(allocated_counters), parent(nullptr), level(VariableContext::Global) {}
 
-        Counters(Counters && src) noexcept;
-
         Counter & operator[] (Event event)
         {
             return counters[event];
@@ -116,13 +114,13 @@ namespace ProfileEvents
         /// Get parent (thread unsafe)
         Counters * getParent()
         {
-            return parent.load(std::memory_order_relaxed);
+            return parent;
         }
 
         /// Set parent (thread unsafe)
         void setParent(Counters * parent_)
         {
-            parent.store(parent_, std::memory_order_relaxed);
+            parent = parent_;
         }
 
         void setTraceProfileEvents(bool value)

src/Core/Settings.cpp

@@ -5521,8 +5521,6 @@ The default value is `CURRENT_USER`.
     DECLARE(UInt64, cache_warmer_threads, 4, R"(
 Only available in ClickHouse Cloud. Number of background threads for speculatively downloading new data parts into file cache, when cache_populated_by_fetch is enabled. Zero to disable.
 )", 0) \
-    DECLARE(Bool, use_async_executor_for_materialized_views, false, R"(
-Use async and potentially multithreaded execution of materialized view query, can speedup views processing during INSERT, but also consume more memory.)", 0) \
     DECLARE(Int64, ignore_cold_parts_seconds, 0, R"(
 Only available in ClickHouse Cloud. Exclude new data parts from SELECT queries until they're either pre-warmed (see cache_populated_by_fetch) or this many seconds old. Only for Replicated-/SharedMergeTree.
 )", 0) \

src/Core/SettingsChangesHistory.cpp

@@ -60,7 +60,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
 {
     {"24.12",
         {
-            {"use_async_executor_for_materialized_views", false, false, "New setting."},
         }
     },
     {"24.11",
@@ -118,7 +117,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"min_free_disk_ratio_to_perform_insert", 0.0, 0.0, "New setting."},
             {"enable_named_columns_in_function_tuple", false, false, "Disabled pending usability improvements"},
             {"cloud_mode_database_engine", 1, 1, "A setting for ClickHouse Cloud"},
-            {"allow_experimental_shared_set_join", 0, 0, "A setting for ClickHouse Cloud"},
+            {"allow_experimental_shared_set_join", 1, 1, "A setting for ClickHouse Cloud"},
             {"read_through_distributed_cache", 0, 0, "A setting for ClickHouse Cloud"},
             {"write_through_distributed_cache", 0, 0, "A setting for ClickHouse Cloud"},
             {"distributed_cache_throw_on_error", 0, 0, "A setting for ClickHouse Cloud"},

src/Parsers/Access/ParserCreateUserQuery.cpp

@@ -549,8 +549,6 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     auto names = typeid_cast<std::shared_ptr<ASTUserNamesWithHost>>(names_ast);
     auto names_ref = names->names;
 
-    auto pos_after_parsing_names = pos;
-
     std::optional<String> new_name;
     std::optional<AllowedClientHosts> hosts;
     std::optional<AllowedClientHosts> add_hosts;
@@ -676,13 +674,6 @@ bool ParserCreateUserQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     else if (alter)
         names->concatParts();
 
-    bool alter_query_with_no_changes = alter && pos_after_parsing_names == pos;
-
-    if (alter_query_with_no_changes)
-    {
-        return false;
-    }
-
     auto query = std::make_shared<ASTCreateUserQuery>();
     node = query;
 

src/Processors/Chunk.cpp

@@ -62,8 +62,8 @@ void Chunk::checkNumRowsIsConsistent()
     {
         auto & column = columns[i];
         if (column->size() != num_rows)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid number of rows in Chunk {} column {} at position {}: expected {}, got {}",
-                dumpStructure(), column->getName(), i, num_rows, column->size());
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid number of rows in Chunk column {}: expected {}, got {}",
+                            column->getName() + " position " + toString(i), toString(num_rows), toString(column->size()));
     }
 }
 
@@ -100,8 +100,8 @@ void Chunk::addColumn(ColumnPtr column)
     if (empty())
         num_rows = column->size();
     else if (column->size() != num_rows)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid number of rows in Chunk {} column {}: expected {}, got {}",
-            dumpStructure(), column->getName(), num_rows, column->size());
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid number of rows in Chunk column {}, got {}",
+                        column->getName()+ ": expected " + toString(num_rows), toString(column->size()));
 
     columns.emplace_back(std::move(column));
 }

src/Processors/Transforms/FillingTransform.cpp

@@ -205,20 +205,6 @@ static bool tryConvertFields(FillColumnDescription & descr, const DataTypePtr &
     return true;
 }
 
-SortDescription duduplicateSortDescription(const SortDescription & sort_description)
-{
-    SortDescription result;
-    std::unordered_set<std::string> unique_columns;
-    for (const auto & desc : sort_description)
-    {
-        const auto & [_, inserted] = unique_columns.insert(desc.column_name);
-        if (!inserted)
-            continue;
-        result.push_back(desc);
-    }
-    return result;
-}
-
 FillingTransform::FillingTransform(
     const Block & header_,
     const SortDescription & sort_description_,
@@ -226,7 +212,7 @@ FillingTransform::FillingTransform(
     InterpolateDescriptionPtr interpolate_description_,
     const bool use_with_fill_by_sorting_prefix_)
     : ISimpleTransform(header_, transformHeader(header_, fill_description_), true)
-    , sort_description(duduplicateSortDescription(sort_description_))
+    , sort_description(sort_description_)
     , fill_description(fill_description_)
     , interpolate_description(interpolate_description_)
     , filling_row(fill_description_)

src/Processors/Transforms/buildPushingToViewsChain.cpp

@@ -11,7 +11,6 @@
 #include <Processors/Transforms/PlanSquashingTransform.h>
 #include <Processors/Transforms/SquashingTransform.h>
 #include <Processors/Transforms/ExpressionTransform.h>
-#include <Processors/Executors/PullingAsyncPipelineExecutor.h>
 #include <Processors/Executors/PullingPipelineExecutor.h>
 #include <Storages/LiveView/StorageLiveView.h>
 #include <Storages/WindowView/StorageWindowView.h>
@@ -64,7 +63,6 @@ namespace Setting
     extern const SettingsUInt64 min_insert_block_size_rows_for_materialized_views;
     extern const SettingsBool parallel_view_processing;
     extern const SettingsBool use_concurrency_control;
-    extern const SettingsBool use_async_executor_for_materialized_views;
 }
 
 namespace ErrorCodes
@@ -131,7 +129,6 @@ private:
 };
 
 /// For source chunk, execute view query over it.
-template <typename Executor>
 class ExecutingInnerQueryFromViewTransform final : public ExceptionKeepingTransform
 {
 public:
@@ -151,7 +148,7 @@ private:
     struct State
     {
         QueryPipeline pipeline;
-        Executor executor;
+        PullingPipelineExecutor executor;
 
         explicit State(QueryPipeline pipeline_)
             : pipeline(std::move(pipeline_))
@@ -431,31 +428,17 @@ std::optional<Chain> generateViewChain(
         out.addSource(std::make_shared<DeduplicationToken::CheckTokenTransform>("Right after Inner query", out.getInputHeader()));
 #endif
 
-        if (context->getSettingsRef()[Setting::use_async_executor_for_materialized_views])
-        {
-             auto executing_inner_query = std::make_shared<ExecutingInnerQueryFromViewTransform<PullingAsyncPipelineExecutor>>(
-                 storage_header, views_data->views.back(), views_data, disable_deduplication_for_children);
-             executing_inner_query->setRuntimeData(view_thread_status, view_counter_ms);
+        auto executing_inner_query = std::make_shared<ExecutingInnerQueryFromViewTransform>(
+            storage_header, views_data->views.back(), views_data, disable_deduplication_for_children);
+        executing_inner_query->setRuntimeData(view_thread_status, view_counter_ms);
 
-             out.addSource(std::move(executing_inner_query));
-        }
-        else
-        {
-
-             auto executing_inner_query = std::make_shared<ExecutingInnerQueryFromViewTransform<PullingPipelineExecutor>>(
-                 storage_header, views_data->views.back(), views_data, disable_deduplication_for_children);
-             executing_inner_query->setRuntimeData(view_thread_status, view_counter_ms);
-
-             out.addSource(std::move(executing_inner_query));
-
-        }
+        out.addSource(std::move(executing_inner_query));
 
 #ifdef ABORT_ON_LOGICAL_ERROR
         out.addSource(std::make_shared<DeduplicationToken::CheckTokenTransform>("Right before Inner query", out.getInputHeader()));
 #endif
     }
 
-
     return out;
 }
 
@@ -783,8 +766,7 @@ IProcessor::Status CopyingDataToViewsTransform::prepare()
 }
 
 
-template <typename Executor>
-ExecutingInnerQueryFromViewTransform<Executor>::ExecutingInnerQueryFromViewTransform(
+ExecutingInnerQueryFromViewTransform::ExecutingInnerQueryFromViewTransform(
     const Block & header,
     ViewRuntimeData & view_,
     std::shared_ptr<ViewsData> views_data_,
@@ -796,16 +778,14 @@ ExecutingInnerQueryFromViewTransform<Executor>::ExecutingInnerQueryFromViewTrans
 {
 }
 
-template <typename Executor>
-void ExecutingInnerQueryFromViewTransform<Executor>::onConsume(Chunk chunk)
+void ExecutingInnerQueryFromViewTransform::onConsume(Chunk chunk)
 {
     auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns());
     state.emplace(process(std::move(block), view, *views_data, std::move(chunk.getChunkInfos()), disable_deduplication_for_children));
 }
 
 
-template <typename Executor>
-ExecutingInnerQueryFromViewTransform<Executor>::GenerateResult ExecutingInnerQueryFromViewTransform<Executor>::onGenerate()
+ExecutingInnerQueryFromViewTransform::GenerateResult ExecutingInnerQueryFromViewTransform::onGenerate()
 {
     GenerateResult res;
     if (!state.has_value())

tests/docker_scripts/setup_hdfs_minicluster.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# shellcheck disable=SC2024
+
+set -e -x -a -u
+
+ls -lha
+
+cd /hadoop-3.3.1
+
+export JAVA_HOME=/usr
+mkdir -p target/test/data
+chown clickhouse ./target/test/data
+sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 >> /test_output/hdfs_minicluster.log 2>&1 &
+
+while ! nc -z localhost 12222; do
+  sleep 1
+done
+
+lsof -i :12222
+
+sleep 5

tests/docker_scripts/stateless_runner.sh

@@ -57,6 +57,8 @@ source /repo/tests/docker_scripts/utils.lib
 
 /repo/tests/docker_scripts/setup_minio.sh stateless
 
+/repo/tests/docker_scripts/setup_hdfs_minicluster.sh
+
 config_logs_export_cluster /etc/clickhouse-server/config.d/system_logs_export.yaml
 
 if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; then

tests/queries/0_stateless/01030_storage_hdfs_syntax.sql

@@ -0,0 +1,10 @@
+-- Tags: no-fasttest, use-hdfs
+
+drop table if exists test_table_hdfs_syntax
+;
+create table test_table_hdfs_syntax (id UInt32) ENGINE = HDFS('')
+; -- { serverError BAD_ARGUMENTS }
+create table test_table_hdfs_syntax (id UInt32) ENGINE = HDFS('','','', '')
+; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+drop table if exists test_table_hdfs_syntax
+;

tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh

@@ -16,5 +16,5 @@ for _ in {1..1000}; do
     if [[ $elapsed -gt 30 ]]; then
         break
     fi
-done 2>&1 | grep -o 'Query memory limit exceeded' | head -n1
+done 2>&1 | grep -o -P 'Query memory limit exceeded' | sed -r -e 's/(.*):([a-Z ]*)([mM]emory limit exceeded)(.*)/\2\3/' | uniq
 echo 'Ok'

tests/queries/0_stateless/02113_hdfs_assert.reference

@@ -0,0 +1 @@
+OK

tests/queries/0_stateless/02113_hdfs_assert.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, use-hdfs
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+TCP_PORT=$($CLICKHOUSE_CLIENT -q "SELECT tcpPort()")
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://localhost:$TCP_PORT/data.csv', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "HDFS_ERROR" && echo 'OK' || echo 'FAIL';

tests/queries/0_stateless/02114_hdfs_bad_url.reference

@@ -0,0 +1,17 @@
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK
+OK

tests/queries/0_stateless/02114_hdfs_bad_url.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, use-hdfs
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd/', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd/', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('://abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('://abcd/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd:9000', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('//abcd:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('://abcd:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('abcd/', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://abcd', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs1:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://hdfs1/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "HDFS_ERROR" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('http://hdfs1:9000/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "BAD_ARGUMENTS" && echo 'OK' || echo 'FAIL';
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hdfs('hdfs://hdfs1@nameservice/abcd/data', 'CSV', 'x UInt32')" 2>&1 | grep -F -q "HDFS_ERROR" && echo 'OK' || echo 'FAIL';

tests/queries/0_stateless/02244_hdfs_cluster.reference

@@ -0,0 +1,48 @@
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+1	2	3
+4	5	6
+7	8	9
+c1	Nullable(Int64)					
+c2	Nullable(Int64)					
+c3	Nullable(Int64)					
+c1	Nullable(Int64)					
+c2	Nullable(Int64)					
+c3	Nullable(Int64)					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					
+c1	Nullable(Int64)					
+c2	Nullable(Int64)					
+c3	Nullable(Int64)					
+c1	Nullable(Int64)					
+c2	Nullable(Int64)					
+c3	Nullable(Int64)					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					
+c1	UInt32					
+c2	UInt32					
+c3	UInt32					

tests/queries/0_stateless/02244_hdfs_cluster.sql

@@ -0,0 +1,26 @@
+-- Tags: no-fasttest, no-parallel
+-- Tag no-fasttest: Depends on Java
+
+insert into table function hdfs('hdfs://localhost:12222/test_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;
+insert into table function hdfs('hdfs://localhost:12222/test_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;
+insert into table function hdfs('hdfs://localhost:12222/test_3.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 7, 8, 9 settings hdfs_truncate_on_insert=1;
+
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
+select * from hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
+
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv') order by c1, c2, c3;
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV') order by c1, c2, c3;
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32') order by c1, c2, c3;
+select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto') order by c1, c2, c3;
+
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv');
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
+desc hdfs('hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');
+
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv');
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV');
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32');
+desc hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_{1,2,3}.tsv', 'TSV', 'c1 UInt32, c2 UInt32, c3 UInt32', 'auto');

tests/queries/0_stateless/02368_cancel_write_into_hdfs.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, no-asan, no-tsan, no-msan, no-ubsan, no-debug
+# FIXME https://github.com/ClickHouse/ClickHouse/issues/47207
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+for i in $(seq 1 10);
+do
+    $CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_insert=1, output_format_parallel_formatting=1" 2> /dev/null &
+done
+
+sleep 2
+
+$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null 2>&1
+
+for i in $(seq 1 10);
+do
+    $CLICKHOUSE_CLIENT --query_id="02368_$i" -q "insert into function hdfs('hdfs://localhost:12222/02368_data_$i.jsonl') select range(number % 1000) from numbers(100000) settings hdfs_truncate_on_insert=1, output_format_parallel_formatting=0" 2> /dev/null &
+done
+
+sleep 2
+
+$CLICKHOUSE_CLIENT -q "kill query where startsWith(query_id, '02368_') sync" > /dev/null 2>&1

tests/queries/0_stateless/02458_empty_hdfs_url.sql

@@ -0,0 +1,5 @@
+-- Tags: no-fasttest
+SELECT * FROM hdfsCluster('test_shard_localhost', '', 'TSV'); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM hdfsCluster('test_shard_localhost', ' ', 'TSV'); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM hdfsCluster('test_shard_localhost', '/', 'TSV'); -- { serverError BAD_ARGUMENTS }
+SELECT * FROM hdfsCluster('test_shard_localhost', 'http/', 'TSV'); -- { serverError BAD_ARGUMENTS }

tests/queries/0_stateless/02458_hdfs_cluster_schema_inference.reference

@@ -0,0 +1,10 @@
+c1	Nullable(Int64)					
+c2	Nullable(Int64)					
+c3	Nullable(Int64)					
+c1	Nullable(Int64)					
+c2	Nullable(Int64)					
+c3	Nullable(Int64)					
+1	2	3
+4	5	6
+1	2	3
+4	5	6

tests/queries/0_stateless/02458_hdfs_cluster_schema_inference.sql

@@ -0,0 +1,11 @@
+-- Tags: no-fasttest, no-parallel
+-- Tag no-fasttest: Depends on Java
+
+insert into table function hdfs('hdfs://localhost:12222/test_02458_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;
+insert into table function hdfs('hdfs://localhost:12222/test_02458_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;
+
+desc hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv');
+desc hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv', 'TSV');
+
+select * from hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv') order by c1, c2, c3;
+select * from hdfsCluster('test_cluster_one_shard_three_replicas_localhost', 'hdfs://localhost:12222/test_02458_{1,2}.tsv', 'TSV') order by c1, c2, c3;

tests/queries/0_stateless/02536_hdfs_cluster_use_structure_from_table.reference

@@ -0,0 +1 @@
+(1,2)

tests/queries/0_stateless/02536_hdfs_cluster_use_structure_from_table.sql

@@ -0,0 +1,11 @@
+-- Tags: no-fasttest, no-parallel
+-- Tag no-fasttest: Depends on Java
+
+insert into table function hdfs('hdfs://localhost:12222/test_02536.jsonl', 'TSV') select '{"x" : {"a" : 1, "b" : 2}}' settings hdfs_truncate_on_insert=1;
+set input_format_json_try_infer_named_tuples_from_objects=0;
+drop table if exists test;
+create table test (x Tuple(a UInt32, b UInt32)) engine=Memory();
+insert into test select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_02536.jsonl') settings use_structure_from_insertion_table_in_table_functions=0; -- {serverError ILLEGAL_COLUMN}
+insert into test select * from hdfsCluster('test_cluster_two_shards_localhost', 'hdfs://localhost:12222/test_02536.jsonl') settings use_structure_from_insertion_table_in_table_functions=1;
+select * from test;
+drop table test;

tests/queries/0_stateless/02725_database_hdfs.reference

@@ -0,0 +1,11 @@
+Test 1: select from hdfs database
+1	2	3
+test_hdfs_1
+1	2	3
+test_hdfs_2
+Test 2: check exceptions
+BAD_ARGUMENTS
+OK
+OK
+OK
+OK

tests/queries/0_stateless/02725_database_hdfs.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, use-hdfs, no-parallel
+
+CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# Prepare data
+${CLICKHOUSE_CLIENT} -q "insert into table function hdfs('hdfs://localhost:12222/test_02725_1.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 1, 2, 3 settings hdfs_truncate_on_insert=1;"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "Insert failed!"
+    exit 1
+fi
+${CLICKHOUSE_CLIENT} -q "insert into table function hdfs('hdfs://localhost:12222/test_02725_2.tsv', 'TSV', 'column1 UInt32, column2 UInt32, column3 UInt32') select 4, 5, 6 settings hdfs_truncate_on_insert=1;"
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "Insert failed!"
+    exit 1
+fi
+
+#################
+echo "Test 1: select from hdfs database"
+
+# Database without specific host
+${CLICKHOUSE_CLIENT} --multiline -q """
+DROP DATABASE IF EXISTS test_hdfs_1;
+CREATE DATABASE test_hdfs_1 ENGINE = HDFS;
+USE test_hdfs_1;
+SELECT * FROM \"hdfs://localhost:12222/test_02725_1.tsv\"
+"""
+${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test_hdfs_1
+
+# Database with host
+${CLICKHOUSE_CLIENT} --multiline -q """
+DROP DATABASE IF EXISTS test_hdfs_2;
+CREATE DATABASE test_hdfs_2 ENGINE = HDFS('hdfs://localhost:12222');
+USE test_hdfs_2;
+SELECT * FROM \"test_02725_1.tsv\"
+"""
+${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test_hdfs_2
+
+#################
+echo "Test 2: check exceptions"
+
+${CLICKHOUSE_CLIENT} --multiline -q """
+DROP DATABASE IF EXISTS test_hdfs_3;
+CREATE DATABASE test_hdfs_3 ENGINE = HDFS('abacaba');
+""" 2>&1 | tr '\n' ' ' | grep -oF "BAD_ARGUMENTS"
+
+${CLICKHOUSE_CLIENT} --multiline -q """
+DROP DATABASE IF EXISTS test_hdfs_4;
+CREATE DATABASE test_hdfs_4 ENGINE = HDFS;
+USE test_hdfs_4;
+SELECT * FROM \"abacaba/file.tsv\"
+""" 2>&1 | tr '\n' ' ' | grep -oF "CANNOT_EXTRACT_TABLE_STRUCTURE"
+
+${CLICKHOUSE_CLIENT} -q "SELECT * FROM test_hdfs_4.\`http://localhost:11111/test/a.tsv\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "BAD_ARGUMENTS" > /dev/null && echo "OK" || echo 'FAIL' ||:
+${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_hdfs_4.\`hdfs://localhost:12222/file.myext\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "The data format cannot be detected" > /dev/null && echo "OK" || echo 'FAIL' ||:
+${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_hdfs_4.\`hdfs://localhost:12222/test_02725_3.tsv\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "The table structure cannot be extracted" > /dev/null && echo "OK" || echo 'FAIL' ||:
+${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_hdfs_4.\`hdfs://localhost:12222\`" 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "BAD_ARGUMENTS" > /dev/null && echo "OK" || echo 'FAIL' ||:
+
+
+# Cleanup
+${CLICKHOUSE_CLIENT} --multiline -q """
+DROP DATABASE IF EXISTS test_hdfs_1;
+DROP DATABASE IF EXISTS test_hdfs_2;
+DROP DATABASE IF EXISTS test_hdfs_3;
+DROP DATABASE IF EXISTS test_hdfs_4;
+"""

tests/queries/0_stateless/02845_table_function_hdfs_filter_by_virtual_columns.reference

@@ -0,0 +1,2 @@
+1
+1

tests/queries/0_stateless/02845_table_function_hdfs_filter_by_virtual_columns.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, use-hdfs
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data1.tsv') select 1 settings hdfs_truncate_on_insert=1;"
+$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data2.tsv') select 2 settings hdfs_truncate_on_insert=1;"
+$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data3.tsv') select 3 settings hdfs_truncate_on_insert=1;"
+
+
+$CLICKHOUSE_CLIENT --print-profile-events -q "select * from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') where _file like '%data1%' format Null" 2>&1 | grep -F -c "EngineFileLikeReadFiles: 1"
+
+$CLICKHOUSE_CLIENT --print-profile-events -q "select * from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') where _path like '%data1%' format Null" 2>&1 | grep -F -c "EngineFileLikeReadFiles: 1"

tests/queries/0_stateless/02923_hdfs_engine_size_virtual_column.reference

@@ -0,0 +1,6 @@
+2
+3
+4
+2
+3
+4

tests/queries/0_stateless/02923_hdfs_engine_size_virtual_column.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, use-hdfs
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data1.tsv') select 1 settings hdfs_truncate_on_insert=1;"
+$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data2.tsv') select 11 settings hdfs_truncate_on_insert=1;"
+$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data3.tsv') select 111 settings hdfs_truncate_on_insert=1;"
+
+
+$CLICKHOUSE_CLIENT -q "select _size from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') order by _size"
+$CLICKHOUSE_CLIENT -q "select _size from hdfs('hdfs://localhost:12222/$CLICKHOUSE_TEST_UNIQUE_NAME.data*.tsv', auto, 'x UInt64') order by _size"
+

tests/queries/0_stateless/03006_parallel_replicas_prewhere.sql

@@ -9,7 +9,7 @@ CREATE TABLE url_na_log
 ENGINE = MergeTree
 PRIMARY KEY SiteId
 ORDER BY (SiteId, DateVisit)
-SETTINGS index_granularity_bytes = 1000000, index_granularity = 1000, min_bytes_for_wide_part = 0;
+SETTINGS index_granularity = 1000, min_bytes_for_wide_part = 0;
 
 CREATE ROW POLICY url_na_log_policy0 ON url_na_log FOR SELECT USING (DateVisit < '2022-08-11') OR (DateVisit > '2022-08-19') TO default;
 

tests/queries/0_stateless/03243_parquet_prefetch.reference

@@ -0,0 +1,2 @@
+99999
+99999

tests/queries/0_stateless/03243_parquet_prefetch.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, no-parallel, use-hdfs
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -q "insert into table function file('test_03243.parquet', 'Parquet') select number as i from numbers(100000) settings output_format_parquet_row_group_size=10000,engine_file_truncate_on_insert=1"
+$CLICKHOUSE_CLIENT -q "select max(i) from file('test_03243.parquet', 'Parquet') settings max_threads = 1;"
+$CLICKHOUSE_CLIENT -q "insert into table function hdfs('hdfs://localhost:12222/test_03243.parquet', 'Parquet') select number as i from numbers(100000) settings output_format_parquet_row_group_size=10000,hdfs_truncate_on_insert=1;"
+$CLICKHOUSE_CLIENT -q "select max(i) from hdfs('hdfs://localhost:12222/test_03243.parquet', 'Parquet') settings max_threads = 1;"

tests/queries/0_stateless/03254_test_alter_user_no_changes.sql

@@ -1,5 +0,0 @@
--- Tags: no-parallel
-
-create user u_03254_alter_user;
-alter user u_03254_alter_user; -- { clientError SYNTAX_ERROR }
-drop user u_03254_alter_user;

tests/queries/0_stateless/03274_with_fill_dup_sort_bug.reference

@@ -1,8 +0,0 @@
-1	2
-1	3
-1	4
-1	5
-1	6
-1	7
-1	8
-1	9

tests/queries/0_stateless/03274_with_fill_dup_sort_bug.sql

@@ -1,7 +0,0 @@
-SELECT
-    1 AS a,
-    2 AS b
-ORDER BY
-    a ASC,
-    1 ASC,
-    b ASC WITH FILL TO 10;

tests/queries/0_stateless/03275_matview_with_union.sql

@@ -2,8 +2,6 @@ DROP TABLE IF EXISTS src;
 DROP TABLE IF EXISTS dst;
 DROP TABLE IF EXISTS matview;
 
-SET use_async_executor_for_materialized_views=1;
-
 CREATE TABLE src (
     event_time DateTime,
     key UInt64,