From 4465a0627f154fa92efe1cad5566b72ad73c7cb0 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Sat, 10 Apr 2021 17:58:29 +0300 Subject: [PATCH] better --- docker/test/fasttest/run.sh | 2 +- src/Storages/StorageS3.cpp | 8 +++--- src/Storages/StorageS3Distributed.cpp | 4 ++- src/Storages/StorageS3Distributed.h | 2 ++ tests/integration/test_s3_distributed/test.py | 8 +++++- tests/queries/0_stateless/trace_raw | 27 +++++++++++++++++++ 6 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/trace_raw diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index 9500a7854ba..a482ab35d78 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -365,7 +365,7 @@ function run_tests 01622_defaults_for_url_engine # JSON functions - 01666_blnsi + 01666_blns # Depends on AWS 01801_s3_distributed diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index a5283448e38..51e8a3225e4 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -67,8 +67,7 @@ public: matcher = std::make_unique(makeRegexpPatternFromGlobs(globbed_uri.key)); - /// Don't forget about iterator invalidation - buffer_iter = buffer.begin(); + fillInternalBufferAssumeLocked(); } std::optional next() @@ -386,7 +385,10 @@ Pipe StorageS3::read( client_auth.uri.bucket, iterator_wrapper)); } - return Pipe::unitePipes(std::move(pipes)); + auto pipe = Pipe::unitePipes(std::move(pipes)); + + narrowPipe(pipe, num_streams); + return pipe; } BlockOutputStreamPtr StorageS3::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context) diff --git a/src/Storages/StorageS3Distributed.cpp b/src/Storages/StorageS3Distributed.cpp index 5bd4a4ffd81..29784bd73df 100644 --- a/src/Storages/StorageS3Distributed.cpp +++ b/src/Storages/StorageS3Distributed.cpp @@ -1,7 +1,8 @@ #include "Storages/StorageS3Distributed.h" +#if !defined(ARCADIA_BUILD) #include -#include "Processors/Sources/SourceWithProgress.h" +#endif #if USE_AWS_S3 @@ -30,6 +31,7 @@ #include #include #include +#include "Processors/Sources/SourceWithProgress.h" #include #include #include diff --git a/src/Storages/StorageS3Distributed.h b/src/Storages/StorageS3Distributed.h index 3ec2b5ec813..34d9612f5d4 100644 --- a/src/Storages/StorageS3Distributed.h +++ b/src/Storages/StorageS3Distributed.h @@ -1,6 +1,8 @@ #pragma once +#if !defined(ARCADIA_BUILD) #include +#endif #if USE_AWS_S3 diff --git a/tests/integration/test_s3_distributed/test.py b/tests/integration/test_s3_distributed/test.py index a7826dc5582..7ad7f201ce6 100644 --- a/tests/integration/test_s3_distributed/test.py +++ b/tests/integration/test_s3_distributed/test.py @@ -95,7 +95,13 @@ def test_union_all(started_cluster): SELECT * from s3Distributed( 'cluster_simple', 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', - 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon)""") + 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon) + UNION ALL + SELECT * from s3Distributed( + 'cluster_simple', + 'http://minio1:9001/root/data/{clickhouse,database}/*', 'minio', 'minio123', 'CSV', + 'name String, value UInt32, polygon Array(Array(Tuple(Float64, Float64)))') ORDER BY (name, value, polygon) + """) # print(s3_distibuted) assert TSV(pure_s3) == TSV(s3_distibuted) diff --git a/tests/queries/0_stateless/trace_raw b/tests/queries/0_stateless/trace_raw new file mode 100644 index 00000000000..032acbef6bf --- /dev/null +++ b/tests/queries/0_stateless/trace_raw @@ -0,0 +1,27 @@ +[Thread debugging using libthread_db enabled] +Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1". +0x00007fb11d0bedd7 in __GI___select (nfds=0, readfds=0x0, writefds=0x0, exceptfds=0x0, timeout=0x7fff4e474300) at ../sysdeps/unix/sysv/linux/select.c:41 + +Thread 1 (Thread 0x7fb11d5ad740 (LWP 38888)): +#0 0x00007fb11d0bedd7 in __GI___select (nfds=0, readfds=0x0, writefds=0x0, exceptfds=0x0, timeout=0x7fff4e474300) at ../sysdeps/unix/sysv/linux/select.c:41 +#1 0x00000000005bbd3a in ?? () +#2 0x000000000050a2cc in ?? () +#3 0x000000000050bf44 in _PyEval_EvalFrameDefault () +#4 0x00000000005096c8 in ?? () +#5 0x000000000050a3fd in ?? () +#6 0x000000000050bf44 in _PyEval_EvalFrameDefault () +#7 0x0000000000507cd4 in ?? () +#8 0x0000000000509a00 in ?? () +#9 0x000000000050a3fd in ?? () +#10 0x000000000050bf44 in _PyEval_EvalFrameDefault () +#11 0x0000000000507cd4 in ?? () +#12 0x0000000000509a00 in ?? () +#13 0x000000000050a3fd in ?? () +#14 0x000000000050bf44 in _PyEval_EvalFrameDefault () +#15 0x0000000000507cd4 in ?? () +#16 0x000000000050ae13 in PyEval_EvalCode () +#17 0x0000000000635262 in ?? () +#18 0x0000000000635317 in PyRun_FileExFlags () +#19 0x0000000000638acf in PyRun_SimpleFileExFlags () +#20 0x0000000000639671 in Py_Main () +#21 0x00000000004b0e40 in main ()