diff --git a/.gitmodules b/.gitmodules index 1d9d4d25baf..4df7798e1e7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -193,7 +193,7 @@ url = https://github.com/danlark1/miniselect [submodule "contrib/rocksdb"] path = contrib/rocksdb - url = https://github.com/ClickHouse-Extras/rocksdb.git + url = https://github.com/ClickHouse-Extras/rocksdb.git [submodule "contrib/xz"] path = contrib/xz url = https://github.com/xz-mirror/xz @@ -228,3 +228,9 @@ [submodule "contrib/libpqxx"] path = contrib/libpqxx url = https://github.com/ClickHouse-Extras/libpqxx.git +[submodule "contrib/sqlite-amalgamation"] + path = contrib/sqlite-amalgamation + url = https://github.com/azadkuh/sqlite-amalgamation +[submodule "contrib/s2geometry"] + path = contrib/s2geometry + url = https://github.com/ClickHouse-Extras/s2geometry.git diff --git a/CMakeLists.txt b/CMakeLists.txt index d23e5f540d3..875a6d1ab61 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -536,10 +536,12 @@ include (cmake/find/rapidjson.cmake) include (cmake/find/fastops.cmake) include (cmake/find/odbc.cmake) include (cmake/find/nanodbc.cmake) +include (cmake/find/sqlite.cmake) include (cmake/find/rocksdb.cmake) include (cmake/find/libpqxx.cmake) include (cmake/find/nuraft.cmake) include (cmake/find/yaml-cpp.cmake) +include (cmake/find/s2geometry.cmake) if(NOT USE_INTERNAL_PARQUET_LIBRARY) set (ENABLE_ORC OFF CACHE INTERNAL "") diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 49cf30d2556..18072566d04 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54453) +SET(VERSION_REVISION 54454) SET(VERSION_MAJOR 21) -SET(VERSION_MINOR 8) +SET(VERSION_MINOR 9) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH fb895056568e26200629c7d19626e92d2dedc70d) -SET(VERSION_DESCRIBE v21.8.1.1-prestable) -SET(VERSION_STRING 21.8.1.1) +SET(VERSION_GITHASH f48c5af90c2ad51955d1ee3b6b05d006b03e4238) +SET(VERSION_DESCRIBE v21.9.1.1-prestable) +SET(VERSION_STRING 21.9.1.1) # end of autochange diff --git a/cmake/find/s2geometry.cmake b/cmake/find/s2geometry.cmake new file mode 100644 index 00000000000..2364c6ba193 --- /dev/null +++ b/cmake/find/s2geometry.cmake @@ -0,0 +1,24 @@ + +option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES}) + +if (ENABLE_S2_GEOMETRY) + if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/s2geometry") + message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init --recursive") + set (ENABLE_S2_GEOMETRY 0) + set (USE_S2_GEOMETRY 0) + else() + if (OPENSSL_FOUND) + set (S2_GEOMETRY_LIBRARY s2) + set (S2_GEOMETRY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src/s2) + set (USE_S2_GEOMETRY 1) + else() + message (WARNING "S2 uses OpenSSL, but the latter is absent.") + endif() + endif() + + if (NOT USE_S2_GEOMETRY) + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable S2 geometry library") + endif() +endif() + +message (STATUS "Using s2geometry=${USE_S2_GEOMETRY} : ${S2_GEOMETRY_INCLUDE_DIR}") diff --git a/cmake/find/sqlite.cmake b/cmake/find/sqlite.cmake new file mode 100644 index 00000000000..cfa33fdebbb --- /dev/null +++ b/cmake/find/sqlite.cmake @@ -0,0 +1,16 @@ +option(ENABLE_SQLITE "Enable sqlite" ${ENABLE_LIBRARIES}) + +if (NOT ENABLE_SQLITE) + return() +endif() + +if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sqlite-amalgamation/sqlite3.c") + message (WARNING "submodule contrib/sqlite3-amalgamation is missing. to fix try run: \n git submodule update --init --recursive") + message (${RECONFIGURE_MESSAGE_LEVEL} "Can't find internal sqlite library") + set (USE_SQLITE 0) + return() +endif() + +set (USE_SQLITE 1) +set (SQLITE_LIBRARY sqlite) +message (STATUS "Using sqlite=${USE_SQLITE}") diff --git a/cmake/find/stats.cmake b/cmake/find/stats.cmake index 339e8524598..dea108ed920 100644 --- a/cmake/find/stats.cmake +++ b/cmake/find/stats.cmake @@ -1,4 +1,4 @@ -option(ENABLE_STATS "Enalbe StatsLib library" ${ENABLE_LIBRARIES}) +option(ENABLE_STATS "Enable StatsLib library" ${ENABLE_LIBRARIES}) if (ENABLE_STATS) if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats") diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 164692fb893..2b6629d0817 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -1,3 +1,4 @@ +# Third-party libraries may have substandard code. # Put all targets defined here and in added subfolders under "contrib/" folder in GUI-based IDEs by default. # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they will @@ -10,10 +11,8 @@ else () endif () unset (_current_dir_name) -# Third-party libraries may have substandard code. -# Also remove a possible source of nondeterminism. -set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -D__DATE__= -D__TIME__= -D__TIMESTAMP__=") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -D__DATE__= -D__TIME__= -D__TIMESTAMP__=") +set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w") +set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w") if (WITH_COVERAGE) set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE}) @@ -329,3 +328,10 @@ endif() add_subdirectory(fast_float) +if (USE_SQLITE) + add_subdirectory(sqlite-cmake) +endif() + +if (USE_S2_GEOMETRY) + add_subdirectory(s2geometry-cmake) +endif() diff --git a/contrib/h3 b/contrib/h3 index e209086ae1b..c7f46cfd71f 160000 --- a/contrib/h3 +++ b/contrib/h3 @@ -1 +1 @@ -Subproject commit e209086ae1b5477307f545a0f6111780edc59940 +Subproject commit c7f46cfd71fb60e2fefc90e28abe81657deff735 diff --git a/contrib/h3-cmake/CMakeLists.txt b/contrib/h3-cmake/CMakeLists.txt index 6b184a175b0..f4c70dc476f 100644 --- a/contrib/h3-cmake/CMakeLists.txt +++ b/contrib/h3-cmake/CMakeLists.txt @@ -3,21 +3,22 @@ set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib") set(SRCS "${H3_SOURCE_DIR}/lib/algos.c" -"${H3_SOURCE_DIR}/lib/baseCells.c" -"${H3_SOURCE_DIR}/lib/bbox.c" "${H3_SOURCE_DIR}/lib/coordijk.c" -"${H3_SOURCE_DIR}/lib/faceijk.c" -"${H3_SOURCE_DIR}/lib/geoCoord.c" -"${H3_SOURCE_DIR}/lib/h3Index.c" -"${H3_SOURCE_DIR}/lib/h3UniEdge.c" -"${H3_SOURCE_DIR}/lib/linkedGeo.c" -"${H3_SOURCE_DIR}/lib/localij.c" -"${H3_SOURCE_DIR}/lib/mathExtensions.c" +"${H3_SOURCE_DIR}/lib/bbox.c" "${H3_SOURCE_DIR}/lib/polygon.c" +"${H3_SOURCE_DIR}/lib/h3Index.c" "${H3_SOURCE_DIR}/lib/vec2d.c" "${H3_SOURCE_DIR}/lib/vec3d.c" "${H3_SOURCE_DIR}/lib/vertex.c" +"${H3_SOURCE_DIR}/lib/linkedGeo.c" +"${H3_SOURCE_DIR}/lib/localij.c" +"${H3_SOURCE_DIR}/lib/latLng.c" +"${H3_SOURCE_DIR}/lib/directedEdge.c" +"${H3_SOURCE_DIR}/lib/mathExtensions.c" +"${H3_SOURCE_DIR}/lib/iterators.c" "${H3_SOURCE_DIR}/lib/vertexGraph.c" +"${H3_SOURCE_DIR}/lib/faceijk.c" +"${H3_SOURCE_DIR}/lib/baseCells.c" ) configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h") diff --git a/contrib/poco b/contrib/poco index 59945069080..7351c4691b5 160000 --- a/contrib/poco +++ b/contrib/poco @@ -1 +1 @@ -Subproject commit 5994506908028612869fee627d68d8212dfe7c1e +Subproject commit 7351c4691b5d401f59e3959adfc5b4fa263b32da diff --git a/contrib/s2geometry b/contrib/s2geometry new file mode 160000 index 00000000000..20ea540d81f --- /dev/null +++ b/contrib/s2geometry @@ -0,0 +1 @@ +Subproject commit 20ea540d81f4575a3fc0aea585aac611bcd03ede diff --git a/contrib/s2geometry-cmake/CMakeLists.txt b/contrib/s2geometry-cmake/CMakeLists.txt new file mode 100644 index 00000000000..f54562652a6 --- /dev/null +++ b/contrib/s2geometry-cmake/CMakeLists.txt @@ -0,0 +1,126 @@ +set(S2_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src") + +set(S2_SRCS + "${S2_SOURCE_DIR}/s2/base/stringprintf.cc" + "${S2_SOURCE_DIR}/s2/base/strtoint.cc" + "${S2_SOURCE_DIR}/s2/encoded_s2cell_id_vector.cc" + "${S2_SOURCE_DIR}/s2/encoded_s2point_vector.cc" + "${S2_SOURCE_DIR}/s2/encoded_s2shape_index.cc" + "${S2_SOURCE_DIR}/s2/encoded_string_vector.cc" + "${S2_SOURCE_DIR}/s2/id_set_lexicon.cc" + "${S2_SOURCE_DIR}/s2/mutable_s2shape_index.cc" + "${S2_SOURCE_DIR}/s2/r2rect.cc" + "${S2_SOURCE_DIR}/s2/s1angle.cc" + "${S2_SOURCE_DIR}/s2/s1chord_angle.cc" + "${S2_SOURCE_DIR}/s2/s1interval.cc" + "${S2_SOURCE_DIR}/s2/s2boolean_operation.cc" + "${S2_SOURCE_DIR}/s2/s2builder.cc" + "${S2_SOURCE_DIR}/s2/s2builder_graph.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_closed_set_normalizer.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_find_polygon_degeneracies.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_lax_polygon_layer.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_s2point_vector_layer.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_s2polygon_layer.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_s2polyline_layer.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_s2polyline_vector_layer.cc" + "${S2_SOURCE_DIR}/s2/s2builderutil_snap_functions.cc" + "${S2_SOURCE_DIR}/s2/s2cap.cc" + "${S2_SOURCE_DIR}/s2/s2cell.cc" + "${S2_SOURCE_DIR}/s2/s2cell_id.cc" + "${S2_SOURCE_DIR}/s2/s2cell_index.cc" + "${S2_SOURCE_DIR}/s2/s2cell_union.cc" + "${S2_SOURCE_DIR}/s2/s2centroids.cc" + "${S2_SOURCE_DIR}/s2/s2closest_cell_query.cc" + "${S2_SOURCE_DIR}/s2/s2closest_edge_query.cc" + "${S2_SOURCE_DIR}/s2/s2closest_point_query.cc" + "${S2_SOURCE_DIR}/s2/s2contains_vertex_query.cc" + "${S2_SOURCE_DIR}/s2/s2convex_hull_query.cc" + "${S2_SOURCE_DIR}/s2/s2coords.cc" + "${S2_SOURCE_DIR}/s2/s2crossing_edge_query.cc" + "${S2_SOURCE_DIR}/s2/s2debug.cc" + "${S2_SOURCE_DIR}/s2/s2earth.cc" + "${S2_SOURCE_DIR}/s2/s2edge_clipping.cc" + "${S2_SOURCE_DIR}/s2/s2edge_crosser.cc" + "${S2_SOURCE_DIR}/s2/s2edge_crossings.cc" + "${S2_SOURCE_DIR}/s2/s2edge_distances.cc" + "${S2_SOURCE_DIR}/s2/s2edge_tessellator.cc" + "${S2_SOURCE_DIR}/s2/s2error.cc" + "${S2_SOURCE_DIR}/s2/s2furthest_edge_query.cc" + "${S2_SOURCE_DIR}/s2/s2latlng.cc" + "${S2_SOURCE_DIR}/s2/s2latlng_rect.cc" + "${S2_SOURCE_DIR}/s2/s2latlng_rect_bounder.cc" + "${S2_SOURCE_DIR}/s2/s2lax_loop_shape.cc" + "${S2_SOURCE_DIR}/s2/s2lax_polygon_shape.cc" + "${S2_SOURCE_DIR}/s2/s2lax_polyline_shape.cc" + "${S2_SOURCE_DIR}/s2/s2loop.cc" + "${S2_SOURCE_DIR}/s2/s2loop_measures.cc" + "${S2_SOURCE_DIR}/s2/s2measures.cc" + "${S2_SOURCE_DIR}/s2/s2metrics.cc" + "${S2_SOURCE_DIR}/s2/s2max_distance_targets.cc" + "${S2_SOURCE_DIR}/s2/s2min_distance_targets.cc" + "${S2_SOURCE_DIR}/s2/s2padded_cell.cc" + "${S2_SOURCE_DIR}/s2/s2point_compression.cc" + "${S2_SOURCE_DIR}/s2/s2point_region.cc" + "${S2_SOURCE_DIR}/s2/s2pointutil.cc" + "${S2_SOURCE_DIR}/s2/s2polygon.cc" + "${S2_SOURCE_DIR}/s2/s2polyline.cc" + "${S2_SOURCE_DIR}/s2/s2polyline_alignment.cc" + "${S2_SOURCE_DIR}/s2/s2polyline_measures.cc" + "${S2_SOURCE_DIR}/s2/s2polyline_simplifier.cc" + "${S2_SOURCE_DIR}/s2/s2predicates.cc" + "${S2_SOURCE_DIR}/s2/s2projections.cc" + "${S2_SOURCE_DIR}/s2/s2r2rect.cc" + "${S2_SOURCE_DIR}/s2/s2region.cc" + "${S2_SOURCE_DIR}/s2/s2region_term_indexer.cc" + "${S2_SOURCE_DIR}/s2/s2region_coverer.cc" + "${S2_SOURCE_DIR}/s2/s2region_intersection.cc" + "${S2_SOURCE_DIR}/s2/s2region_union.cc" + "${S2_SOURCE_DIR}/s2/s2shape_index.cc" + "${S2_SOURCE_DIR}/s2/s2shape_index_buffered_region.cc" + "${S2_SOURCE_DIR}/s2/s2shape_index_measures.cc" + "${S2_SOURCE_DIR}/s2/s2shape_measures.cc" + "${S2_SOURCE_DIR}/s2/s2shapeutil_build_polygon_boundaries.cc" + "${S2_SOURCE_DIR}/s2/s2shapeutil_coding.cc" + "${S2_SOURCE_DIR}/s2/s2shapeutil_contains_brute_force.cc" + "${S2_SOURCE_DIR}/s2/s2shapeutil_edge_iterator.cc" + "${S2_SOURCE_DIR}/s2/s2shapeutil_get_reference_point.cc" + "${S2_SOURCE_DIR}/s2/s2shapeutil_range_iterator.cc" + "${S2_SOURCE_DIR}/s2/s2shapeutil_visit_crossing_edge_pairs.cc" + "${S2_SOURCE_DIR}/s2/s2text_format.cc" + "${S2_SOURCE_DIR}/s2/s2wedge_relations.cc" + "${S2_SOURCE_DIR}/s2/strings/ostringstream.cc" + "${S2_SOURCE_DIR}/s2/strings/serialize.cc" + # ClickHouse doesn't use strings from abseil. + # So, there is no duplicate symbols. + "${S2_SOURCE_DIR}/s2/third_party/absl/base/dynamic_annotations.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/raw_logging.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/throw_delegate.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/numeric/int128.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/ascii.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/match.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/numbers.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_cat.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_split.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/string_view.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/strip.cc" + "${S2_SOURCE_DIR}/s2/third_party/absl/strings/internal/memutil.cc" + "${S2_SOURCE_DIR}/s2/util/bits/bit-interleave.cc" + "${S2_SOURCE_DIR}/s2/util/bits/bits.cc" + "${S2_SOURCE_DIR}/s2/util/coding/coder.cc" + "${S2_SOURCE_DIR}/s2/util/coding/varint.cc" + "${S2_SOURCE_DIR}/s2/util/math/exactfloat/exactfloat.cc" + "${S2_SOURCE_DIR}/s2/util/math/mathutil.cc" + "${S2_SOURCE_DIR}/s2/util/units/length-units.cc" +) + +add_library(s2 ${S2_SRCS}) + +if (OPENSSL_FOUND) + target_link_libraries(s2 PRIVATE ${OPENSSL_LIBRARIES}) +endif() + +target_include_directories(s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/") + +if(M_LIBRARY) + target_link_libraries(s2 PRIVATE ${M_LIBRARY}) +endif() diff --git a/contrib/sqlite-amalgamation b/contrib/sqlite-amalgamation new file mode 160000 index 00000000000..9818baa5d02 --- /dev/null +++ b/contrib/sqlite-amalgamation @@ -0,0 +1 @@ +Subproject commit 9818baa5d027ffb26d57f810dc4c597d4946781c diff --git a/contrib/sqlite-cmake/CMakeLists.txt b/contrib/sqlite-cmake/CMakeLists.txt new file mode 100644 index 00000000000..495cb63798d --- /dev/null +++ b/contrib/sqlite-cmake/CMakeLists.txt @@ -0,0 +1,6 @@ +set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/sqlite-amalgamation") + +set(SRCS ${LIBRARY_DIR}/sqlite3.c) + +add_library(sqlite ${SRCS}) +target_include_directories(sqlite SYSTEM PUBLIC "${LIBRARY_DIR}") diff --git a/debian/changelog b/debian/changelog index 36c29fce1d0..38f740ae062 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (21.8.1.1) unstable; urgency=low +clickhouse (21.9.1.1) unstable; urgency=low * Modified source code - -- clickhouse-release Mon, 28 Jun 2021 00:50:15 +0300 + -- clickhouse-release Sat, 10 Jul 2021 08:22:49 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 19cadccb926..f17fa8ade16 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.8.1.* +ARG version=21.9.1.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 65d90bf52ce..5da9e703f4d 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.8.1.* +ARG version=21.9.1.* ARG gosu_ver=1.10 # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 687393025f0..5768753cd7c 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.8.1.* +ARG version=21.9.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh index bba20f64e5a..3e8bf306a83 100755 --- a/docker/test/fasttest/run.sh +++ b/docker/test/fasttest/run.sh @@ -378,6 +378,16 @@ function run_tests # needs pv 01923_network_receive_time_metric_insert + + 01889_sqlite_read_write + + # needs s2 + 01849_geoToS2 + 01851_s2_to_geo + 01852_s2_get_neighbours + 01853_s2_cells_intersect + 01854_s2_cap_contains + 01854_s2_cap_union ) time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \ diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index e15697da029..344c1b9a698 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -32,7 +32,7 @@ RUN rm -rf \ RUN apt-get clean # Install MySQL ODBC driver -RUN curl 'https://cdn.mysql.com//Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit.tar.gz' --output 'mysql-connector.tar.gz' && tar -xzf mysql-connector.tar.gz && cd mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit/lib && mv * /usr/local/lib && ln -s /usr/local/lib/libmyodbc8a.so /usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so +RUN curl 'https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit.tar.gz' --location --output 'mysql-connector.tar.gz' && tar -xzf mysql-connector.tar.gz && cd mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit/lib && mv * /usr/local/lib && ln -s /usr/local/lib/libmyodbc8a.so /usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so # Unfortunately this is required for a single test for conversion data from zookeeper to clickhouse-keeper. # ZooKeeper is not started by default, but consumes some space in containers. @@ -49,4 +49,3 @@ RUN mkdir /zookeeper && chmod -R 777 /zookeeper ENV TZ=Europe/Moscow RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - diff --git a/docker/test/stateless/Dockerfile b/docker/test/stateless/Dockerfile index 658ae1f27ba..17c89232e17 100644 --- a/docker/test/stateless/Dockerfile +++ b/docker/test/stateless/Dockerfile @@ -29,7 +29,8 @@ RUN apt-get update -y \ unixodbc \ wget \ mysql-client=5.7* \ - postgresql-client + postgresql-client \ + sqlite3 RUN pip3 install numpy scipy pandas diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 428fdb9fdb7..0dffb373905 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -118,7 +118,7 @@ clickhouse-client --query "SELECT 'Server successfully started', 'OK'" >> /test_ [ -f /var/log/clickhouse-server/stderr.log ] || echo -e "Stderr log does not exist\tFAIL" # Print Fatal log messages to stdout -zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log +zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log* # Grep logs for sanitizer asserts, crashes and other critical errors @@ -131,22 +131,22 @@ zgrep -Fav "ASan doesn't fully support makecontext/swapcontext functions" > /dev rm -f /test_output/tmp # OOM -zgrep -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ +zgrep -Fa " Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ && echo -e 'OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv # Logical errors -zgrep -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ +zgrep -Fa "Code: 49, e.displayText() = DB::Exception:" /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ && echo -e 'Logical error thrown (see clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'No logical errors\tOK' >> /test_output/test_results.tsv # Crash -zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ +zgrep -Fa "########################################" /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ && echo -e 'Killed by signal (in clickhouse-server.log)\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'Not crashed\tOK' >> /test_output/test_results.tsv # It also checks for crash without stacktrace (printed by watchdog) -zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log > /dev/null \ +zgrep -Fa " " /var/log/clickhouse-server/clickhouse-server.log* > /dev/null \ && echo -e 'Fatal message in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \ || echo -e 'No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv diff --git a/docs/en/development/adding_test_queries.md b/docs/en/development/adding_test_queries.md index 95dfd076a12..547d8b0fa37 100644 --- a/docs/en/development/adding_test_queries.md +++ b/docs/en/development/adding_test_queries.md @@ -105,11 +105,11 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te 5) ensure everything is correct, if the test output is incorrect (due to some bug for example), adjust the reference file using text editor. -#### How to create good test +#### How to create a good test -- test should be +- A test should be - minimal - create only tables related to tested functionality, remove unrelated columns and parts of query - - fast - should not take longer than few seconds (better subseconds) + - fast - should not take longer than a few seconds (better subseconds) - correct - fails then feature is not working - deterministic - isolated / stateless @@ -126,6 +126,16 @@ clickhouse-client -nmT < tests/queries/0_stateless/01521_dummy_test.sql | tee te - use other SQL files in the `0_stateless` folder as an example - ensure the feature / feature combination you want to test is not yet covered with existing tests +#### Test naming rules + +It's important to name tests correctly, so one could turn some tests subset off in clickhouse-test invocation. + +| Tester flag| What should be in test name | When flag should be added | +|---|---|---|---| +| `--[no-]zookeeper`| "zookeeper" or "replica" | Test uses tables from ReplicatedMergeTree family | +| `--[no-]shard` | "shard" or "distributed" or "global"| Test using connections to 127.0.0.2 or similar | +| `--[no-]long` | "long" or "deadlock" or "race" | Test runs longer than 60 seconds | + #### Commit / push / create PR. 1) commit & push your changes diff --git a/docs/en/development/build.md b/docs/en/development/build.md index 8ef12221e8d..97b477d55a5 100644 --- a/docs/en/development/build.md +++ b/docs/en/development/build.md @@ -134,10 +134,10 @@ $ ./release ## Faster builds for development -Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. Two common ways to improve linking time are to use `lld` linker, and use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into several shared libraries. To enable these tweaks, pass the following flags to `cmake`: +Normally all tools of the ClickHouse bundle, such as `clickhouse-server`, `clickhouse-client` etc., are linked into a single static executable, `clickhouse`. This executable must be re-linked on every change, which might be slow. One common way to improve build time is to use the 'split' build configuration, which builds a separate binary for every tool, and further splits the code into several shared libraries. To enable this tweak, pass the following flags to `cmake`: ``` --DCMAKE_C_FLAGS="--ld-path=lld" -DCMAKE_CXX_FLAGS="--ld-path=lld" -DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 +-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1 -DCLICKHOUSE_SPLIT_BINARY=1 ``` ## You Don’t Have to Build ClickHouse {#you-dont-have-to-build-clickhouse} diff --git a/docs/en/development/contrib.md b/docs/en/development/contrib.md index ac39c496c72..a65ddb40af0 100644 --- a/docs/en/development/contrib.md +++ b/docs/en/development/contrib.md @@ -79,6 +79,7 @@ SELECT library_name, license_type, license_path FROM system.licenses ORDER BY li | re2 | BSD 3-clause | /contrib/re2/LICENSE | | replxx | BSD 3-clause | /contrib/replxx/LICENSE.md | | rocksdb | BSD 3-clause | /contrib/rocksdb/LICENSE.leveldb | +| s2geometry | Apache | /contrib/s2geometry/LICENSE | | sentry-native | MIT | /contrib/sentry-native/LICENSE | | simdjson | Apache | /contrib/simdjson/LICENSE | | snappy | Public Domain | /contrib/snappy/COPYING | diff --git a/docs/en/engines/table-engines/integrations/materialized-postgresql.md b/docs/en/engines/table-engines/integrations/materialized-postgresql.md index 70f61c5b550..142639507d6 100644 --- a/docs/en/engines/table-engines/integrations/materialized-postgresql.md +++ b/docs/en/engines/table-engines/integrations/materialized-postgresql.md @@ -1,6 +1,6 @@ --- toc_priority: 12 -toc_title: MateriaziePostgreSQL +toc_title: MaterializedPostgreSQL --- # MaterializedPostgreSQL {#materialize-postgresql} diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index eb288721231..f32b4b26c8b 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -1246,12 +1246,14 @@ The table below shows supported data types and how they match ClickHouse [data t | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | -| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `STRING` | -| — | [FixedString](../sql-reference/data-types/fixedstring.md) | `STRING` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | +| — | [FixedString](../sql-reference/data-types/fixedstring.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](../sql-reference/data-types/tuple.md) | `STRUCT` | +| `MAP` | [Map](../sql-reference/data-types/map.md) | `MAP` | -Arrays can be nested and can have a value of the `Nullable` type as an argument. +Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested. ClickHouse supports configurable precision of `Decimal` type. The `INSERT` query treats the Parquet `DECIMAL` type as the ClickHouse `Decimal128` type. @@ -1299,13 +1301,17 @@ The table below shows supported data types and how they match ClickHouse [data t | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `FLOAT64` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | -| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `UTF8` | -| `STRING`, `BINARY` | [FixedString](../sql-reference/data-types/fixedstring.md) | `UTF8` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | +| `STRING`, `BINARY` | [FixedString](../sql-reference/data-types/fixedstring.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `DECIMAL256` | [Decimal256](../sql-reference/data-types/decimal.md)| `DECIMAL256` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](../sql-reference/data-types/tuple.md) | `STRUCT` | +| `MAP` | [Map](../sql-reference/data-types/map.md) | `MAP` | -Arrays can be nested and can have a value of the `Nullable` type as an argument. +Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested. + +The `DICTIONARY` type is supported for `INSERT` queries, and for `SELECT` queries there is an [output_format_arrow_low_cardinality_as_dictionary](../operations/settings/settings.md#output-format-arrow-low-cardinality-as-dictionary) setting that allows to output [LowCardinality](../sql-reference/data-types/lowcardinality.md) type as a `DICTIONARY` type. ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` query treats the Arrow `DECIMAL` type as the ClickHouse `Decimal128` type. @@ -1358,8 +1364,10 @@ The table below shows supported data types and how they match ClickHouse [data t | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](../sql-reference/data-types/tuple.md) | `STRUCT` | +| `MAP` | [Map](../sql-reference/data-types/map.md) | `MAP` | -Arrays can be nested and can have a value of the `Nullable` type as an argument. +Arrays can be nested and can have a value of the `Nullable` type as an argument. `Tuple` and `Map` types also can be nested. ClickHouse supports configurable precision of the `Decimal` type. The `INSERT` query treats the ORC `DECIMAL` type as the ClickHouse `Decimal128` type. diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 990cb30346c..eed673234ba 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -157,5 +157,6 @@ toc_title: Adopters | SigNoz | Observability Platform | Main Product | — | — | [Source code](https://github.com/SigNoz/signoz) | | ChelPipe Group | Analytics | — | — | — | [Blog post, June 2021](https://vc.ru/trade/253172-tyazhelomu-proizvodstvu-user-friendly-sayt-internet-magazin-trub-dlya-chtpz) | | Zagrava Trading | — | — | — | — | [Job offer, May 2021](https://twitter.com/datastackjobs/status/1394707267082063874) | +| Beeline | Telecom | Data Platform | — | — | [Blog post, July 2021](https://habr.com/en/company/beeline/blog/567508/) | [Original article](https://clickhouse.tech/docs/en/introduction/adopters/) diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md new file mode 100644 index 00000000000..6af12eb9b01 --- /dev/null +++ b/docs/en/operations/clickhouse-keeper.md @@ -0,0 +1,114 @@ +--- +toc_priority: 66 +toc_title: ClickHouse Keeper +--- + +# [pre-production] clickhouse-keeper + +ClickHouse server use [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper. + +!!! warning "Warning" + This feature currently in pre-production stage. We test it in our CI and on small internal installations. + +## Implemetation details + +ZooKeeper is one of the first well-known open-source coordination systems. It's implemented in Java, has quite a simple and powerful data model. ZooKeeper's coordination algorithm called ZAB (ZooKeeper Atomic Broadcast) doesn't provide linearizability guarantees for reads, because each ZooKeeper node serves reads locally. Unlike ZooKeeper `clickhouse-keeper` written in C++ and use [RAFT algorithm](https://raft.github.io/) [implementation](https://github.com/eBay/NuRaft). This algorithm allows to have linearizability for reads and writes, has several open-source implementations in different languages. + +By default, `clickhouse-keeper` provides the same guarantees as ZooKeeper (linearizable writes, non-linearizable reads). It has a compatible client-server protocol, so any standard ZooKeeper client can be used to interact with `clickhouse-keeper`. Snapshots and logs have an incompatible format with ZooKeeper, but `clickhouse-keeper-converter` tool allows to convert ZooKeeper data to `clickhouse-keeper` snapshot. Interserver protocol in `clickhouse-keeper` also incompatible with ZooKeeper so mixed ZooKeeper/clickhouse-keeper cluster is impossible. + +## Configuration + +`clickhouse-keeper` can be used as a standalone replacement for ZooKeeper or as an internal part of the `clickhouse-server`, but in both cases configuration is almost the same `.xml` file. The main `clickhouse-keeper` configuration tag is ``. Keeper configuration has the following parameters: + +- `tcp_port` — the port for a client to connect (default for ZooKeeper is `2181`) +- `tcp_port_secure` — the secure port for a client to connect +- `server_id` — unique server id, each participant of the clickhouse-keeper cluster must have a unique number (1, 2, 3, and so on) +- `log_storage_path` — path to coordination logs, better to store logs on the non-busy device (same for ZooKeeper) +- `snapshot_storage_path` — path to coordination snapshots + +Other common parameters are inherited from clickhouse-server config (`listen_host`, `logger` and so on). + +Internal coordination settings are located in `.` section: + +- `operation_timeout_ms` — timeout for a single client operation +- `session_timeout_ms` — timeout for client session +- `dead_session_check_period_ms` — how often clickhouse-keeper check dead sessions and remove them +- `heart_beat_interval_ms` — how often a clickhouse-keeper leader will send heartbeats to followers +- `election_timeout_lower_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it can initiate leader election +- `election_timeout_upper_bound_ms` — if follower didn't receive heartbeats from the leader in this interval, then it must initiate leader election +- `rotate_log_storage_interval` — how many logs to store in a single file +- `reserved_log_items` — how many coordination logs to store before compaction +- `snapshot_distance` — how often clickhouse-keeper will create new snapshots (in the number of logs) +- `snapshots_to_keep` — how many snapshots to keep +- `stale_log_gap` — the threshold when leader consider follower as stale and send snapshot to it instead of logs +- `force_sync` — call `fsync` on each write to coordination log +- `raft_logs_level` — text logging level about coordination (trace, debug, and so on) +- `shutdown_timeout` — wait to finish internal connections and shutdown +- `startup_timeout` — if the server doesn't connect to other quorum participants in the specified timeout it will terminate + +Quorum configuration is located in `.` section and contain servers description. The only parameter for the whole quorum is `secure`, which enables encrypted connection for communication between quorum participants. The main parameters for each `` are: + +- `id` — server_id in quorum +- `hostname` — hostname where this server placed +- `port` — port where this server listen for connections + + +Examples of configuration for quorum with three nodes can be found in [integration tests](https://github.com/ClickHouse/ClickHouse/tree/master/tests/integration) with `test_keeper_` prefix. Example configuration for server #1: + +```xml + + 2181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 10000 + 30000 + trace + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + +``` + +## How to run + +`clickhouse-keeper` is bundled into `clickhouse-server` package, just add configuration of `` and start clickhouse-server as always. If you want to run standalone `clickhouse-keeper` you can start it in a similar way with: + +```bash +clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon +``` + +## [experimental] Migration from ZooKeeper + +Seamlessly migration from ZooKeeper to `clickhouse-keeper` is impossible you have to stop your ZooKeeper cluster, convert data and start `clickhouse-keeper`. `clickhouse-keeper-converter` tool allows to convert ZooKeeper logs and snapshots to `clickhouse-keeper` snapshot. It works only with ZooKeeper > 3.4. Steps for migration: + +1. Stop all ZooKeeper nodes. + +2. [optional, but recommended] Found ZooKeeper leader node, start and stop it again. It will force ZooKeeper to create consistent snapshot. + +3. Run `clickhouse-keeper-converter` on leader, example + +```bash +clickhouse-keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/version-2 --output-dir /path/to/clickhouse/keeper/snapshots +``` + +4. Copy snapshot to `clickhouse-server` nodes with configured `keeper` or start `clickhouse-keeper` instead of ZooKeeper. Snapshot must persist only on leader node, leader will sync it automatically to other nodes. + diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md index 96009c75af1..5c942efc77f 100644 --- a/docs/en/operations/configuration-files.md +++ b/docs/en/operations/configuration-files.md @@ -22,6 +22,23 @@ Some settings specified in the main configuration file can be overridden in othe The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md)). +If you want to replace an entire element with a substitution use `include` as element name. + +XML substitution example: + +```xml + + + + + + + + + + +``` + Substitutions can also be performed from ZooKeeper. To do this, specify the attribute `from_zk = "/path/to/node"`. The element value is replaced with the contents of the node at `/path/to/node` in ZooKeeper. You can also put an entire XML subtree on the ZooKeeper node and it will be fully inserted into the source element. ## User Settings {#user-settings} @@ -32,6 +49,8 @@ Users configuration can be splitted into separate files similar to `config.xml` Directory name is defined as `users_config` setting without `.xml` postfix concatenated with `.d`. Directory `users.d` is used by default, as `users_config` defaults to `users.xml`. +Note that configuration files are first merged taking into account [Override](#override) settings and includes are processed after that. + ## XML example {#example} For example, you can have separate config file for each user like this: diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index fc5a911cd7a..aed769731fd 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1213,7 +1213,15 @@ Default value: `3`. ## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers} -If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes. +Controls quoting of 64-bit or bigger [integers](../../sql-reference/data-types/int-uint.md) (like `UInt64` or `Int128`) when they are output in a [JSON](../../interfaces/formats.md#json) format. +Such integers are enclosed in quotes by default. This behavior is compatible with most JavaScript implementations. + +Possible values: + +- 0 — Integers are output without quotes. +- 1 — Integers are enclosed in quotes. + +Default value: 1. ## output_format_json_quote_denormals {#settings-output_format_json_quote_denormals} @@ -1990,6 +1998,16 @@ Possible values: Default value: 16. +## merge_selecting_sleep_ms {#merge_selecting_sleep_ms} + +Sleep time for merge selecting when no part selected, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters + +Possible values: + +- Any positive integer. + +Default value: 5000 + ## parallel_distributed_insert_select {#parallel_distributed_insert_select} Enables parallel distributed `INSERT ... SELECT` query. @@ -3202,3 +3220,14 @@ Default value: `1`. **Usage** If the setting is set to `0`, the table function does not make Nullable columns and inserts default values instead of NULL. This is also applicable for NULL values inside arrays. + +## output_format_arrow_low_cardinality_as_dictionary {#output-format-arrow-low-cardinality-as-dictionary} + +Allows to convert the [LowCardinality](../../sql-reference/data-types/lowcardinality.md) type to the `DICTIONARY` type of the [Arrow](../../interfaces/formats.md#data-format-arrow) format for `SELECT` queries. + +Possible values: + +- 0 — The `LowCardinality` type is not converted to the `DICTIONARY` type. +- 1 — The `LowCardinality` type is converted to the `DICTIONARY` type. + +Default value: `0`. diff --git a/docs/en/operations/system-tables/data_skipping_indices.md b/docs/en/operations/system-tables/data_skipping_indices.md index 515f704797a..683666e1f77 100644 --- a/docs/en/operations/system-tables/data_skipping_indices.md +++ b/docs/en/operations/system-tables/data_skipping_indices.md @@ -8,12 +8,11 @@ Columns: - `table` ([String](../../sql-reference/data-types/string.md)) — Table name. - `name` ([String](../../sql-reference/data-types/string.md)) — Index name. - `type` ([String](../../sql-reference/data-types/string.md)) — Index type. -- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression used to calculate the index. -- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of granules in the block. +- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression for the index calculation. +- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of granules in the block. **Example** - ```sql SELECT * FROM system.data_skipping_indices LIMIT 2 FORMAT Vertical; ``` diff --git a/docs/en/sql-reference/aggregate-functions/reference/median.md b/docs/en/sql-reference/aggregate-functions/reference/median.md index d487a187945..619e9a5093e 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/median.md +++ b/docs/en/sql-reference/aggregate-functions/reference/median.md @@ -34,7 +34,7 @@ Input table: Query: ``` sql -SELECT medianDeterministic(val, 1) FROM t +SELECT medianDeterministic(val, 1) FROM t; ``` Result: diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md index 5f0f400ce43..b3ff26a943d 100644 --- a/docs/en/sql-reference/data-types/lowcardinality.md +++ b/docs/en/sql-reference/data-types/lowcardinality.md @@ -47,6 +47,7 @@ Settings: - [low_cardinality_use_single_dictionary_for_part](../../operations/settings/settings.md#low_cardinality_use_single_dictionary_for_part) - [low_cardinality_allow_in_native_format](../../operations/settings/settings.md#low_cardinality_allow_in_native_format) - [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types) +- [output_format_arrow_low_cardinality_as_dictionary](../../operations/settings/settings.md#output-format-arrow-low-cardinality-as-dictionary) Functions: @@ -57,5 +58,3 @@ Functions: - [A Magical Mystery Tour of the LowCardinality Data Type](https://www.altinity.com/blog/2019/3/27/low-cardinality). - [Reducing ClickHouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). - [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). - -[Original article](https://clickhouse.tech/docs/en/sql-reference/data-types/lowcardinality/) diff --git a/docs/en/sql-reference/data-types/map.md b/docs/en/sql-reference/data-types/map.md index 86ea55004fd..6abd150b20f 100644 --- a/docs/en/sql-reference/data-types/map.md +++ b/docs/en/sql-reference/data-types/map.md @@ -12,9 +12,6 @@ toc_title: Map(key, value) - `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md). - `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md). -!!! warning "Warning" - Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`. - To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. This lookup works now with a linear complexity. **Examples** diff --git a/docs/en/sql-reference/functions/ext-dict-functions.md b/docs/en/sql-reference/functions/ext-dict-functions.md index 7c0fe11ae64..d7f142dd8b1 100644 --- a/docs/en/sql-reference/functions/ext-dict-functions.md +++ b/docs/en/sql-reference/functions/ext-dict-functions.md @@ -12,7 +12,7 @@ For information on connecting and configuring external dictionaries, see [Extern ## dictGet, dictGetOrDefault, dictGetOrNull {#dictget} -Retrieves values from an external dictionary. +Retrieves values from an external dictionary. ``` sql dictGet('dict_name', attr_names, id_expr) @@ -24,7 +24,7 @@ dictGetOrNull('dict_name', attr_name, id_expr) - `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). - `attr_names` — Name of the column of the dictionary, [String literal](../../sql-reference/syntax.md#syntax-string-literal), or tuple of column names, [Tuple](../../sql-reference/data-types/tuple.md)([String literal](../../sql-reference/syntax.md#syntax-string-literal)). -- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration. +- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration. - `default_value_expr` — Values returned if the dictionary does not contain a row with the `id_expr` key. [Expression](../../sql-reference/syntax.md#syntax-expressions) or [Tuple](../../sql-reference/data-types/tuple.md)([Expression](../../sql-reference/syntax.md#syntax-expressions)), returning the value (or values) in the data types configured for the `attr_names` attribute. **Returned value** @@ -138,7 +138,7 @@ Configure the external dictionary: c2 String - + 0 @@ -237,7 +237,7 @@ dictHas('dict_name', id_expr) **Arguments** - `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). -- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md) or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration. +- `id_expr` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning dictionary key-type value or [Tuple](../../sql-reference/data-types/tuple.md)-type value depending on the dictionary configuration. **Returned value** @@ -292,16 +292,16 @@ Type: `UInt8`. Returns first-level children as an array of indexes. It is the inverse transformation for [dictGetHierarchy](#dictgethierarchy). -**Syntax** +**Syntax** ``` sql dictGetChildren(dict_name, key) ``` -**Arguments** +**Arguments** -- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). -- `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value. +- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). +- `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value. **Returned values** @@ -339,7 +339,7 @@ SELECT dictGetChildren('hierarchy_flat_dictionary', number) FROM system.numbers ## dictGetDescendant {#dictgetdescendant} -Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively. +Returns all descendants as if [dictGetChildren](#dictgetchildren) function was applied `level` times recursively. **Syntax** @@ -347,9 +347,9 @@ Returns all descendants as if [dictGetChildren](#dictgetchildren) function was a dictGetDescendants(dict_name, key, level) ``` -**Arguments** +**Arguments** -- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). +- `dict_name` — Name of the dictionary. [String literal](../../sql-reference/syntax.md#syntax-string-literal). - `key` — Key value. [Expression](../../sql-reference/syntax.md#syntax-expressions) returning a [UInt64](../../sql-reference/data-types/int-uint.md)-type value. - `level` — Hierarchy level. If `level = 0` returns all descendants to the end. [UInt8](../../sql-reference/data-types/int-uint.md). diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 20dc7b29902..6c03f55cebe 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -195,6 +195,41 @@ Result: └────────────────────┘ ``` +## h3ToGeo {#h3togeo} + +Returns `(lon, lat)` that corresponds to the provided H3 index. + +**Syntax** + +``` sql +h3ToGeo(h3Index) +``` + +**Arguments** + +- `h3Index` — H3 Index. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Returned values** + +- `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md). +- `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md). + + +**Example** + +Query: + +``` sql +SELECT h3ToGeo(644325524701193974) coordinates; +``` + +Result: + +``` text +┌─coordinates───────────────────────────┐ +│ (37.79506616830252,55.71290243145668) │ +└───────────────────────────────────────┘ +``` ## h3kRing {#h3kring} Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order. diff --git a/docs/en/sql-reference/functions/json-functions.md b/docs/en/sql-reference/functions/json-functions.md index e731180c393..596ad17f07d 100644 --- a/docs/en/sql-reference/functions/json-functions.md +++ b/docs/en/sql-reference/functions/json-functions.md @@ -306,3 +306,49 @@ Result: └───────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` +## toJSONString {#tojsonstring} + +Serializes a value to its JSON representation. Various data types and nested structures are supported. +64-bit [integers](../../sql-reference/data-types/int-uint.md) or bigger (like `UInt64` or `Int128`) are enclosed in quotes by default. [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) controls this behavior. +Special values `NaN` and `inf` are replaced with `null`. Enable [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals) setting to show them. +When serializing an [Enum](../../sql-reference/data-types/enum.md) value, the function outputs its name. + +**Syntax** + +``` sql +toJSONString(value) +``` + +**Arguments** + +- `value` — Value to serialize. Value may be of any data type. + +**Returned value** + +- JSON representation of the value. + +Type: [String](../../sql-reference/data-types/string.md). + +**Example** + +The first example shows serialization of a [Map](../../sql-reference/data-types/map.md). +The second example shows some special values wrapped into a [Tuple](../../sql-reference/data-types/tuple.md). + +Query: + +``` sql +SELECT toJSONString(map('key1', 1, 'key2', 2)); +SELECT toJSONString(tuple(1.25, NULL, NaN, +inf, -inf, [])) SETTINGS output_format_json_quote_denormals = 1; +``` + +Result: + +``` text +{"key1":1,"key2":2} +[1.25,null,"nan","inf","-inf",[]] +``` + +**See Also** + +- [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) +- [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals) diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 4189d0feeb5..39e59ae2ba9 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -87,6 +87,8 @@ Result: └───────┴───────┘ ``` +Note: the names are implementation specific and are subject to change. You should not assume specific names of the columns after application of the `untuple`. + Example of using an `EXCEPT` expression: Query: diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md deleted file mode 120000 index 8e172b919d8..00000000000 --- a/docs/ru/development/build-osx.md +++ /dev/null @@ -1 +0,0 @@ -../../en/development/build-osx.md \ No newline at end of file diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md new file mode 100644 index 00000000000..49da9f2b359 --- /dev/null +++ b/docs/ru/development/build-osx.md @@ -0,0 +1,125 @@ +--- +toc_priority: 65 +toc_title: Сборка на Mac OS X +--- +# Как собрать ClickHouse на Mac OS X {#how-to-build-clickhouse-on-mac-os-x} + +Сборка должна запускаться с x86_64 (Intel) на macOS версии 10.15 (Catalina) и выше в последней версии компилятора Xcode's native AppleClang, Homebrew's vanilla Clang или в GCC-компиляторах. + +## Установка Homebrew {#install-homebrew} + +``` bash +$ /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" +``` + +## Установка Xcode и инструментов командной строки {#install-xcode-and-command-line-tools} + + 1. Установите из App Store последнюю версию [Xcode](https://apps.apple.com/am/app/xcode/id497799835?mt=12). + + 2. Запустите ее, чтобы принять лицензионное соглашение. Необходимые компоненты установятся автоматически. + + 3. Затем убедитесь, что в системе выбрана последняя версия инструментов командной строки: + + ``` bash + $ sudo rm -rf /Library/Developer/CommandLineTools + $ sudo xcode-select --install + ``` + + 4. Перезагрузитесь. + +## Установка компиляторов, инструментов и библиотек {#install-required-compilers-tools-and-libraries} + + ``` bash + $ brew update + $ brew install cmake ninja libtool gettext llvm gcc + ``` + +## Просмотр исходников ClickHouse {#checkout-clickhouse-sources} + + ``` bash + $ git clone --recursive git@github.com:ClickHouse/ClickHouse.git # or https://github.com/ClickHouse/ClickHouse.git + ``` + +## Сборка ClickHouse {#build-clickhouse} + + Чтобы запустить сборку в компиляторе Xcode's native AppleClang: + + ``` bash + $ cd ClickHouse + $ rm -rf build + $ mkdir build + $ cd build + $ cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. + $ cmake --build . --config RelWithDebInfo + $ cd .. + ``` + +Чтобы запустить сборку в компиляторе Homebrew's vanilla Clang: + + ``` bash + $ cd ClickHouse + $ rm -rf build + $ mkdir build + $ cd build + $ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER==$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. + $ cmake -DCMAKE_C_COMPILER=$(brew --prefix llvm)/bin/clang -DCMAKE_CXX_COMPILER=$(brew --prefix llvm)/bin/clang++ -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. + $ cmake --build . --config RelWithDebInfo + $ cd .. + ``` + +Чтобы собрать с помощью компилятора Homebrew's vanilla GCC: + + ``` bash + $ cd ClickHouse + $ rm -rf build + $ mkdir build + $ cd build + $ cmake -DCMAKE_C_COMPILER=$(brew --prefix gcc)/bin/gcc-10 -DCMAKE_CXX_COMPILER=$(brew --prefix gcc)/bin/g++-10 -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_JEMALLOC=OFF .. + $ cmake --build . --config RelWithDebInfo + $ cd .. + ``` + +## Предупреждения {#caveats} + +Если будете запускать `clickhouse-server`, убедитесь, что увеличили системную переменную `maxfiles`. + +!!! info "Note" + Вам понадобится команда `sudo`. + +1. Создайте файл `/Library/LaunchDaemons/limit.maxfiles.plist` и поместите в него следующее: + + ``` xml + + + + + Label + limit.maxfiles + ProgramArguments + + launchctl + limit + maxfiles + 524288 + 524288 + + RunAtLoad + + ServiceIPC + + + + ``` + +2. Выполните команду: + + ``` bash + $ sudo chown root:wheel /Library/LaunchDaemons/limit.maxfiles.plist + ``` + +3. Перезагрузитесь. + +4. Чтобы проверить, как это работает, выполните команду `ulimit -n`. + +[Original article](https://clickhouse.tech/docs/en/development/build_osx/) diff --git a/docs/ru/engines/database-engines/materialize-mysql.md b/docs/ru/engines/database-engines/materialize-mysql.md index 2067dfecca0..db2208a9016 100644 --- a/docs/ru/engines/database-engines/materialize-mysql.md +++ b/docs/ru/engines/database-engines/materialize-mysql.md @@ -49,6 +49,7 @@ ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'passwor | DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) | | DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | | DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) | +| ENUM | [Enum](../../sql-reference/data-types/enum.md) | | STRING | [String](../../sql-reference/data-types/string.md) | | VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) | | BLOB | [String](../../sql-reference/data-types/string.md) | diff --git a/docs/ru/getting-started/install.md b/docs/ru/getting-started/install.md index 66a94bcfbca..b6e7c3a2793 100644 --- a/docs/ru/getting-started/install.md +++ b/docs/ru/getting-started/install.md @@ -100,9 +100,9 @@ sudo ./clickhouse install Для других операционных систем и архитектуры AArch64 сборки ClickHouse предоставляются в виде кросс-компилированного бинарного файла из последнего коммита ветки `master` (с задержкой в несколько часов). -- [macOS](https://builds.clickhouse.tech/master/macos/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/macos/clickhouse' && chmod a+x ./clickhouse` -- [AArch64](https://builds.clickhouse.tech/master/aarch64/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/aarch64/clickhouse' && chmod a+x ./clickhouse` -- [FreeBSD](https://builds.clickhouse.tech/master/freebsd/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/freebsd/clickhouse' && chmod a+x ./clickhouse` +- [macOS](https://builds.clickhouse.tech/master/macos/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/macos/clickhouse' && chmod a+x ./clickhouse` +- [FreeBSD](https://builds.clickhouse.tech/master/freebsd/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/freebsd/clickhouse' && chmod a+x ./clickhouse` +- [AArch64](https://builds.clickhouse.tech/master/aarch64/clickhouse) — `curl -O 'https://builds.clickhouse.tech/master/aarch64/clickhouse' && chmod a+x ./clickhouse` После скачивания можно воспользоваться `clickhouse client` для подключения к серверу или `clickhouse local` для обработки локальных данных. diff --git a/docs/ru/interfaces/formats.md b/docs/ru/interfaces/formats.md index 7780a75a706..563a137ac17 100644 --- a/docs/ru/interfaces/formats.md +++ b/docs/ru/interfaces/formats.md @@ -1165,12 +1165,14 @@ SELECT * FROM topic1_stream; | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `DOUBLE` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | -| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `STRING` | -| — | [FixedString](../sql-reference/data-types/fixedstring.md) | `STRING` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | +| — | [FixedString](../sql-reference/data-types/fixedstring.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](../sql-reference/data-types/tuple.md) | `STRUCT` | +| `MAP` | [Map](../sql-reference/data-types/map.md) | `MAP` | -Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. +Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. Типы `Tuple` и `Map` также могут быть вложенными. ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При выполнении запроса `INSERT` ClickHouse обрабатывает тип данных Parquet `DECIMAL` как `Decimal128`. @@ -1218,12 +1220,17 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Parquet" > {some_ | `DOUBLE` | [Float64](../sql-reference/data-types/float.md) | `FLOAT64` | | `DATE32` | [Date](../sql-reference/data-types/date.md) | `UINT16` | | `DATE64`, `TIMESTAMP` | [DateTime](../sql-reference/data-types/datetime.md) | `UINT32` | -| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `UTF8` | -| `STRING`, `BINARY` | [FixedString](../sql-reference/data-types/fixedstring.md) | `UTF8` | +| `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | +| `STRING`, `BINARY` | [FixedString](../sql-reference/data-types/fixedstring.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | +| `DECIMAL256` | [Decimal256](../sql-reference/data-types/decimal.md)| `DECIMAL256` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](../sql-reference/data-types/tuple.md) | `STRUCT` | +| `MAP` | [Map](../sql-reference/data-types/map.md) | `MAP` | -Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. +Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. Типы `Tuple` и `Map` также могут быть вложенными. + +Тип `DICTIONARY` поддерживается для запросов `INSERT`. Для запросов `SELECT` есть настройка [output_format_arrow_low_cardinality_as_dictionary](../operations/settings/settings.md#output-format-arrow-low-cardinality-as-dictionary), которая позволяет выводить тип [LowCardinality](../sql-reference/data-types/lowcardinality.md) как `DICTIONARY`. ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При выполнении запроса `INSERT` ClickHouse обрабатывает тип данных Arrow `DECIMAL` как `Decimal128`. @@ -1276,8 +1283,10 @@ $ clickhouse-client --query="SELECT * FROM {some_table} FORMAT Arrow" > {filenam | `STRING`, `BINARY` | [String](../sql-reference/data-types/string.md) | `BINARY` | | `DECIMAL` | [Decimal](../sql-reference/data-types/decimal.md) | `DECIMAL` | | `LIST` | [Array](../sql-reference/data-types/array.md) | `LIST` | +| `STRUCT` | [Tuple](../sql-reference/data-types/tuple.md) | `STRUCT` | +| `MAP` | [Map](../sql-reference/data-types/map.md) | `MAP` | -Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. +Массивы могут быть вложенными и иметь в качестве аргумента значение типа `Nullable`. Типы `Tuple` и `Map` также могут быть вложенными. ClickHouse поддерживает настраиваемую точность для формата `Decimal`. При выполнении запроса `INSERT` ClickHouse обрабатывает тип данных ORC `DECIMAL` как `Decimal128`. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index 625453c94c6..85299010aea 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1204,8 +1204,15 @@ load_balancing = round_robin Работает для форматов JSONEachRow и TSKV. ## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers} +Управляет кавычками при выводе 64-битных или более [целых чисел](../../sql-reference/data-types/int-uint.md) (например, `UInt64` или `Int128`) в формате [JSON](../../interfaces/formats.md#json). +По умолчанию такие числа заключаются в кавычки. Это поведение соответствует большинству реализаций JavaScript. -Если значение истинно, то при использовании JSON\* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек. +Возможные значения: + +- 0 — числа выводятся без кавычек. +- 1 — числа выводятся в кавычках. + +Значение по умолчанию: 1. ## output_format_json_quote_denormals {#settings-output_format_json_quote_denormals} @@ -3059,3 +3066,14 @@ SETTINGS index_granularity = 8192 │ **Использование** Если установлено значение `0`, то табличная функция не делает Nullable столбцы, а вместо NULL выставляет значения по умолчанию для скалярного типа. Это также применимо для значений NULL внутри массивов. + +## output_format_arrow_low_cardinality_as_dictionary {#output-format-arrow-low-cardinality-as-dictionary} + +Позволяет конвертировать тип [LowCardinality](../../sql-reference/data-types/lowcardinality.md) в тип `DICTIONARY` формата [Arrow](../../interfaces/formats.md#data-format-arrow) для запросов `SELECT`. + +Возможные значения: + +- 0 — тип `LowCardinality` не конвертируется в тип `DICTIONARY`. +- 1 — тип `LowCardinality` конвертируется в тип `DICTIONARY`. + +Значение по умолчанию: `0`. diff --git a/docs/ru/operations/system-tables/data_skipping_indices.md b/docs/ru/operations/system-tables/data_skipping_indices.md new file mode 100644 index 00000000000..39e13ed1d5a --- /dev/null +++ b/docs/ru/operations/system-tables/data_skipping_indices.md @@ -0,0 +1,38 @@ +# system.data_skipping_indices {#system-data-skipping-indices} + +Содержит информацию о существующих индексах пропуска данных во всех таблицах. + +Столбцы: + +- `database` ([String](../../sql-reference/data-types/string.md)) — имя базы данных. +- `table` ([String](../../sql-reference/data-types/string.md)) — имя таблицы. +- `name` ([String](../../sql-reference/data-types/string.md)) — имя индекса. +- `type` ([String](../../sql-reference/data-types/string.md)) — тип индекса. +- `expr` ([String](../../sql-reference/data-types/string.md)) — выражение, используемое для вычисления индекса. +- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество гранул в блоке данных. + +**Пример** + +```sql +SELECT * FROM system.data_skipping_indices LIMIT 2 FORMAT Vertical; +``` + +```text +Row 1: +────── +database: default +table: user_actions +name: clicks_idx +type: minmax +expr: clicks +granularity: 1 + +Row 2: +────── +database: default +table: users +name: contacts_null_idx +type: minmax +expr: assumeNotNull(contacts_null) +granularity: 1 +``` diff --git a/docs/ru/sql-reference/aggregate-functions/reference/median.md b/docs/ru/sql-reference/aggregate-functions/reference/median.md index 1472809e2e3..0c4b0db12c5 100644 --- a/docs/ru/sql-reference/aggregate-functions/reference/median.md +++ b/docs/ru/sql-reference/aggregate-functions/reference/median.md @@ -4,7 +4,6 @@ Функции: - - `median` — синоним для [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile). - `medianDeterministic` — синоним для [quantileDeterministic](../../../sql-reference/aggregate-functions/reference/quantiledeterministic.md#quantiledeterministic). - `medianExact` — синоним для [quantileExact](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexact). @@ -31,7 +30,7 @@ Запрос: ``` sql -SELECT medianDeterministic(val, 1) FROM t +SELECT medianDeterministic(val, 1) FROM t; ``` Результат: @@ -41,4 +40,3 @@ SELECT medianDeterministic(val, 1) FROM t │ 1.5 │ └─────────────────────────────┘ ``` - diff --git a/docs/ru/sql-reference/data-types/lowcardinality.md b/docs/ru/sql-reference/data-types/lowcardinality.md index fe9118b1e14..71282835372 100644 --- a/docs/ru/sql-reference/data-types/lowcardinality.md +++ b/docs/ru/sql-reference/data-types/lowcardinality.md @@ -15,7 +15,7 @@ LowCardinality(data_type) **Параметры** -- `data_type` — [String](string.md), [FixedString](fixedstring.md), [Date](date.md), [DateTime](datetime.md) и числа за исключением типа [Decimal](decimal.md). `LowCardinality` неэффективен для некоторых типов данных, см. описание настройки [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types). +- `data_type` — [String](string.md), [FixedString](fixedstring.md), [Date](date.md), [DateTime](datetime.md) и числа за исключением типа [Decimal](decimal.md). `LowCardinality` неэффективен для некоторых типов данных, см. описание настройки [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types). ## Описание {#lowcardinality-dscr} @@ -23,11 +23,11 @@ LowCardinality(data_type) Эффективность использования типа данных `LowCarditality` зависит от разнообразия данных. Если словарь содержит менее 10 000 различных значений, ClickHouse в основном показывает более высокую эффективность чтения и хранения данных. Если же словарь содержит более 100 000 различных значений, ClickHouse может работать хуже, чем при использовании обычных типов данных. -При работе со строками, использование `LowCardinality` вместо [Enum](enum.md) обеспечивает большую гибкость в использовании и часто показывает такую же или более высокую эффективность. +При работе со строками использование `LowCardinality` вместо [Enum](enum.md) обеспечивает большую гибкость в использовании и часто показывает такую же или более высокую эффективность. ## Пример -Создать таблицу со столбцами типа `LowCardinality`: +Создание таблицы со столбцами типа `LowCardinality`: ```sql CREATE TABLE lc_t @@ -43,18 +43,18 @@ ORDER BY id Настройки: -- [low_cardinality_max_dictionary_size](../../operations/settings/settings.md#low_cardinality_max_dictionary_size) -- [low_cardinality_use_single_dictionary_for_part](../../operations/settings/settings.md#low_cardinality_use_single_dictionary_for_part) -- [low_cardinality_allow_in_native_format](../../operations/settings/settings.md#low_cardinality_allow_in_native_format) -- [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types) +- [low_cardinality_max_dictionary_size](../../operations/settings/settings.md#low_cardinality_max_dictionary_size) +- [low_cardinality_use_single_dictionary_for_part](../../operations/settings/settings.md#low_cardinality_use_single_dictionary_for_part) +- [low_cardinality_allow_in_native_format](../../operations/settings/settings.md#low_cardinality_allow_in_native_format) +- [allow_suspicious_low_cardinality_types](../../operations/settings/settings.md#allow_suspicious_low_cardinality_types) +- [output_format_arrow_low_cardinality_as_dictionary](../../operations/settings/settings.md#output-format-arrow-low-cardinality-as-dictionary) Функции: -- [toLowCardinality](../functions/type-conversion-functions.md#tolowcardinality) +- [toLowCardinality](../functions/type-conversion-functions.md#tolowcardinality) ## Смотрите также -- [A Magical Mystery Tour of the LowCardinality Data Type](https://www.altinity.com/blog/2019/3/27/low-cardinality). -- [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). -- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). - +- [A Magical Mystery Tour of the LowCardinality Data Type](https://www.altinity.com/blog/2019/3/27/low-cardinality). +- [Reducing Clickhouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://www.instana.com/blog/reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer/). +- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/yandex/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf). diff --git a/docs/ru/sql-reference/data-types/map.md b/docs/ru/sql-reference/data-types/map.md index a703eb1b0ac..aceeb21b6e6 100644 --- a/docs/ru/sql-reference/data-types/map.md +++ b/docs/ru/sql-reference/data-types/map.md @@ -12,9 +12,6 @@ toc_title: Map(key, value) - `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md). - `value` — значение. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) или [Array](../../sql-reference/data-types/array.md). -!!! warning "Предупреждение" - Сейчас использование типа данных `Map` является экспериментальной возможностью. Чтобы использовать этот тип данных, включите настройку `allow_experimental_map_type = 1`. - Чтобы получить значение из колонки `a Map('key', 'value')`, используйте синтаксис `a['key']`. В настоящее время такая подстановка работает по алгоритму с линейной сложностью. **Примеры** diff --git a/docs/ru/sql-reference/functions/json-functions.md b/docs/ru/sql-reference/functions/json-functions.md index 8941ccc1691..b935244e821 100644 --- a/docs/ru/sql-reference/functions/json-functions.md +++ b/docs/ru/sql-reference/functions/json-functions.md @@ -306,3 +306,51 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello" │ [('d','"hello"'),('f','"world"')] │ └───────────────────────────────────────────────────────────────────────────────────────────────────────┘ ``` + + +## toJSONString {#tojsonstring} + +Сериализует значение в JSON представление. Поддерживаются различные типы данных и вложенные структуры. +По умолчанию 64-битные [целые числа](../../sql-reference/data-types/int-uint.md) и более (например, `UInt64` или `Int128`) заключаются в кавычки. Настройка [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) управляет этим поведением. +Специальные значения `NaN` и `inf` заменяются на `null`. Чтобы они отображались, включите настройку [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals). +Когда сериализуется значение [Enum](../../sql-reference/data-types/enum.md), то функция выводит его имя. + +**Синтаксис** + +``` sql +toJSONString(value) +``` + +**Аргументы** + +- `value` — значение, которое необходимо сериализовать. Может быть любого типа. + +**Возвращаемое значение** + +- JSON представление значения. + +Тип: [String](../../sql-reference/data-types/string.md). + +**Пример** + +Первый пример показывает сериализацию [Map](../../sql-reference/data-types/map.md). +Во втором примере есть специальные значения, обернутые в [Tuple](../../sql-reference/data-types/tuple.md). + +Запрос: + +``` sql +SELECT toJSONString(map('key1', 1, 'key2', 2)); +SELECT toJSONString(tuple(1.25, NULL, NaN, +inf, -inf, [])) SETTINGS output_format_json_quote_denormals = 1; +``` + +Результат: + +``` text +{"key1":1,"key2":2} +[1.25,null,"nan","inf","-inf",[]] +``` + +**Смотрите также** + +- [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) +- [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals) diff --git a/docs/ru/sql-reference/statements/alter/partition.md b/docs/ru/sql-reference/statements/alter/partition.md index 0a485c7b591..f875103a498 100644 --- a/docs/ru/sql-reference/statements/alter/partition.md +++ b/docs/ru/sql-reference/statements/alter/partition.md @@ -17,7 +17,7 @@ toc_title: PARTITION - [CLEAR INDEX IN PARTITION](#alter_clear-index-partition) — очистить построенные вторичные индексы для заданной партиции; - [FREEZE PARTITION](#alter_freeze-partition) — создать резервную копию партиции; - [UNFREEZE PARTITION](#alter_unfreeze-partition) — удалить резервную копию партиции; -- [FETCH PARTITION](#alter_fetch-partition) — скачать партицию с другого сервера; +- [FETCH PARTITION\|PART](#alter_fetch-partition) — скачать партицию/кусок с другого сервера; - [MOVE PARTITION\|PART](#alter_move-partition) — переместить партицию/кускок на другой диск или том. - [UPDATE IN PARTITION](#update-in-partition) — обновить данные внутри партиции по условию. - [DELETE IN PARTITION](#delete-in-partition) — удалить данные внутри партиции по условию. @@ -209,29 +209,35 @@ ALTER TABLE 'table_name' UNFREEZE [PARTITION 'part_expr'] WITH NAME 'backup_name Удаляет с диска "замороженные" партиции с указанным именем. Если секция `PARTITION` опущена, запрос удаляет резервную копию всех партиций сразу. -## FETCH PARTITION {#alter_fetch-partition} +## FETCH PARTITION\|PART {#alter_fetch-partition} ``` sql -ALTER TABLE table_name FETCH PARTITION partition_expr FROM 'path-in-zookeeper' +ALTER TABLE table_name FETCH PARTITION|PART partition_expr FROM 'path-in-zookeeper' ``` Загружает партицию с другого сервера. Этот запрос работает только для реплицированных таблиц. Запрос выполняет следующее: -1. Загружает партицию с указанного шарда. Путь к шарду задается в секции `FROM` (‘path-in-zookeeper’). Обратите внимание, нужно задавать путь к шарду в ZooKeeper. +1. Загружает партицию/кусок с указанного шарда. Путь к шарду задается в секции `FROM` (‘path-in-zookeeper’). Обратите внимание, нужно задавать путь к шарду в ZooKeeper. 2. Помещает загруженные данные в директорию `detached` таблицы `table_name`. Чтобы прикрепить эти данные к таблице, используйте запрос [ATTACH PARTITION\|PART](#alter_attach-partition). Например: +1. FETCH PARTITION ``` sql ALTER TABLE users FETCH PARTITION 201902 FROM '/clickhouse/tables/01-01/visits'; ALTER TABLE users ATTACH PARTITION 201902; ``` +2. FETCH PART +``` sql +ALTER TABLE users FETCH PART 201901_2_2_0 FROM '/clickhouse/tables/01-01/visits'; +ALTER TABLE users ATTACH PART 201901_2_2_0; +``` Следует иметь в виду: -- Запрос `ALTER TABLE t FETCH PARTITION` не реплицируется. Он загружает партицию в директорию `detached` только на локальном сервере. +- Запрос `ALTER TABLE t FETCH PARTITION|PART` не реплицируется. Он загружает партицию в директорию `detached` только на локальном сервере. - Запрос `ALTER TABLE t ATTACH` реплицируется — он добавляет данные в таблицу сразу на всех репликах. На одной из реплик данные будут добавлены из директории `detached`, а на других — из соседних реплик. Перед загрузкой данных система проверяет, существует ли партиция и совпадает ли её структура со структурой таблицы. При этом автоматически выбирается наиболее актуальная реплика среди всех живых реплик. diff --git a/docs/ru/sql-reference/statements/grant.md b/docs/ru/sql-reference/statements/grant.md index 05ffaa22bbd..1d8ec3c60b0 100644 --- a/docs/ru/sql-reference/statements/grant.md +++ b/docs/ru/sql-reference/statements/grant.md @@ -282,7 +282,7 @@ GRANT INSERT(x,y) ON db.table TO john - `ALTER MATERIALIZE TTL`. Уровень: `TABLE`. Алиасы: `MATERIALIZE TTL` - `ALTER SETTINGS`. Уровень: `TABLE`. Алиасы: `ALTER SETTING`, `ALTER MODIFY SETTING`, `MODIFY SETTING` - `ALTER MOVE PARTITION`. Уровень: `TABLE`. Алиасы: `ALTER MOVE PART`, `MOVE PARTITION`, `MOVE PART` - - `ALTER FETCH PARTITION`. Уровень: `TABLE`. Алиасы: `FETCH PARTITION` + - `ALTER FETCH PARTITION`. Уровень: `TABLE`. Алиасы: `ALTER FETCH PART`, `FETCH PARTITION`, `FETCH PART` - `ALTER FREEZE PARTITION`. Уровень: `TABLE`. Алиасы: `FREEZE PARTITION` - `ALTER VIEW` Уровень: `GROUP` - `ALTER VIEW REFRESH `. Уровень: `VIEW`. Алиасы: `ALTER LIVE VIEW REFRESH`, `REFRESH VIEW` diff --git a/docs/zh/interfaces/third-party/gui.md b/docs/zh/interfaces/third-party/gui.md index e85f8b2ec79..46baf55d564 100644 --- a/docs/zh/interfaces/third-party/gui.md +++ b/docs/zh/interfaces/third-party/gui.md @@ -57,9 +57,9 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix). - 表格预览。 - 自动完成。 -### ﾂ环板-ｮﾂ嘉ｯﾂ偲 {#clickhouse-cli} +### clickhouse-cli {#clickhouse-cli} -[ﾂ环板-ｮﾂ嘉ｯﾂ偲](https://github.com/hatarist/clickhouse-cli) 是ClickHouse的替代命令行客户端，用Python 3编写。 +[clickhouse-cli](https://github.com/hatarist/clickhouse-cli) 是ClickHouse的替代命令行客户端，用Python 3编写。 特征： @@ -68,15 +68,15 @@ ClickHouse Web 界面 [Tabix](https://github.com/tabixio/tabix). - 寻呼机支持数据输出。 - 自定义PostgreSQL类命令。 -### ﾂ暗ｪﾂ氾环催ﾂ団ﾂ法ﾂ人 {#clickhouse-flamegraph} +### clickhouse-flamegraph {#clickhouse-flamegraph} [clickhouse-flamegraph](https://github.com/Slach/clickhouse-flamegraph) 是一个可视化的专业工具`system.trace_log`如[flamegraph](http://www.brendangregg.com/flamegraphs.html). ## 商业 {#shang-ye} -### ﾂ环板Softwareｮﾂ嘉ｯ {#holistics-software} +### Holistics {#holistics-software} -[整体学](https://www.holistics.io/) 在2019年被Gartner FrontRunners列为可用性最高排名第二的商业智能工具之一。 Holistics是一个基于SQL的全栈数据平台和商业智能工具，用于设置您的分析流程。 +[Holistics](https://www.holistics.io/) 在2019年被Gartner FrontRunners列为可用性最高排名第二的商业智能工具之一。 Holistics是一个基于SQL的全栈数据平台和商业智能工具，用于设置您的分析流程。 特征： diff --git a/docs/zh/sql-reference/table-functions/mysql.md b/docs/zh/sql-reference/table-functions/mysql.md index c54cd7d2a06..3ed0001b0a0 100644 --- a/docs/zh/sql-reference/table-functions/mysql.md +++ b/docs/zh/sql-reference/table-functions/mysql.md @@ -1,13 +1,8 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 42 -toc_title: mysql ---- - # mysql {#mysql} -允许 `SELECT` 要对存储在远程MySQL服务器上的数据执行的查询。 +允许对存储在远程MySQL服务器上的数据执行`SELECT`和`INSERT`查询。 + +**语法** ``` sql mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_duplicate_clause']); @@ -15,31 +10,44 @@ mysql('host:port', 'database', 'table', 'user', 'password'[, replace_query, 'on_ **参数** -- `host:port` — MySQL server address. +- `host:port` — MySQL服务器地址. -- `database` — Remote database name. +- `database` — 远程数据库名称. -- `table` — Remote table name. +- `table` — 远程表名称. -- `user` — MySQL user. +- `user` — MySQL用户. -- `password` — User password. +- `password` — 用户密码. -- `replace_query` — Flag that converts `INSERT INTO` 查询到 `REPLACE INTO`. 如果 `replace_query=1`，查询被替换。 +- `replace_query` — 将INSERT INTO` 查询转换为 `REPLACE INTO`的标志。如果 `replace_query=1`，查询被替换。 -- `on_duplicate_clause` — The `ON DUPLICATE KEY on_duplicate_clause` 表达式被添加到 `INSERT` 查询。 +- `on_duplicate_clause` — 添加 `ON DUPLICATE KEY on_duplicate_clause` 表达式到 `INSERT` 查询。明确规定只能使用 `replace_query = 0` ，如果你同时设置replace_query = 1`和`on_duplicate_clause`，ClickHouse将产生异常。 - Example: `INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1`, where `on_duplicate_clause` is `UPDATE c2 = c2 + 1`. See the MySQL documentation to find which `on_duplicate_clause` you can use with the `ON DUPLICATE KEY` clause. + 示例：`INSERT INTO t (c1,c2) VALUES ('a', 2) ON DUPLICATE KEY UPDATE c2 = c2 + 1` - To specify `on_duplicate_clause` you need to pass `0` to the `replace_query` parameter. If you simultaneously pass `replace_query = 1` and `on_duplicate_clause`, ClickHouse generates an exception. + `on_duplicate_clause`这里是`UPDATE c2 = c2 + 1`。请查阅MySQL文档，来找到可以和`ON DUPLICATE KEY`一起使用的 `on_duplicate_clause`子句。 -简单 `WHERE` 条款如 `=, !=, >, >=, <, <=` 当前在MySQL服务器上执行。 +简单的 `WHERE` 子句如 `=, !=, >, >=, <, <=` 将即时在MySQL服务器上执行。其余的条件和 `LIMIT` 只有在对MySQL的查询完成后，才会在ClickHouse中执行采样约束。 -其余的条件和 `LIMIT` 只有在对MySQL的查询完成后，才会在ClickHouse中执行采样约束。 +支持使用`|`并列进行多副本查询，示例如下： + +```sql +SELECT name FROM mysql(`mysql{1|2|3}:3306`, 'mysql_database', 'mysql_table', 'user', 'password'); +``` + +或 + +```sql +SELECT name FROM mysql(`mysql1:3306|mysql2:3306|mysql3:3306`, 'mysql_database', 'mysql_table', 'user', 'password'); +``` **返回值** -与原始MySQL表具有相同列的table对象。 +与原始MySQL表具有相同列的表对象。 + +!!! note "注意" + 在`INSERT`查询中为了区分`mysql(...)`与带有列名列表的表名的表函数，你必须使用关键字`FUNCTION`或`TABLE FUNCTION`。查看如下示例。 ## 用法示例 {#usage-example} @@ -66,7 +74,7 @@ mysql> select * from test; 1 row in set (0,00 sec) ``` -从ClickHouse中选择数据: +从ClickHouse中查询数据: ``` sql SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123') @@ -78,6 +86,21 @@ SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123') └────────┴──────────────┴───────┴────────────────┘ ``` +替换和插入： + +```sql +INSERT INTO FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 1) (int_id, float) VALUES (1, 3); +INSERT INTO TABLE FUNCTION mysql('localhost:3306', 'test', 'test', 'bayonet', '123', 0, 'UPDATE int_id = int_id + 1') (int_id, float) VALUES (1, 4); +SELECT * FROM mysql('localhost:3306', 'test', 'test', 'bayonet', '123'); +``` + +```text +┌─int_id─┬─float─┐ +│ 1 │ 3 │ +│ 2 │ 4 │ +└────────┴───────┘ +``` + ## 另请参阅 {#see-also} - [该 ‘MySQL’ 表引擎](../../engines/table-engines/integrations/mysql.md) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index c4aef014971..9c1c8338321 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -430,6 +430,7 @@ private: {TokenType::ClosingRoundBracket, Replxx::Color::BROWN}, {TokenType::OpeningSquareBracket, Replxx::Color::BROWN}, {TokenType::ClosingSquareBracket, Replxx::Color::BROWN}, + {TokenType::DoubleColon, Replxx::Color::BROWN}, {TokenType::OpeningCurlyBrace, Replxx::Color::INTENSE}, {TokenType::ClosingCurlyBrace, Replxx::Color::INTENSE}, diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 2633f0e9426..6be7ba1ad73 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -388,24 +388,32 @@ void LocalServer::processQueries() /// Use the same query_id (and thread group) for all queries CurrentThread::QueryScope query_scope_holder(context); - ///Set progress show + /// Set progress show need_render_progress = config().getBool("progress", false); + std::function finalize_progress; if (need_render_progress) { + /// Set progress callback, which can be run from multiple threads. context->setProgressCallback([&](const Progress & value) { /// Write progress only if progress was updated if (progress_indication.updateProgress(value)) progress_indication.writeProgress(); }); + + /// Set finalizing callback for progress, which is called right before finalizing query output. + finalize_progress = [&]() + { + progress_indication.clearProgressOutput(); + }; + + /// Set callback for file processing progress. + progress_indication.setFileProgressCallback(context); } bool echo_queries = config().hasOption("echo") || config().hasOption("verbose"); - if (need_render_progress) - progress_indication.setFileProgressCallback(context); - std::exception_ptr exception; for (const auto & query : queries) @@ -425,7 +433,7 @@ void LocalServer::processQueries() try { - executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {}); + executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {}, finalize_progress); } catch (...) { diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 313523d19dc..d4f830e5a0c 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -477,17 +477,6 @@ int Server::main(const std::vector & /*args*/) CurrentMetrics::set(CurrentMetrics::Revision, ClickHouseRevision::getVersionRevision()); CurrentMetrics::set(CurrentMetrics::VersionInteger, ClickHouseRevision::getVersionInteger()); - if (ThreadFuzzer::instance().isEffective()) - LOG_WARNING(log, "ThreadFuzzer is enabled. Application will run slowly and unstable."); - -#if !defined(NDEBUG) || !defined(__OPTIMIZE__) - LOG_WARNING(log, "Server was built in debug mode. It will work slowly."); -#endif - -#if defined(SANITIZER) - LOG_WARNING(log, "Server was built with sanitizer. It will work slowly."); -#endif - /** Context contains all that query execution is dependent: * settings, available functions, data types, aggregate functions, databases, ... */ @@ -497,6 +486,18 @@ int Server::main(const std::vector & /*args*/) global_context->makeGlobalContext(); global_context->setApplicationType(Context::ApplicationType::SERVER); +#if !defined(NDEBUG) || !defined(__OPTIMIZE__) + global_context->addWarningMessage("Server was built in debug mode. It will work slowly."); +#endif + +if (ThreadFuzzer::instance().isEffective()) + global_context->addWarningMessage("ThreadFuzzer is enabled. Application will run slowly and unstable."); + +#if defined(SANITIZER) + global_context->addWarningMessage("Server was built with sanitizer. It will work slowly."); +#endif + + // Initialize global thread pool. Do it before we fetch configs from zookeeper // nodes (`from_zk`), because ZooKeeper interface uses the pool. We will // ignore `max_thread_pool_size` in configs we fetch from ZK, but oh well. @@ -552,8 +553,10 @@ int Server::main(const std::vector & /*args*/) if (ptrace(PTRACE_TRACEME, 0, nullptr, nullptr) == -1) { /// Program is run under debugger. Modification of it's binary image is ok for breakpoints. - LOG_WARNING(log, "Server is run under debugger and its binary image is modified (most likely with breakpoints).", - calculated_binary_hash); + global_context->addWarningMessage( + fmt::format("Server is run under debugger and its binary image is modified (most likely with breakpoints).", + calculated_binary_hash) + ); } else { @@ -636,7 +639,7 @@ int Server::main(const std::vector & /*args*/) } else { - LOG_WARNING(log, message); + global_context->addWarningMessage(message); } } diff --git a/programs/server/play.html b/programs/server/play.html index 4165a2829bd..5e0377aa8f7 100644 --- a/programs/server/play.html +++ b/programs/server/play.html @@ -9,7 +9,7 @@ Do not use any JavaScript or CSS frameworks or preprocessors. This HTML page should not require any build systems (node.js, npm, gulp, etc.) This HTML page should not be minified, instead it should be reasonably minimalistic by itself. - This HTML page should not load any external resources + This HTML page should not load any external resources on load. (CSS and JavaScript must be embedded directly to the page. No external fonts or images should be loaded). This UI should look as lightweight, clean and fast as possible. All UI elements must be aligned in pixel-perfect way. @@ -343,13 +343,18 @@ /// Save query in history only if it is different. let previous_query = ''; - /// Substitute the address of the server where the page is served. - if (location.protocol != 'file:') { + const current_url = new URL(window.location); + + const server_address = current_url.searchParams.get('url'); + if (server_address) { + document.getElementById('url').value = server_address; + } else if (location.protocol != 'file:') { + /// Substitute the address of the server where the page is served. document.getElementById('url').value = location.origin; } /// Substitute user name if it's specified in the query string - let user_from_url = (new URL(window.location)).searchParams.get('user'); + const user_from_url = current_url.searchParams.get('user'); if (user_from_url) { document.getElementById('user').value = user_from_url; } @@ -361,7 +366,9 @@ let user = document.getElementById('user').value; let password = document.getElementById('password').value; - let url = document.getElementById('url').value + + let server_address = document.getElementById('url').value; + + let url = server_address + /// Ask server to allow cross-domain requests. '?add_http_cors_header=1' + '&user=' + encodeURIComponent(user) + @@ -390,11 +397,18 @@ response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit. }; let title = "ClickHouse Query: " + query; - let url = window.location.pathname + '?user=' + encodeURIComponent(user) + '#' + window.btoa(query); + + let history_url = window.location.pathname + '?user=' + encodeURIComponent(user); + if (server_address != location.origin) { + /// Save server's address in URL if it's not identical to the address of the play UI. + history_url += '&url=' + encodeURIComponent(server_address); + } + history_url += '#' + window.btoa(query); + if (previous_query == '') { - history.replaceState(state, title, url); + history.replaceState(state, title, history_url); } else { - history.pushState(state, title, url); + history.pushState(state, title, history_url); } document.title = title; previous_query = query; diff --git a/src/Access/AccessType.h b/src/Access/AccessType.h index 47153b5ab63..02d7e4982f9 100644 --- a/src/Access/AccessType.h +++ b/src/Access/AccessType.h @@ -173,6 +173,7 @@ enum class AccessType M(MONGO, "", GLOBAL, SOURCES) \ M(MYSQL, "", GLOBAL, SOURCES) \ M(POSTGRES, "", GLOBAL, SOURCES) \ + M(SQLITE, "", GLOBAL, SOURCES) \ M(ODBC, "", GLOBAL, SOURCES) \ M(JDBC, "", GLOBAL, SOURCES) \ M(HDFS, "", GLOBAL, SOURCES) \ diff --git a/src/AggregateFunctions/AggregateFunctionBitwise.h b/src/AggregateFunctions/AggregateFunctionBitwise.h index 5582a200921..90db2469828 100644 --- a/src/AggregateFunctions/AggregateFunctionBitwise.h +++ b/src/AggregateFunctions/AggregateFunctionBitwise.h @@ -9,6 +9,14 @@ #include +#if !defined(ARCADIA_BUILD) +# include +#endif + +#if USE_EMBEDDED_COMPILER +# include +# include +#endif namespace DB { @@ -21,6 +29,21 @@ struct AggregateFunctionGroupBitOrData T value = 0; static const char * name() { return "groupBitOr"; } void update(T x) { value |= x; } + +#if USE_EMBEDDED_COMPILER + + static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr) + { + auto type = toNativeType(builder); + builder.CreateStore(llvm::Constant::getNullValue(type), value_ptr); + } + + static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs) + { + return builder.CreateOr(lhs, rhs); + } + +#endif }; template @@ -29,6 +52,21 @@ struct AggregateFunctionGroupBitAndData T value = -1; /// Two's complement arithmetic, sign extension. static const char * name() { return "groupBitAnd"; } void update(T x) { value &= x; } + +#if USE_EMBEDDED_COMPILER + + static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr) + { + auto type = toNativeType(builder); + builder.CreateStore(llvm::ConstantInt::get(type, -1), value_ptr); + } + + static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs) + { + return builder.CreateAnd(lhs, rhs); + } + +#endif }; template @@ -37,6 +75,21 @@ struct AggregateFunctionGroupBitXorData T value = 0; static const char * name() { return "groupBitXor"; } void update(T x) { value ^= x; } + +#if USE_EMBEDDED_COMPILER + + static void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * value_ptr) + { + auto type = toNativeType(builder); + builder.CreateStore(llvm::Constant::getNullValue(type), value_ptr); + } + + static llvm::Value* compileUpdate(llvm::IRBuilderBase & builder, llvm::Value * lhs, llvm::Value * rhs) + { + return builder.CreateXor(lhs, rhs); + } + +#endif }; @@ -45,7 +98,7 @@ template class AggregateFunctionBitwise final : public IAggregateFunctionDataHelper> { public: - AggregateFunctionBitwise(const DataTypePtr & type) + explicit AggregateFunctionBitwise(const DataTypePtr & type) : IAggregateFunctionDataHelper>({type}, {}) {} String getName() const override { return Data::name(); } @@ -81,6 +134,68 @@ public: { assert_cast &>(to).getData().push_back(this->data(place).value); } + +#if USE_EMBEDDED_COMPILER + + bool isCompilable() const override + { + auto return_type = getReturnType(); + return canBeNativeType(*return_type); + } + + void compileCreate(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override + { + llvm::IRBuilder<> & b = static_cast &>(builder); + + auto * return_type = toNativeType(b, getReturnType()); + auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo()); + Data::compileCreate(builder, value_ptr); + } + + void compileAdd(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr, const DataTypes &, const std::vector & argument_values) const override + { + llvm::IRBuilder<> & b = static_cast &>(builder); + + auto * return_type = toNativeType(b, getReturnType()); + + auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo()); + auto * value = b.CreateLoad(return_type, value_ptr); + + const auto & argument_value = argument_values[0]; + auto * result_value = Data::compileUpdate(builder, value, argument_value); + + b.CreateStore(result_value, value_ptr); + } + + void compileMerge(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_dst_ptr, llvm::Value * aggregate_data_src_ptr) const override + { + llvm::IRBuilder<> & b = static_cast &>(builder); + + auto * return_type = toNativeType(b, getReturnType()); + + auto * value_dst_ptr = b.CreatePointerCast(aggregate_data_dst_ptr, return_type->getPointerTo()); + auto * value_dst = b.CreateLoad(return_type, value_dst_ptr); + + auto * value_src_ptr = b.CreatePointerCast(aggregate_data_src_ptr, return_type->getPointerTo()); + auto * value_src = b.CreateLoad(return_type, value_src_ptr); + + auto * result_value = Data::compileUpdate(builder, value_dst, value_src); + + b.CreateStore(result_value, value_dst_ptr); + } + + llvm::Value * compileGetResult(llvm::IRBuilderBase & builder, llvm::Value * aggregate_data_ptr) const override + { + llvm::IRBuilder<> & b = static_cast &>(builder); + + auto * return_type = toNativeType(b, getReturnType()); + auto * value_ptr = b.CreatePointerCast(aggregate_data_ptr, return_type->getPointerTo()); + + return b.CreateLoad(return_type, value_ptr); + } + +#endif + }; diff --git a/src/AggregateFunctions/AggregateFunctionSum.h b/src/AggregateFunctions/AggregateFunctionSum.h index 4be2455d71e..3355cb0d6fc 100644 --- a/src/AggregateFunctions/AggregateFunctionSum.h +++ b/src/AggregateFunctions/AggregateFunctionSum.h @@ -101,6 +101,24 @@ struct AggregateFunctionSumData { const auto * end = ptr + count; + if constexpr ( + (is_integer_v && !is_big_int_v) + || (IsDecimalNumber && !std::is_same_v && !std::is_same_v)) + { + /// For integers we can vectorize the operation if we replace the null check using a multiplication (by 0 for null, 1 for not null) + /// https://quick-bench.com/q/MLTnfTvwC2qZFVeWHfOBR3U7a8I + T local_sum{}; + while (ptr < end) + { + T multiplier = !*null_map; + Impl::add(local_sum, *ptr * multiplier); + ++ptr; + ++null_map; + } + Impl::add(sum, local_sum); + return; + } + if constexpr (std::is_floating_point_v) { constexpr size_t unroll_count = 128 / sizeof(T); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 272bea4f6d7..31286c740d4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -76,6 +76,10 @@ add_headers_and_sources(clickhouse_common_io IO) add_headers_and_sources(clickhouse_common_io IO/S3) list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp) +if (USE_SQLITE) + add_headers_and_sources(dbms Databases/SQLite) +endif() + if(USE_RDKAFKA) add_headers_and_sources(dbms Storages/Kafka) endif() @@ -415,6 +419,11 @@ if (USE_AWS_S3) target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR}) endif() +if (USE_S2_GEOMETRY) + dbms_target_link_libraries (PUBLIC ${S2_GEOMETRY_LIBRARY}) + dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${S2_GEOMETRY_INCLUDE_DIR}) +endif() + if (USE_BROTLI) target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY}) target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR}) @@ -425,6 +434,10 @@ if (USE_AMQPCPP) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${AMQPCPP_INCLUDE_DIR}) endif() +if (USE_SQLITE) + dbms_target_link_libraries(PUBLIC sqlite) +endif() + if (USE_CASSANDRA) dbms_target_link_libraries(PUBLIC ${CASSANDRA_LIBRARY}) dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 81360c6794b..03ee76240cb 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -298,11 +298,19 @@ void ConfigProcessor::doIncludesRecursive( { const auto * subst = attributes->getNamedItem(attr_name); attr_nodes[attr_name] = subst; - substs_count += static_cast(subst == nullptr); + substs_count += static_cast(subst != nullptr); } - if (substs_count < SUBSTITUTION_ATTRS.size() - 1) /// only one substitution is allowed - throw Poco::Exception("several substitutions attributes set for element <" + node->nodeName() + ">"); + if (substs_count > 1) /// only one substitution is allowed + throw Poco::Exception("More than one substitution attribute is set for element <" + node->nodeName() + ">"); + + if (node->nodeName() == "include") + { + if (node->hasChildNodes()) + throw Poco::Exception(" element must have no children"); + if (substs_count == 0) + throw Poco::Exception("No substitution attributes set for element , must have exactly one"); + } /// Replace the original contents, not add to it. bool replace = attributes->getNamedItem("replace"); @@ -320,37 +328,57 @@ void ConfigProcessor::doIncludesRecursive( else if (throw_on_bad_incl) throw Poco::Exception(error_msg + name); else + { + if (node->nodeName() == "include") + node->parentNode()->removeChild(node); + LOG_WARNING(log, "{}{}", error_msg, name); + } } else { - Element & element = dynamic_cast(*node); - - for (const auto & attr_name : SUBSTITUTION_ATTRS) - element.removeAttribute(attr_name); - - if (replace) + /// Replace the whole node not just contents. + if (node->nodeName() == "include") { - while (Node * child = node->firstChild()) - node->removeChild(child); + const NodeListPtr children = node_to_include->childNodes(); + for (size_t i = 0, size = children->length(); i < size; ++i) + { + NodePtr new_node = config->importNode(children->item(i), true); + node->parentNode()->insertBefore(new_node, node); + } - element.removeAttribute("replace"); + node->parentNode()->removeChild(node); } - - const NodeListPtr children = node_to_include->childNodes(); - for (size_t i = 0, size = children->length(); i < size; ++i) + else { - NodePtr new_node = config->importNode(children->item(i), true); - node->appendChild(new_node); - } + Element & element = dynamic_cast(*node); - const NamedNodeMapPtr from_attrs = node_to_include->attributes(); - for (size_t i = 0, size = from_attrs->length(); i < size; ++i) - { - element.setAttributeNode(dynamic_cast(config->importNode(from_attrs->item(i), true))); - } + for (const auto & attr_name : SUBSTITUTION_ATTRS) + element.removeAttribute(attr_name); - included_something = true; + if (replace) + { + while (Node * child = node->firstChild()) + node->removeChild(child); + + element.removeAttribute("replace"); + } + + const NodeListPtr children = node_to_include->childNodes(); + for (size_t i = 0, size = children->length(); i < size; ++i) + { + NodePtr new_node = config->importNode(children->item(i), true); + node->appendChild(new_node); + } + + const NamedNodeMapPtr from_attrs = node_to_include->attributes(); + for (size_t i = 0, size = from_attrs->length(); i < size; ++i) + { + element.setAttributeNode(dynamic_cast(config->importNode(from_attrs->item(i), true))); + } + + included_something = true; + } } }; diff --git a/src/Common/Config/configReadClient.cpp b/src/Common/Config/configReadClient.cpp index cbe5b3f7bc2..e7bc0b72814 100644 --- a/src/Common/Config/configReadClient.cpp +++ b/src/Common/Config/configReadClient.cpp @@ -10,16 +10,10 @@ namespace fs = std::filesystem; namespace DB { -/// Checks if file exists without throwing an exception but with message in console. -bool safeFsExists(const auto & path) +bool safeFsExists(const String & path) { std::error_code ec; - bool res = fs::exists(path, ec); - if (ec) - { - std::cerr << "Can't check '" << path << "': [" << ec.value() << "] " << ec.message() << std::endl; - } - return res; + return fs::exists(path, ec); }; bool configReadClient(Poco::Util::LayeredConfiguration & config, const std::string & home_path) diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 8301ea656bf..7904d0ac61d 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -558,6 +558,9 @@ M(588, DISTRIBUTED_BROKEN_BATCH_INFO) \ M(589, DISTRIBUTED_BROKEN_BATCH_FILES) \ M(590, CANNOT_SYSCONF) \ + M(591, SQLITE_ENGINE_ERROR) \ + M(592, DATA_ENCRYPTION_ERROR) \ + M(593, ZERO_COPY_REPLICATION_ERROR) \ \ M(998, POSTGRESQL_CONNECTION_FAILURE) \ M(999, KEEPER_EXCEPTION) \ diff --git a/src/Common/HashTable/StringHashTable.h b/src/Common/HashTable/StringHashTable.h index b05d119e0e9..d30271d65db 100644 --- a/src/Common/HashTable/StringHashTable.h +++ b/src/Common/HashTable/StringHashTable.h @@ -237,7 +237,12 @@ public: // 1. Always memcpy 8 times bytes // 2. Use switch case extension to generate fast dispatching table // 3. Funcs are named callables that can be force_inlined + // // NOTE: It relies on Little Endianness + // + // NOTE: It requires padded to 8 bytes keys (IOW you cannot pass + // std::string here, but you can pass i.e. ColumnString::getDataAt()), + // since it copies 8 bytes at a time. template static auto ALWAYS_INLINE dispatch(Self & self, KeyHolder && key_holder, Func && func) { diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index dffe2239e62..915d14466b6 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -22,10 +22,6 @@ M(WriteBufferFromFileDescriptorWrite, "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.") \ M(WriteBufferFromFileDescriptorWriteFailed, "Number of times the write (write/pwrite) to a file descriptor have failed.") \ M(WriteBufferFromFileDescriptorWriteBytes, "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.") \ - M(ReadBufferAIORead, "") \ - M(ReadBufferAIOReadBytes, "") \ - M(WriteBufferAIOWrite, "") \ - M(WriteBufferAIOWriteBytes, "") \ M(ReadCompressedBytes, "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).") \ M(CompressedReadBufferBlocks, "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).") \ M(CompressedReadBufferBytes, "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).") \ @@ -34,6 +30,10 @@ M(UncompressedCacheWeightLost, "") \ M(MMappedFileCacheHits, "") \ M(MMappedFileCacheMisses, "") \ + M(AIOWrite, "Number of writes with Linux or FreeBSD AIO interface") \ + M(AIOWriteBytes, "Number of bytes written with Linux or FreeBSD AIO interface") \ + M(AIORead, "Number of reads with Linux or FreeBSD AIO interface") \ + M(AIOReadBytes, "Number of bytes read with Linux or FreeBSD AIO interface") \ M(IOBufferAllocs, "") \ M(IOBufferAllocBytes, "") \ M(ArenaAllocChunks, "") \ @@ -43,8 +43,8 @@ M(MarkCacheHits, "") \ M(MarkCacheMisses, "") \ M(CreatedReadBufferOrdinary, "") \ - M(CreatedReadBufferAIO, "") \ - M(CreatedReadBufferAIOFailed, "") \ + M(CreatedReadBufferDirectIO, "") \ + M(CreatedReadBufferDirectIOFailed, "") \ M(CreatedReadBufferMMap, "") \ M(CreatedReadBufferMMapFailed, "") \ M(DiskReadElapsedMicroseconds, "Total time spent waiting for read syscall. This include reads from page cache.") \ diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index e1a7c420c54..0d65eaece86 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -4,9 +4,6 @@ #include #include -/// FIXME: progress bar in clickhouse-local needs to be cleared after query execution -/// - same as it is now in clickhouse-client. Also there is no writeFinalProgress call -/// in clickhouse-local. namespace DB { diff --git a/src/Common/ZooKeeper/ZooKeeperCommon.h b/src/Common/ZooKeeper/ZooKeeperCommon.h index a816c1eb8bb..eb7f42f900a 100644 --- a/src/Common/ZooKeeper/ZooKeeperCommon.h +++ b/src/Common/ZooKeeper/ZooKeeperCommon.h @@ -45,6 +45,8 @@ struct ZooKeeperRequest : virtual Request /// If the request was sent and we didn't get the response and the error happens, then we cannot be sure was it processed or not. bool probably_sent = false; + bool restored_from_zookeeper_log = false; + ZooKeeperRequest() = default; ZooKeeperRequest(const ZooKeeperRequest &) = default; virtual ~ZooKeeperRequest() override = default; @@ -172,6 +174,9 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest { + /// used only during restore from zookeeper log + int32_t parent_cversion = -1; + ZooKeeperCreateRequest() = default; explicit ZooKeeperCreateRequest(const CreateRequest & base) : CreateRequest(base) {} @@ -183,9 +188,6 @@ struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest bool isReadRequest() const override { return false; } size_t bytesSize() const override { return CreateRequest::bytesSize() + sizeof(xid) + sizeof(has_watch); } - - /// During recovery from log we don't rehash ACLs - bool need_to_hash_acls = true; }; struct ZooKeeperCreateResponse final : CreateResponse, ZooKeeperResponse @@ -362,8 +364,6 @@ struct ZooKeeperSetACLRequest final : SetACLRequest, ZooKeeperRequest bool isReadRequest() const override { return false; } size_t bytesSize() const override { return SetACLRequest::bytesSize() + sizeof(xid); } - - bool need_to_hash_acls = true; }; struct ZooKeeperSetACLResponse final : SetACLResponse, ZooKeeperResponse diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index e14a1784b14..22ffb74f61a 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -47,13 +47,13 @@ CompressedReadBufferFromFile::CompressedReadBufferFromFile(std::unique_ptr(0) - , p_file_in(createReadBufferFromFileBase(path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size)) + , p_file_in(createReadBufferFromFileBase(path, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache, buf_size)) , file_in(*p_file_in) { compressed_in = &file_in; diff --git a/src/Compression/CompressedReadBufferFromFile.h b/src/Compression/CompressedReadBufferFromFile.h index 2ee7021b35a..fe9add6f015 100644 --- a/src/Compression/CompressedReadBufferFromFile.h +++ b/src/Compression/CompressedReadBufferFromFile.h @@ -33,7 +33,7 @@ public: CompressedReadBufferFromFile(std::unique_ptr buf, bool allow_different_codecs_ = false); CompressedReadBufferFromFile( - const std::string & path, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache, + const std::string & path, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, bool allow_different_codecs_ = false); void seek(size_t offset_in_compressed_file, size_t offset_in_decompressed_block); diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 97c78e04f05..4c3f649a6b6 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -267,13 +267,12 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest } else { - auto & session_auth_ids = storage.session_and_auth[session_id]; KeeperStorage::Node created_node; Coordination::ACLs node_acls; - if (!fixupACL(request.acls, session_auth_ids, node_acls, request.need_to_hash_acls)) + if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log)) { response.error = Coordination::Error::ZINVALIDACL; return {response_ptr, {}}; @@ -307,16 +306,28 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest path_created += seq_num_str.str(); } + int32_t parent_cversion = request.parent_cversion; auto child_path = getBaseName(path_created); int64_t prev_parent_zxid; - container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid] (KeeperStorage::Node & parent) + int32_t prev_parent_cversion; + container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid, + parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent) { + + parent.children.insert(child_path); + prev_parent_cversion = parent.stat.cversion; + prev_parent_zxid = parent.stat.pzxid; + /// Increment sequential number even if node is not sequential ++parent.seq_num; - parent.children.insert(child_path); - ++parent.stat.cversion; - prev_parent_zxid = parent.stat.pzxid; - parent.stat.pzxid = zxid; + + if (parent_cversion == -1) + ++parent.stat.cversion; + else if (parent_cversion > parent.stat.cversion) + parent.stat.cversion = parent_cversion; + + if (zxid > parent.stat.pzxid) + parent.stat.pzxid = zxid; ++parent.stat.numChildren; }); @@ -326,7 +337,7 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest if (request.is_ephemeral) ephemerals[session_id].emplace(path_created); - undo = [&storage, prev_parent_zxid, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] + undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] { storage.container.erase(path_created); storage.acl_map.removeUsage(acl_id); @@ -334,11 +345,11 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest if (is_ephemeral) storage.ephemerals[session_id].erase(path_created); - storage.container.updateValue(parent_path, [child_path, prev_parent_zxid] (KeeperStorage::Node & undo_parent) + storage.container.updateValue(parent_path, [child_path, prev_parent_zxid, prev_parent_cversion] (KeeperStorage::Node & undo_parent) { - --undo_parent.stat.cversion; --undo_parent.stat.numChildren; --undo_parent.seq_num; + undo_parent.stat.cversion = prev_parent_cversion; undo_parent.stat.pzxid = prev_parent_zxid; undo_parent.children.erase(child_path); }); @@ -394,6 +405,24 @@ struct KeeperStorageGetRequest final : public KeeperStorageRequest } }; +namespace +{ + /// Garbage required to apply log to "fuzzy" zookeeper snapshot + void updateParentPzxid(const std::string & child_path, int64_t zxid, KeeperStorage::Container & container) + { + auto parent_path = parentPath(child_path); + auto parent_it = container.find(parent_path); + if (parent_it != container.end()) + { + container.updateValue(parent_path, [zxid](KeeperStorage::Node & parent) + { + if (parent.stat.pzxid < zxid) + parent.stat.pzxid = zxid; + }); + } + } +} + struct KeeperStorageRemoveRequest final : public KeeperStorageRequest { bool checkAuth(KeeperStorage & storage, int64_t session_id) const override @@ -412,7 +441,7 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest } using KeeperStorageRequest::KeeperStorageRequest; - std::pair process(KeeperStorage & storage, int64_t /*zxid*/, int64_t /*session_id*/) const override + std::pair process(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/) const override { auto & container = storage.container; auto & ephemerals = storage.ephemerals; @@ -425,6 +454,8 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest auto it = container.find(request.path); if (it == container.end()) { + if (request.restored_from_zookeeper_log) + updateParentPzxid(request.path, zxid, container); response.error = Coordination::Error::ZNONODE; } else if (request.version != -1 && request.version != it->value.stat.version) @@ -437,6 +468,9 @@ struct KeeperStorageRemoveRequest final : public KeeperStorageRequest } else { + if (request.restored_from_zookeeper_log) + updateParentPzxid(request.path, zxid, container); + auto prev_node = it->value; if (prev_node.stat.ephemeralOwner != 0) { @@ -719,7 +753,7 @@ struct KeeperStorageSetACLRequest final : public KeeperStorageRequest auto & session_auth_ids = storage.session_and_auth[session_id]; Coordination::ACLs node_acls; - if (!fixupACL(request.acls, session_auth_ids, node_acls, request.need_to_hash_acls)) + if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log)) { response.error = Coordination::Error::ZINVALIDACL; return {response_ptr, {}}; diff --git a/src/Coordination/ZooKeeperDataReader.cpp b/src/Coordination/ZooKeeperDataReader.cpp index 8bcce25cfee..cf644110786 100644 --- a/src/Coordination/ZooKeeperDataReader.cpp +++ b/src/Coordination/ZooKeeperDataReader.cpp @@ -174,7 +174,22 @@ void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::st LOG_INFO(log, "Deserializing data from snapshot"); int64_t zxid_from_nodes = deserializeStorageData(storage, reader, log); - storage.zxid = std::max(zxid, zxid_from_nodes); + /// In ZooKeeper Snapshots can contain inconsistent state of storage. They call + /// this inconsistent state "fuzzy". So it's guaranteed that snapshot contain all + /// records up to zxid from snapshot name and also some records for future. + /// But it doesn't mean that we have just some state of storage from future (like zxid + 100 log records). + /// We have incorrect state of storage where some random log entries from future were applied.... + /// + /// In ZooKeeper they say that their transactions log is idempotent and can be applied to "fuzzy" state as is. + /// It's true but there is no any general invariant which produces this property. They just have ad-hoc "if's" which detects + /// "fuzzy" state inconsistencies and apply log records in special way. Several examples: + /// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L453-L463 + /// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L476-L480 + /// https://github.com/apache/zookeeper/blob/master/zookeeper-server/src/main/java/org/apache/zookeeper/server/DataTree.java#L547-L549 + if (zxid_from_nodes > zxid) + LOG_WARNING(log, "ZooKeeper snapshot was in inconsistent (fuzzy) state. Will try to apply log. ZooKeeper create non fuzzy snapshot with restart. You can just restart ZooKeeper server and get consistent version."); + + storage.zxid = zxid; LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid); } @@ -210,16 +225,18 @@ void deserializeLogMagic(ReadBuffer & in) static constexpr int32_t LOG_HEADER = 1514884167; /// "ZKLG" if (magic_header != LOG_HEADER) - throw Exception(ErrorCodes::CORRUPTED_DATA ,"Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header); + throw Exception(ErrorCodes::CORRUPTED_DATA, "Incorrect magic header in file, expected {}, got {}", LOG_HEADER, magic_header); if (version != 2) - throw Exception(ErrorCodes::NOT_IMPLEMENTED,"Cannot deserialize ZooKeeper data other than version 2, got version {}", version); + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot deserialize ZooKeeper data other than version 2, got version {}", version); } -/// For some reason zookeeper stores slightly different records in log then -/// requests. For example: -/// class CreateTxn { +/// ZooKeeper transactions log differs from requests. The main reason: to store records in log +/// in some "finalized" state (for example with concrete versions). +/// +/// Example: +/// class CreateTxn { /// ustring path; /// buffer data; /// vector acl; @@ -289,10 +306,9 @@ Coordination::ZooKeeperRequestPtr deserializeCreateTxn(ReadBuffer & in) Coordination::read(result->data, in); Coordination::read(result->acls, in); Coordination::read(result->is_ephemeral, in); - result->need_to_hash_acls = false; - /// How we should use it? It should just increment on request execution - int32_t parent_c_version; - Coordination::read(parent_c_version, in); + Coordination::read(result->parent_cversion, in); + + result->restored_from_zookeeper_log = true; return result; } @@ -300,6 +316,7 @@ Coordination::ZooKeeperRequestPtr deserializeDeleteTxn(ReadBuffer & in) { std::shared_ptr result = std::make_shared(); Coordination::read(result->path, in); + result->restored_from_zookeeper_log = true; return result; } @@ -309,6 +326,7 @@ Coordination::ZooKeeperRequestPtr deserializeSetTxn(ReadBuffer & in) Coordination::read(result->path, in); Coordination::read(result->data, in); Coordination::read(result->version, in); + result->restored_from_zookeeper_log = true; /// It stores version + 1 (which should be, not for request) result->version -= 1; @@ -320,6 +338,7 @@ Coordination::ZooKeeperRequestPtr deserializeCheckVersionTxn(ReadBuffer & in) std::shared_ptr result = std::make_shared(); Coordination::read(result->path, in); Coordination::read(result->version, in); + result->restored_from_zookeeper_log = true; return result; } @@ -329,14 +348,19 @@ Coordination::ZooKeeperRequestPtr deserializeCreateSession(ReadBuffer & in) int32_t timeout; Coordination::read(timeout, in); result->session_timeout_ms = timeout; + result->restored_from_zookeeper_log = true; return result; } -Coordination::ZooKeeperRequestPtr deserializeCloseSession(ReadBuffer & in) +Coordination::ZooKeeperRequestPtr deserializeCloseSession(ReadBuffer & in, bool empty) { std::shared_ptr result = std::make_shared(); - std::vector data; - Coordination::read(data, in); + if (!empty) + { + std::vector data; + Coordination::read(data, in); + } + result->restored_from_zookeeper_log = true; return result; } @@ -356,14 +380,14 @@ Coordination::ZooKeeperRequestPtr deserializeSetACLTxn(ReadBuffer & in) Coordination::read(result->version, in); /// It stores version + 1 (which should be, not for request) result->version -= 1; - result->need_to_hash_acls = false; + result->restored_from_zookeeper_log = true; return result; } Coordination::ZooKeeperRequestPtr deserializeMultiTxn(ReadBuffer & in); -Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtxn) +Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtxn, int64_t txn_length = 0) { int32_t type; Coordination::read(type, in); @@ -372,6 +396,11 @@ Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtx if (subtxn) Coordination::read(sub_txn_length, in); + bool empty_txn = !subtxn && txn_length == 32; /// Possible for old-style CloseTxn's + + if (empty_txn && type != -11) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Empty non close-session transaction found"); + int64_t in_count_before = in.count(); switch (type) @@ -398,7 +427,7 @@ Coordination::ZooKeeperRequestPtr deserializeTxnImpl(ReadBuffer & in, bool subtx result = deserializeCreateSession(in); break; case -11: - result = deserializeCloseSession(in); + result = deserializeCloseSession(in, empty_txn); break; case -1: result = deserializeErrorTxn(in); @@ -442,7 +471,7 @@ bool hasErrorsInMultiRequest(Coordination::ZooKeeperRequestPtr request) if (request == nullptr) return true; - for (const auto & subrequest : dynamic_cast(request.get())->requests) //-V522 + for (const auto & subrequest : dynamic_cast(request.get())->requests) // -V522 if (subrequest == nullptr) return true; return false; @@ -470,7 +499,7 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * /*l int64_t time; Coordination::read(time, in); - Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false); + Coordination::ZooKeeperRequestPtr request = deserializeTxnImpl(in, false, txn_len); /// Skip all other bytes int64_t bytes_read = in.count() - count_before; diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index fa78f052f37..7a735569238 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -436,7 +436,7 @@ Block Block::sortColumns() const Block sorted_block; /// std::unordered_map (index_by_name) cannot be used to guarantee the sort order - std::vector sorted_index_by_name(index_by_name.size()); + std::vector sorted_index_by_name(index_by_name.size()); { size_t i = 0; for (auto it = index_by_name.begin(); it != index_by_name.end(); ++it) diff --git a/src/Core/Block.h b/src/Core/Block.h index a21bd290571..fb94a205bf5 100644 --- a/src/Core/Block.h +++ b/src/Core/Block.h @@ -68,7 +68,7 @@ public: const_cast(this)->findByName(name)); } - const ColumnWithTypeAndName* findByName(const std::string & name) const; + const ColumnWithTypeAndName * findByName(const std::string & name) const; ColumnWithTypeAndName & getByName(const std::string & name) { diff --git a/src/Core/MySQL/MySQLClient.cpp b/src/Core/MySQL/MySQLClient.cpp index 3650818c543..d103ea873e5 100644 --- a/src/Core/MySQL/MySQLClient.cpp +++ b/src/Core/MySQL/MySQLClient.cpp @@ -26,13 +26,14 @@ namespace ErrorCodes MySQLClient::MySQLClient(const String & host_, UInt16 port_, const String & user_, const String & password_) : host(host_), port(port_), user(user_), password(std::move(password_)) { - client_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_PLUGIN_AUTH | CLIENT_SECURE_CONNECTION; + mysql_context.client_capabilities = CLIENT_PROTOCOL_41 | CLIENT_PLUGIN_AUTH | CLIENT_SECURE_CONNECTION; } MySQLClient::MySQLClient(MySQLClient && other) : host(std::move(other.host)), port(other.port), user(std::move(other.user)), password(std::move(other.password)) - , client_capability_flags(other.client_capability_flags) + , mysql_context(other.mysql_context) { + mysql_context.sequence_id = 0; } void MySQLClient::connect() @@ -56,7 +57,7 @@ void MySQLClient::connect() in = std::make_shared(*socket); out = std::make_shared(*socket); - packet_endpoint = std::make_shared(*in, *out, seq); + packet_endpoint = mysql_context.makeEndpoint(*in, *out); handshake(); } @@ -68,7 +69,7 @@ void MySQLClient::disconnect() socket->close(); socket = nullptr; connected = false; - seq = 0; + mysql_context.sequence_id = 0; } /// https://dev.mysql.com/doc/internals/en/connection-phase-packets.html @@ -87,10 +88,10 @@ void MySQLClient::handshake() String auth_plugin_data = native41.getAuthPluginData(); HandshakeResponse handshake_response( - client_capability_flags, MAX_PACKET_LENGTH, charset_utf8, user, "", auth_plugin_data, mysql_native_password); + mysql_context.client_capabilities, MAX_PACKET_LENGTH, charset_utf8, user, "", auth_plugin_data, mysql_native_password); packet_endpoint->sendPacket(handshake_response, true); - ResponsePacket packet_response(client_capability_flags, true); + ResponsePacket packet_response(mysql_context.client_capabilities, true); packet_endpoint->receivePacket(packet_response); packet_endpoint->resetSequenceId(); @@ -105,7 +106,7 @@ void MySQLClient::writeCommand(char command, String query) WriteCommand write_command(command, query); packet_endpoint->sendPacket(write_command, true); - ResponsePacket packet_response(client_capability_flags); + ResponsePacket packet_response(mysql_context.client_capabilities); packet_endpoint->receivePacket(packet_response); switch (packet_response.getType()) { @@ -124,7 +125,7 @@ void MySQLClient::registerSlaveOnMaster(UInt32 slave_id) RegisterSlave register_slave(slave_id); packet_endpoint->sendPacket(register_slave, true); - ResponsePacket packet_response(client_capability_flags); + ResponsePacket packet_response(mysql_context.client_capabilities); packet_endpoint->receivePacket(packet_response); packet_endpoint->resetSequenceId(); if (packet_response.getType() == PACKET_ERR) diff --git a/src/Core/MySQL/MySQLClient.h b/src/Core/MySQL/MySQLClient.h index e503c985584..6144b14690d 100644 --- a/src/Core/MySQL/MySQLClient.h +++ b/src/Core/MySQL/MySQLClient.h @@ -45,9 +45,7 @@ private: String password; bool connected = false; - UInt32 client_capability_flags = 0; - - uint8_t seq = 0; + MySQLWireContext mysql_context; const UInt8 charset_utf8 = 33; const String mysql_native_password = "mysql_native_password"; diff --git a/src/Core/MySQL/PacketEndpoint.cpp b/src/Core/MySQL/PacketEndpoint.cpp index 0bc5c585516..fa1d60034d2 100644 --- a/src/Core/MySQL/PacketEndpoint.cpp +++ b/src/Core/MySQL/PacketEndpoint.cpp @@ -68,4 +68,15 @@ String PacketEndpoint::packetToText(const String & payload) } + +MySQLProtocol::PacketEndpointPtr MySQLWireContext::makeEndpoint(WriteBuffer & out) +{ + return MySQLProtocol::PacketEndpoint::create(out, sequence_id); +} + +MySQLProtocol::PacketEndpointPtr MySQLWireContext::makeEndpoint(ReadBuffer & in, WriteBuffer & out) +{ + return MySQLProtocol::PacketEndpoint::create(in, out, sequence_id); +} + } diff --git a/src/Core/MySQL/PacketEndpoint.h b/src/Core/MySQL/PacketEndpoint.h index d027934eafb..3aa76ac93de 100644 --- a/src/Core/MySQL/PacketEndpoint.h +++ b/src/Core/MySQL/PacketEndpoint.h @@ -5,6 +5,7 @@ #include "IMySQLReadPacket.h" #include "IMySQLWritePacket.h" #include "IO/MySQLPacketPayloadReadBuffer.h" +#include namespace DB { @@ -15,19 +16,13 @@ namespace MySQLProtocol /* Writes and reads packets, keeping sequence-id. * Throws ProtocolError, if packet with incorrect sequence-id was received. */ -class PacketEndpoint +class PacketEndpoint : public shared_ptr_helper { public: uint8_t & sequence_id; ReadBuffer * in; WriteBuffer * out; - /// For writing. - PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_); - - /// For reading and writing. - PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_); - MySQLPacketPayloadReadBuffer getPayload(); void receivePacket(IMySQLReadPacket & packet); @@ -48,8 +43,29 @@ public: /// Converts packet to text. Is used for debug output. static String packetToText(const String & payload); + +protected: + /// For writing. + PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_); + + /// For reading and writing. + PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_); + + friend struct shared_ptr_helper; +}; + +using PacketEndpointPtr = std::shared_ptr; + +} + +struct MySQLWireContext +{ + uint8_t sequence_id = 0; + uint32_t client_capabilities = 0; + size_t max_packet_size = 0; + + MySQLProtocol::PacketEndpointPtr makeEndpoint(WriteBuffer & out); + MySQLProtocol::PacketEndpointPtr makeEndpoint(ReadBuffer & in, WriteBuffer & out); }; } - -} diff --git a/src/Core/PostgreSQL/Connection.cpp b/src/Core/PostgreSQL/Connection.cpp index c423d75981e..e5c61c19963 100644 --- a/src/Core/PostgreSQL/Connection.cpp +++ b/src/Core/PostgreSQL/Connection.cpp @@ -1,4 +1,7 @@ #include "Connection.h" + +#if USE_LIBPQXX + #include namespace postgres @@ -72,3 +75,5 @@ void Connection::connect() updateConnection(); } } + +#endif diff --git a/src/Core/PostgreSQL/Connection.h b/src/Core/PostgreSQL/Connection.h index e01de419c17..681681a38bf 100644 --- a/src/Core/PostgreSQL/Connection.h +++ b/src/Core/PostgreSQL/Connection.h @@ -1,5 +1,11 @@ #pragma once +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_LIBPQXX + #include // Y_IGNORE #include #include @@ -45,3 +51,5 @@ private: Poco::Logger * log; }; } + +#endif diff --git a/src/Core/PostgreSQL/ConnectionHolder.h b/src/Core/PostgreSQL/ConnectionHolder.h index 98ab7df182d..cbdde7062b5 100644 --- a/src/Core/PostgreSQL/ConnectionHolder.h +++ b/src/Core/PostgreSQL/ConnectionHolder.h @@ -1,5 +1,11 @@ #pragma once +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_LIBPQXX + #include // Y_IGNORE #include #include @@ -35,3 +41,5 @@ private: using ConnectionHolderPtr = std::unique_ptr; } + +#endif diff --git a/src/Core/PostgreSQL/PoolWithFailover.cpp b/src/Core/PostgreSQL/PoolWithFailover.cpp index 6bf756b8a12..b8b8e78396c 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.cpp +++ b/src/Core/PostgreSQL/PoolWithFailover.cpp @@ -1,4 +1,7 @@ #include "PoolWithFailover.h" + +#if USE_LIBPQXX + #include "Utils.h" #include #include @@ -136,3 +139,5 @@ ConnectionHolderPtr PoolWithFailover::get() throw DB::Exception(DB::ErrorCodes::POSTGRESQL_CONNECTION_FAILURE, "Unable to connect to any of the replicas"); } } + +#endif diff --git a/src/Core/PostgreSQL/PoolWithFailover.h b/src/Core/PostgreSQL/PoolWithFailover.h index f4ae2c6cd1b..9150262e242 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.h +++ b/src/Core/PostgreSQL/PoolWithFailover.h @@ -1,5 +1,12 @@ #pragma once +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_LIBPQXX + + #include "ConnectionHolder.h" #include #include @@ -63,3 +70,5 @@ private: using PoolWithFailoverPtr = std::shared_ptr; } + +#endif diff --git a/src/Core/PostgreSQL/Utils.cpp b/src/Core/PostgreSQL/Utils.cpp index 98e76da99d2..ebfdacd0fea 100644 --- a/src/Core/PostgreSQL/Utils.cpp +++ b/src/Core/PostgreSQL/Utils.cpp @@ -1,4 +1,7 @@ #include "Utils.h" + +#if USE_LIBPQXX + #include namespace postgres @@ -17,3 +20,5 @@ ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, S } } + +#endif diff --git a/src/Core/PostgreSQL/Utils.h b/src/Core/PostgreSQL/Utils.h index 34d66fefb70..4a58fcffb9a 100644 --- a/src/Core/PostgreSQL/Utils.h +++ b/src/Core/PostgreSQL/Utils.h @@ -1,5 +1,11 @@ #pragma once +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_LIBPQXX + #include // Y_IGNORE #include #include "Connection.h" @@ -15,3 +21,5 @@ namespace postgres { ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, String user, String password); } + +#endif diff --git a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp index a72c6205cd5..0de75ad00a1 100644 --- a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp +++ b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -99,7 +100,16 @@ void insertPostgreSQLValue( assert_cast(column).insertValue(time); break; } - case ExternalResultDescription::ValueType::vtDateTime64:[[fallthrough]]; + case ExternalResultDescription::ValueType::vtDateTime64: + { + ReadBufferFromString in(value); + DateTime64 time = 0; + readDateTime64Text(time, 6, in, assert_cast(data_type.get())->getTimeZone()); + if (time < 0) + time = 0; + assert_cast &>(column).insertValue(time); + break; + } case ExternalResultDescription::ValueType::vtDecimal32: [[fallthrough]]; case ExternalResultDescription::ValueType::vtDecimal64: [[fallthrough]]; case ExternalResultDescription::ValueType::vtDecimal128: [[fallthrough]]; @@ -201,6 +211,18 @@ void preparePostgreSQLArrayInfo( ReadBufferFromString in(field); time_t time = 0; readDateTimeText(time, in, assert_cast(nested.get())->getTimeZone()); + if (time < 0) + time = 0; + return time; + }; + else if (which.isDateTime64()) + parser = [nested](std::string & field) -> Field + { + ReadBufferFromString in(field); + DateTime64 time = 0; + readDateTime64Text(time, 6, in, assert_cast(nested.get())->getTimeZone()); + if (time < 0) + time = 0; return time; }; else if (which.isDecimal32()) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 28e46160a98..77e6d0c674a 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -108,7 +108,7 @@ class IColumn; M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \ M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \ M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \ - M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \ + M(UInt64, min_count_to_compile_aggregate_expression, 0, "The number of identical aggregate expressions before they are JIT-compiled", 0) \ M(UInt64, group_by_two_level_threshold, 100000, "From what number of keys, a two-level aggregation starts. 0 - the threshold is not set.", 0) \ M(UInt64, group_by_two_level_threshold_bytes, 50000000, "From what size of the aggregation state in bytes, a two-level aggregation begins to be used. 0 - the threshold is not set. Two-level aggregation is used when at least one of the thresholds is triggered.", 0) \ M(Bool, distributed_aggregation_memory_efficient, true, "Is the memory-saving mode of distributed aggregation enabled.", 0) \ @@ -482,6 +482,8 @@ class IColumn; M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \ M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \ \ + M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function 'range' per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \ + \ /** Experimental functions */ \ M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ \ diff --git a/src/Core/config_core.h.in b/src/Core/config_core.h.in index e250e013913..45cbc6efe19 100644 --- a/src/Core/config_core.h.in +++ b/src/Core/config_core.h.in @@ -13,5 +13,6 @@ #cmakedefine01 USE_LDAP #cmakedefine01 USE_ROCKSDB #cmakedefine01 USE_LIBPQXX +#cmakedefine01 USE_SQLITE #cmakedefine01 USE_NURAFT #cmakedefine01 USE_KRB5 diff --git a/src/Core/ya.make b/src/Core/ya.make index d1e352ee846..6946d7a47bb 100644 --- a/src/Core/ya.make +++ b/src/Core/ya.make @@ -31,6 +31,10 @@ SRCS( MySQL/PacketsProtocolText.cpp MySQL/PacketsReplication.cpp NamesAndTypes.cpp + PostgreSQL/Connection.cpp + PostgreSQL/PoolWithFailover.cpp + PostgreSQL/Utils.cpp + PostgreSQL/insertPostgreSQLValue.cpp PostgreSQLProtocol.cpp QueryProcessingStage.cpp Settings.cpp diff --git a/src/DataStreams/SQLiteBlockInputStream.cpp b/src/DataStreams/SQLiteBlockInputStream.cpp new file mode 100644 index 00000000000..da7645d968d --- /dev/null +++ b/src/DataStreams/SQLiteBlockInputStream.cpp @@ -0,0 +1,163 @@ +#include "SQLiteBlockInputStream.h" + +#if USE_SQLITE +#include +#include +#include + +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SQLITE_ENGINE_ERROR; +} + +SQLiteBlockInputStream::SQLiteBlockInputStream( + SQLitePtr sqlite_db_, + const String & query_str_, + const Block & sample_block, + const UInt64 max_block_size_) + : query_str(query_str_) + , max_block_size(max_block_size_) + , sqlite_db(std::move(sqlite_db_)) +{ + description.init(sample_block); +} + + +void SQLiteBlockInputStream::readPrefix() +{ + sqlite3_stmt * compiled_stmt = nullptr; + int status = sqlite3_prepare_v2(sqlite_db.get(), query_str.c_str(), query_str.size() + 1, &compiled_stmt, nullptr); + + if (status != SQLITE_OK) + throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR, + "Cannot prepate sqlite statement. Status: {}. Message: {}", + status, sqlite3_errstr(status)); + + compiled_statement = std::unique_ptr(compiled_stmt, StatementDeleter()); +} + + +Block SQLiteBlockInputStream::readImpl() +{ + if (!compiled_statement) + return Block(); + + MutableColumns columns = description.sample_block.cloneEmptyColumns(); + size_t num_rows = 0; + + while (true) + { + int status = sqlite3_step(compiled_statement.get()); + + if (status == SQLITE_BUSY) + { + continue; + } + else if (status == SQLITE_DONE) + { + compiled_statement.reset(); + break; + } + else if (status != SQLITE_ROW) + { + throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR, + "Expected SQLITE_ROW status, but got status {}. Error: {}, Message: {}", + status, sqlite3_errstr(status), sqlite3_errmsg(sqlite_db.get())); + } + + int column_count = sqlite3_column_count(compiled_statement.get()); + for (const auto idx : collections::range(0, column_count)) + { + const auto & sample = description.sample_block.getByPosition(idx); + + if (sqlite3_column_type(compiled_statement.get(), idx) == SQLITE_NULL) + { + insertDefaultSQLiteValue(*columns[idx], *sample.column); + continue; + } + + if (description.types[idx].second) + { + ColumnNullable & column_nullable = assert_cast(*columns[idx]); + insertValue(column_nullable.getNestedColumn(), description.types[idx].first, idx); + column_nullable.getNullMapData().emplace_back(0); + } + else + { + insertValue(*columns[idx], description.types[idx].first, idx); + } + } + + if (++num_rows == max_block_size) + break; + } + + return description.sample_block.cloneWithColumns(std::move(columns)); +} + + +void SQLiteBlockInputStream::readSuffix() +{ + if (compiled_statement) + compiled_statement.reset(); +} + + +void SQLiteBlockInputStream::insertValue(IColumn & column, const ExternalResultDescription::ValueType type, size_t idx) +{ + switch (type) + { + case ValueType::vtUInt8: + assert_cast(column).insertValue(sqlite3_column_int(compiled_statement.get(), idx)); + break; + case ValueType::vtUInt16: + assert_cast(column).insertValue(sqlite3_column_int(compiled_statement.get(), idx)); + break; + case ValueType::vtUInt32: + assert_cast(column).insertValue(sqlite3_column_int64(compiled_statement.get(), idx)); + break; + case ValueType::vtUInt64: + /// There is no uint64 in sqlite3, only int and int64 + assert_cast(column).insertValue(sqlite3_column_int64(compiled_statement.get(), idx)); + break; + case ValueType::vtInt8: + assert_cast(column).insertValue(sqlite3_column_int(compiled_statement.get(), idx)); + break; + case ValueType::vtInt16: + assert_cast(column).insertValue(sqlite3_column_int(compiled_statement.get(), idx)); + break; + case ValueType::vtInt32: + assert_cast(column).insertValue(sqlite3_column_int(compiled_statement.get(), idx)); + break; + case ValueType::vtInt64: + assert_cast(column).insertValue(sqlite3_column_int64(compiled_statement.get(), idx)); + break; + case ValueType::vtFloat32: + assert_cast(column).insertValue(sqlite3_column_double(compiled_statement.get(), idx)); + break; + case ValueType::vtFloat64: + assert_cast(column).insertValue(sqlite3_column_double(compiled_statement.get(), idx)); + break; + default: + const char * data = reinterpret_cast(sqlite3_column_text(compiled_statement.get(), idx)); + int len = sqlite3_column_bytes(compiled_statement.get(), idx); + assert_cast(column).insertData(data, len); + break; + } +} + +} + +#endif diff --git a/src/DataStreams/SQLiteBlockInputStream.h b/src/DataStreams/SQLiteBlockInputStream.h new file mode 100644 index 00000000000..35fc4801b4b --- /dev/null +++ b/src/DataStreams/SQLiteBlockInputStream.h @@ -0,0 +1,62 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_SQLITE +#include +#include + +#include // Y_IGNORE + + +namespace DB +{ +class SQLiteBlockInputStream : public IBlockInputStream +{ +using SQLitePtr = std::shared_ptr; + +public: + SQLiteBlockInputStream(SQLitePtr sqlite_db_, + const String & query_str_, + const Block & sample_block, + UInt64 max_block_size_); + + String getName() const override { return "SQLite"; } + + Block getHeader() const override { return description.sample_block.cloneEmpty(); } + +private: + void insertDefaultSQLiteValue(IColumn & column, const IColumn & sample_column) + { + column.insertFrom(sample_column, 0); + } + + using ValueType = ExternalResultDescription::ValueType; + + struct StatementDeleter + { + void operator()(sqlite3_stmt * stmt) { sqlite3_finalize(stmt); } + }; + + void readPrefix() override; + + Block readImpl() override; + + void readSuffix() override; + + void insertValue(IColumn & column, const ExternalResultDescription::ValueType type, size_t idx); + + String query_str; + UInt64 max_block_size; + + ExternalResultDescription description; + + SQLitePtr sqlite_db; + std::unique_ptr compiled_statement; +}; + +} + +#endif diff --git a/src/DataStreams/ya.make b/src/DataStreams/ya.make index 29e6eb3afc3..e6534ebc2f7 100644 --- a/src/DataStreams/ya.make +++ b/src/DataStreams/ya.make @@ -41,6 +41,7 @@ SRCS( RemoteBlockOutputStream.cpp RemoteQueryExecutor.cpp RemoteQueryExecutorReadContext.cpp + SQLiteBlockInputStream.cpp SizeLimits.cpp SquashingBlockInputStream.cpp SquashingBlockOutputStream.cpp diff --git a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp index a68dc30d5c2..41ba81814d0 100644 --- a/src/DataTypes/DataTypeLowCardinalityHelpers.cpp +++ b/src/DataTypes/DataTypeLowCardinalityHelpers.cpp @@ -1,11 +1,13 @@ #include #include #include +#include #include #include #include #include +#include #include @@ -39,6 +41,11 @@ DataTypePtr recursiveRemoveLowCardinality(const DataTypePtr & type) return std::make_shared(elements); } + if (const auto * map_type = typeid_cast(type.get())) + { + return std::make_shared(recursiveRemoveLowCardinality(map_type->getKeyType()), recursiveRemoveLowCardinality(map_type->getValueType())); + } + if (const auto * low_cardinality_type = typeid_cast(type.get())) return low_cardinality_type->getDictionaryType(); @@ -78,6 +85,16 @@ ColumnPtr recursiveRemoveLowCardinality(const ColumnPtr & column) return ColumnTuple::create(columns); } + if (const auto * column_map = typeid_cast(column.get())) + { + const auto & nested = column_map->getNestedColumnPtr(); + auto nested_no_lc = recursiveRemoveLowCardinality(nested); + if (nested.get() == nested_no_lc.get()) + return column; + + return ColumnMap::create(nested_no_lc); + } + if (const auto * column_low_cardinality = typeid_cast(column.get())) return column_low_cardinality->convertToFullColumn(); diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index 3f2d9987018..8fd375aa86e 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -53,12 +54,24 @@ DataTypeMap::DataTypeMap(const DataTypePtr & key_type_, const DataTypePtr & valu void DataTypeMap::assertKeyType() const { - if (!key_type->isValueRepresentedByInteger() + bool type_error = false; + if (key_type->getTypeId() == TypeIndex::LowCardinality) + { + const auto & low_cardinality_data_type = assert_cast(*key_type); + if (!isStringOrFixedString(*(low_cardinality_data_type.getDictionaryType()))) + type_error = true; + } + else if (!key_type->isValueRepresentedByInteger() && !isStringOrFixedString(*key_type) && !WhichDataType(key_type).isNothing() && !WhichDataType(key_type).isUUID()) + { + type_error = true; + } + + if (type_error) throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Type of Map key must be a type, that can be represented by integer or string or UUID," + "Type of Map key must be a type, that can be represented by integer or String or FixedString (possibly LowCardinality) or UUID," " but {} given", key_type->getName()); } diff --git a/src/DataTypes/EnumValues.h b/src/DataTypes/EnumValues.h index d03a8867e42..1e5e4f55ea7 100644 --- a/src/DataTypes/EnumValues.h +++ b/src/DataTypes/EnumValues.h @@ -42,11 +42,23 @@ public: return it; } + /// throws exception if value is not valid const StringRef & getNameForValue(const T & value) const { return findByValue(value)->second; } + /// returns false if value is not valid + bool getNameForValue(const T & value, StringRef & result) const + { + const auto it = value_to_name_map.find(value); + if (it == std::end(value_to_name_map)) + return false; + + result = it->second; + return true; + } + T getValue(StringRef field_name, bool try_treat_as_id = false) const; template diff --git a/src/DataTypes/Native.h b/src/DataTypes/Native.h index 88f99b60ed7..970b70f9f0b 100644 --- a/src/DataTypes/Native.h +++ b/src/DataTypes/Native.h @@ -29,7 +29,7 @@ namespace ErrorCodes static inline bool typeIsSigned(const IDataType & type) { WhichDataType data_type(type); - return data_type.isNativeInt() || data_type.isFloat(); + return data_type.isNativeInt() || data_type.isFloat() || data_type.isEnum(); } static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDataType & type) @@ -57,6 +57,10 @@ static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const IDa return builder.getFloatTy(); else if (data_type.isFloat64()) return builder.getDoubleTy(); + else if (data_type.isEnum8()) + return builder.getInt8Ty(); + else if (data_type.isEnum16()) + return builder.getInt16Ty(); return nullptr; } @@ -109,7 +113,7 @@ static inline bool canBeNativeType(const IDataType & type) return canBeNativeType(*data_type_nullable.getNestedType()); } - return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate(); + return data_type.isNativeInt() || data_type.isNativeUInt() || data_type.isFloat() || data_type.isDate() || data_type.isEnum(); } static inline llvm::Type * toNativeType(llvm::IRBuilderBase & builder, const DataTypePtr & type) @@ -266,7 +270,7 @@ static inline llvm::Constant * getColumnNativeValue(llvm::IRBuilderBase & builde { return llvm::ConstantInt::get(type, column.getUInt(index)); } - else if (column_data_type.isNativeInt()) + else if (column_data_type.isNativeInt() || column_data_type.isEnum()) { return llvm::ConstantInt::get(type, column.getInt(index)); } diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 48b923c4756..6a1914bf046 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -1,17 +1,17 @@ #include #include -#include #include #include #include #include +#include +#include #include #include #include #include #include -#include #include #include @@ -40,6 +40,10 @@ #include #endif +#if USE_SQLITE +#include +#endif + namespace fs = std::filesystem; namespace DB @@ -100,7 +104,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String const UUID & uuid = create.uuid; bool engine_may_have_arguments = engine_name == "MySQL" || engine_name == "MaterializeMySQL" || engine_name == "Lazy" || - engine_name == "Replicated" || engine_name == "PostgreSQL" || engine_name == "MaterializedPostgreSQL"; + engine_name == "Replicated" || engine_name == "PostgreSQL" || engine_name == "MaterializedPostgreSQL" || engine_name == "SQLite"; if (engine_define->engine->arguments && !engine_may_have_arguments) throw Exception("Database engine " + engine_name + " cannot have arguments", ErrorCodes::BAD_ARGUMENTS); @@ -299,6 +303,22 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String } +#endif + +#if USE_SQLITE + else if (engine_name == "SQLite") + { + const ASTFunction * engine = engine_define->engine; + + if (!engine->arguments || engine->arguments->children.size() != 1) + throw Exception("SQLite database requires 1 argument: database path", ErrorCodes::BAD_ARGUMENTS); + + const auto & arguments = engine->arguments->children; + + String database_path = safeGetLiteralValue(arguments[0], "SQLite"); + + return std::make_shared(context, engine_define, database_path); + } #endif throw Exception("Unknown database engine: " + engine_name, ErrorCodes::UNKNOWN_DATABASE_ENGINE); diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index 64d47720af9..a5eccc817d0 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include #include @@ -71,7 +71,7 @@ static DataTypePtr convertPostgreSQLDataType(String & type, const std::function< else if (type == "bigserial") res = std::make_shared(); else if (type.starts_with("timestamp")) - res = std::make_shared(); + res = std::make_shared(6); else if (type == "date") res = std::make_shared(); else if (type.starts_with("numeric")) diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp new file mode 100644 index 00000000000..f8e31517f77 --- /dev/null +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -0,0 +1,215 @@ +#include "DatabaseSQLite.h" + +#if USE_SQLITE + +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SQLITE_ENGINE_ERROR; + extern const int UNKNOWN_TABLE; +} + +DatabaseSQLite::DatabaseSQLite( + ContextPtr context_, + const ASTStorage * database_engine_define_, + const String & database_path_) + : IDatabase("SQLite") + , WithContext(context_->getGlobalContext()) + , database_engine_define(database_engine_define_->clone()) + , log(&Poco::Logger::get("DatabaseSQLite")) +{ + sqlite_db = openSQLiteDB(database_path_, context_); +} + + +bool DatabaseSQLite::empty() const +{ + std::lock_guard lock(mutex); + return fetchTablesList().empty(); +} + + +DatabaseTablesIteratorPtr DatabaseSQLite::getTablesIterator(ContextPtr local_context, const IDatabase::FilterByNameFunction &) +{ + std::lock_guard lock(mutex); + + Tables tables; + auto table_names = fetchTablesList(); + for (const auto & table_name : table_names) + tables[table_name] = fetchTable(table_name, local_context, true); + + return std::make_unique(tables, database_name); +} + + +std::unordered_set DatabaseSQLite::fetchTablesList() const +{ + std::unordered_set tables; + std::string query = "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"; + + auto callback_get_data = [](void * res, int col_num, char ** data_by_col, char ** /* col_names */) -> int + { + for (int i = 0; i < col_num; ++i) + static_cast *>(res)->insert(data_by_col[i]); + return 0; + }; + + char * err_message = nullptr; + int status = sqlite3_exec(sqlite_db.get(), query.c_str(), callback_get_data, &tables, &err_message); + if (status != SQLITE_OK) + { + String err_msg(err_message); + sqlite3_free(err_message); + throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR, + "Cannot fetch sqlite database tables. Error status: {}. Message: {}", + status, err_msg); + } + + return tables; +} + + +bool DatabaseSQLite::checkSQLiteTable(const String & table_name) const +{ + const String query = fmt::format("SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';", table_name); + + auto callback_get_data = [](void * res, int, char **, char **) -> int + { + *(static_cast(res)) += 1; + return 0; + }; + + int count = 0; + char * err_message = nullptr; + int status = sqlite3_exec(sqlite_db.get(), query.c_str(), callback_get_data, &count, &err_message); + if (status != SQLITE_OK) + { + String err_msg(err_message); + sqlite3_free(err_message); + throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR, + "Cannot check sqlite table. Error status: {}. Message: {}", + status, err_msg); + } + + return (count != 0); +} + + +bool DatabaseSQLite::isTableExist(const String & table_name, ContextPtr) const +{ + std::lock_guard lock(mutex); + return checkSQLiteTable(table_name); +} + + +StoragePtr DatabaseSQLite::tryGetTable(const String & table_name, ContextPtr local_context) const +{ + std::lock_guard lock(mutex); + return fetchTable(table_name, local_context, false); +} + + +StoragePtr DatabaseSQLite::fetchTable(const String & table_name, ContextPtr local_context, bool table_checked) const +{ + if (!table_checked && !checkSQLiteTable(table_name)) + return StoragePtr{}; + + auto columns = fetchSQLiteTableStructure(sqlite_db.get(), table_name); + + if (!columns) + return StoragePtr{}; + + auto storage = StorageSQLite::create( + StorageID(database_name, table_name), + sqlite_db, + table_name, + ColumnsDescription{*columns}, + ConstraintsDescription{}, + local_context); + + return storage; +} + + +ASTPtr DatabaseSQLite::getCreateDatabaseQuery() const +{ + const auto & create_query = std::make_shared(); + create_query->database = getDatabaseName(); + create_query->set(create_query->storage, database_engine_define); + return create_query; +} + + +ASTPtr DatabaseSQLite::getCreateTableQueryImpl(const String & table_name, ContextPtr local_context, bool throw_on_error) const +{ + auto storage = fetchTable(table_name, local_context, false); + if (!storage) + { + if (throw_on_error) + throw Exception(ErrorCodes::UNKNOWN_TABLE, "SQLite table {}.{} does not exist", + database_name, table_name); + return nullptr; + } + + auto create_table_query = std::make_shared(); + auto table_storage_define = database_engine_define->clone(); + create_table_query->set(create_table_query->storage, table_storage_define); + + auto columns_declare_list = std::make_shared(); + auto columns_expression_list = std::make_shared(); + + columns_declare_list->set(columns_declare_list->columns, columns_expression_list); + create_table_query->set(create_table_query->columns_list, columns_declare_list); + + /// init create query. + auto table_id = storage->getStorageID(); + create_table_query->table = table_id.table_name; + create_table_query->database = table_id.database_name; + + auto metadata_snapshot = storage->getInMemoryMetadataPtr(); + for (const auto & column_type_and_name : metadata_snapshot->getColumns().getOrdinary()) + { + const auto & column_declaration = std::make_shared(); + column_declaration->name = column_type_and_name.name; + column_declaration->type = getColumnDeclaration(column_type_and_name.type); + columns_expression_list->children.emplace_back(column_declaration); + } + + ASTStorage * ast_storage = table_storage_define->as(); + ASTs storage_children = ast_storage->children; + auto storage_engine_arguments = ast_storage->engine->arguments; + + /// Add table_name to engine arguments + storage_engine_arguments->children.insert(storage_engine_arguments->children.begin() + 1, std::make_shared(table_id.table_name)); + + return create_table_query; +} + + +ASTPtr DatabaseSQLite::getColumnDeclaration(const DataTypePtr & data_type) const +{ + WhichDataType which(data_type); + + if (which.isNullable()) + return makeASTFunction("Nullable", getColumnDeclaration(typeid_cast(data_type.get())->getNestedType())); + + return std::make_shared(data_type->getName()); +} + +} + +#endif diff --git a/src/Databases/SQLite/DatabaseSQLite.h b/src/Databases/SQLite/DatabaseSQLite.h new file mode 100644 index 00000000000..35b1200f397 --- /dev/null +++ b/src/Databases/SQLite/DatabaseSQLite.h @@ -0,0 +1,65 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_SQLITE +#include +#include +#include + +#include // Y_IGNORE + + +namespace DB +{ +class DatabaseSQLite final : public IDatabase, protected WithContext +{ +public: + using SQLitePtr = std::shared_ptr; + + DatabaseSQLite(ContextPtr context_, const ASTStorage * database_engine_define_, const String & database_path_); + + String getEngineName() const override { return "SQLite"; } + + bool canContainMergeTreeTables() const override { return false; } + + bool canContainDistributedTables() const override { return false; } + + bool shouldBeEmptyOnDetach() const override { return false; } + + bool isTableExist(const String & name, ContextPtr context) const override; + + StoragePtr tryGetTable(const String & name, ContextPtr context) const override; + + DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name) override; + + bool empty() const override; + + ASTPtr getCreateDatabaseQuery() const override; + + void shutdown() override {} + +protected: + ASTPtr getCreateTableQueryImpl(const String & table_name, ContextPtr context, bool throw_on_error) const override; + +private: + ASTPtr database_engine_define; + + SQLitePtr sqlite_db; + + Poco::Logger * log; + + bool checkSQLiteTable(const String & table_name) const; + + NameSet fetchTablesList() const; + + StoragePtr fetchTable(const String & table_name, ContextPtr context, bool table_checked) const; + + ASTPtr getColumnDeclaration(const DataTypePtr & data_type) const; +}; + +} + +#endif diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp new file mode 100644 index 00000000000..f3568673acb --- /dev/null +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -0,0 +1,57 @@ +#include "SQLiteUtils.h" + +#if USE_SQLITE +#include + +namespace fs = std::filesystem; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int PATH_ACCESS_DENIED; +} + + +String validateSQLiteDatabasePath(const String & path, const String & user_files_path) +{ + String canonical_user_files_path = fs::canonical(user_files_path); + + String canonical_path; + std::error_code err; + + if (fs::path(path).is_relative()) + canonical_path = fs::canonical(fs::path(user_files_path) / path, err); + else + canonical_path = fs::canonical(path, err); + + if (err) + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "SQLite database path '{}' is invalid. Error: {}", path, err.message()); + + if (!canonical_path.starts_with(canonical_user_files_path)) + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, + "SQLite database file path '{}' must be inside 'user_files' directory", path); + + return canonical_path; +} + + +SQLitePtr openSQLiteDB(const String & database_path, ContextPtr context) +{ + auto validated_path = validateSQLiteDatabasePath(database_path, context->getUserFilesPath()); + + sqlite3 * tmp_sqlite_db = nullptr; + int status = sqlite3_open(validated_path.c_str(), &tmp_sqlite_db); + + if (status != SQLITE_OK) + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, + "Cannot access sqlite database. Error status: {}. Message: {}", + status, sqlite3_errstr(status)); + + return std::shared_ptr(tmp_sqlite_db, sqlite3_close); +} + +} + +#endif diff --git a/src/Databases/SQLite/SQLiteUtils.h b/src/Databases/SQLite/SQLiteUtils.h new file mode 100644 index 00000000000..d405e869b85 --- /dev/null +++ b/src/Databases/SQLite/SQLiteUtils.h @@ -0,0 +1,22 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_SQLITE +#include +#include +#include + + +namespace DB +{ + +using SQLitePtr = std::shared_ptr; + +SQLitePtr openSQLiteDB(const String & database_path, ContextPtr context); + +} + +#endif diff --git a/src/Databases/SQLite/fetchSQLiteTableStructure.cpp b/src/Databases/SQLite/fetchSQLiteTableStructure.cpp new file mode 100644 index 00000000000..c4acf5b3a3a --- /dev/null +++ b/src/Databases/SQLite/fetchSQLiteTableStructure.cpp @@ -0,0 +1,104 @@ +#include + +#if USE_SQLITE + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int SQLITE_ENGINE_ERROR; +} + +static DataTypePtr convertSQLiteDataType(String type) +{ + DataTypePtr res; + type = Poco::toLower(type); + + if (type == "tinyint") + res = std::make_shared(); + else if (type == "smallint") + res = std::make_shared(); + else if (type.starts_with("int") || type == "mediumint") + res = std::make_shared(); + else if (type == "bigint") + res = std::make_shared(); + else if (type == "float") + res = std::make_shared(); + else if (type.starts_with("double") || type == "real") + res = std::make_shared(); + else + res = std::make_shared(); // No decimal when fetching data through API + + return res; +} + + +std::shared_ptr fetchSQLiteTableStructure(sqlite3 * connection, const String & sqlite_table_name) +{ + auto columns = NamesAndTypesList(); + auto query = fmt::format("pragma table_info({});", quoteString(sqlite_table_name)); + + auto callback_get_data = [](void * res, int col_num, char ** data_by_col, char ** col_names) -> int + { + NameAndTypePair name_and_type; + bool is_nullable = false; + + for (int i = 0; i < col_num; ++i) + { + if (strcmp(col_names[i], "name") == 0) + { + name_and_type.name = data_by_col[i]; + } + else if (strcmp(col_names[i], "type") == 0) + { + name_and_type.type = convertSQLiteDataType(data_by_col[i]); + } + else if (strcmp(col_names[i], "notnull") == 0) + { + is_nullable = (data_by_col[i][0] == '0'); + } + } + + if (is_nullable) + name_and_type.type = std::make_shared(name_and_type.type); + + static_cast(res)->push_back(name_and_type); + + return 0; + }; + + char * err_message = nullptr; + int status = sqlite3_exec(connection, query.c_str(), callback_get_data, &columns, &err_message); + + if (status != SQLITE_OK) + { + String err_msg(err_message); + sqlite3_free(err_message); + + throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR, + "Failed to fetch SQLite data. Status: {}. Message: {}", + status, err_msg); + } + + if (columns.empty()) + return nullptr; + + return std::make_shared(columns); +} + +} + +#endif diff --git a/src/Databases/SQLite/fetchSQLiteTableStructure.h b/src/Databases/SQLite/fetchSQLiteTableStructure.h new file mode 100644 index 00000000000..80f50173e5e --- /dev/null +++ b/src/Databases/SQLite/fetchSQLiteTableStructure.h @@ -0,0 +1,19 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_SQLITE + +#include +#include // Y_IGNORE + + +namespace DB +{ +std::shared_ptr fetchSQLiteTableStructure(sqlite3 * connection, + const String & sqlite_table_name); +} + +#endif diff --git a/src/Databases/ya.make b/src/Databases/ya.make index 15c14ac5fc2..7c5e310f7b4 100644 --- a/src/Databases/ya.make +++ b/src/Databases/ya.make @@ -27,6 +27,9 @@ SRCS( MySQL/MaterializeMetadata.cpp MySQL/MaterializeMySQLSettings.cpp MySQL/MaterializeMySQLSyncThread.cpp + SQLite/DatabaseSQLite.cpp + SQLite/SQLiteUtils.cpp + SQLite/fetchSQLiteTableStructure.cpp ) diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index 7232d2d01b7..395328a904d 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -26,8 +26,10 @@ namespace ProfileEvents { extern const Event FileOpen; - extern const Event WriteBufferAIOWrite; - extern const Event WriteBufferAIOWriteBytes; + extern const Event AIOWrite; + extern const Event AIOWriteBytes; + extern const Event AIORead; + extern const Event AIOReadBytes; } namespace DB @@ -531,8 +533,8 @@ public: auto bytes_written = eventResult(event); - ProfileEvents::increment(ProfileEvents::WriteBufferAIOWrite); - ProfileEvents::increment(ProfileEvents::WriteBufferAIOWriteBytes, bytes_written); + ProfileEvents::increment(ProfileEvents::AIOWrite); + ProfileEvents::increment(ProfileEvents::AIOWriteBytes, bytes_written); if (bytes_written != static_cast(block_size * buffer_size_in_blocks)) throw Exception(ErrorCodes::AIO_WRITE_ERROR, @@ -600,6 +602,9 @@ public: buffer_size_in_bytes, read_bytes); + ProfileEvents::increment(ProfileEvents::AIORead); + ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes); + SSDCacheBlock block(block_size); for (size_t i = 0; i < blocks_length; ++i) @@ -687,6 +692,9 @@ public: throw Exception(ErrorCodes::AIO_READ_ERROR, "GC: AIO failed to read file ({}). Expected bytes ({}). Actual bytes ({})", file_path, block_size, read_bytes); + ProfileEvents::increment(ProfileEvents::AIORead); + ProfileEvents::increment(ProfileEvents::AIOReadBytes, read_bytes); + char * request_buffer = getRequestBuffer(request); // Unpoison the memory returned from an uninstrumented system function. diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index d5b82edb134..f672376841e 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -90,17 +90,17 @@ DiskCacheWrapper::readFile( const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const { if (!cache_file_predicate(path)) - return DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + return DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache); LOG_DEBUG(log, "Read file {} from cache", backQuote(path)); if (cache_disk->exists(path)) - return cache_disk->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + return cache_disk->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache); auto metadata = acquireDownloadMetadata(path); @@ -134,7 +134,7 @@ DiskCacheWrapper::readFile( auto tmp_path = path + ".tmp"; { - auto src_buffer = DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + auto src_buffer = DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache); auto dst_buffer = cache_disk->writeFile(tmp_path, buf_size, WriteMode::Rewrite); copyData(*src_buffer, *dst_buffer); } @@ -158,9 +158,9 @@ DiskCacheWrapper::readFile( } if (metadata->status == DOWNLOADED) - return cache_disk->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + return cache_disk->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache); - return DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + return DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache); } std::unique_ptr diff --git a/src/Disks/DiskCacheWrapper.h b/src/Disks/DiskCacheWrapper.h index 6d58394640f..7e711dd521c 100644 --- a/src/Disks/DiskCacheWrapper.h +++ b/src/Disks/DiskCacheWrapper.h @@ -38,7 +38,7 @@ public: const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const override; diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index d1ff3f9f827..58059dbe355 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -115,9 +115,9 @@ void DiskDecorator::listFiles(const String & path, std::vector & file_na std::unique_ptr DiskDecorator::readFile( - const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const + const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const { - return delegate->readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + return delegate->readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache); } std::unique_ptr @@ -206,9 +206,9 @@ void DiskDecorator::startup() delegate->startup(); } -void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) +void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) { - delegate->applyNewSettings(config, context); + delegate->applyNewSettings(config, context, config_prefix, map); } } diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index 401078e6b2e..6586675d1de 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -39,7 +39,7 @@ public: const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const override; @@ -65,11 +65,12 @@ public: String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); } bool checkUniqueId(const String & id) const override { return delegate->checkUniqueId(id); } DiskType::Type getType() const override { return delegate->getType(); } + bool supportZeroCopyReplication() const override { return delegate->supportZeroCopyReplication(); } void onFreeze(const String & path) override; SyncGuardPtr getDirectorySyncGuard(const String & path) const override; void shutdown() override; void startup() override; - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) override; + void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override; protected: Executor & getExecutor() override; diff --git a/src/Disks/DiskEncrypted.cpp b/src/Disks/DiskEncrypted.cpp new file mode 100644 index 00000000000..cec033ef465 --- /dev/null +++ b/src/Disks/DiskEncrypted.cpp @@ -0,0 +1,201 @@ +#include + +#if USE_SSL +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int INCORRECT_DISK_INDEX; + extern const int UNKNOWN_ELEMENT_IN_CONFIG; + extern const int LOGICAL_ERROR; +} + +using DiskEncryptedPtr = std::shared_ptr; +using namespace FileEncryption; + +class DiskEncryptedReservation : public IReservation +{ +public: + DiskEncryptedReservation(DiskEncryptedPtr disk_, std::unique_ptr reservation_) + : disk(std::move(disk_)), reservation(std::move(reservation_)) + { + } + + UInt64 getSize() const override { return reservation->getSize(); } + + DiskPtr getDisk(size_t i) const override + { + if (i != 0) + throw Exception("Can't use i != 0 with single disk reservation", ErrorCodes::INCORRECT_DISK_INDEX); + return disk; + } + + Disks getDisks() const override { return {disk}; } + + void update(UInt64 new_size) override { reservation->update(new_size); } + +private: + DiskEncryptedPtr disk; + std::unique_ptr reservation; +}; + +ReservationPtr DiskEncrypted::reserve(UInt64 bytes) +{ + auto reservation = delegate->reserve(bytes); + if (!reservation) + return {}; + return std::make_unique(std::static_pointer_cast(shared_from_this()), std::move(reservation)); +} + +DiskEncrypted::DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_) + : DiskDecorator(disk_) + , name(name_), key(key_), disk_path(path_) + , disk_absolute_path(delegate->getPath() + disk_path) +{ + initialize(); +} + +void DiskEncrypted::initialize() +{ + // use wrapped_disk as an EncryptedDisk store + if (disk_path.empty()) + return; + + if (disk_path.back() != '/') + throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); + + delegate->createDirectories(disk_path); +} + +std::unique_ptr DiskEncrypted::readFile( + const String & path, + size_t buf_size, + size_t estimated_size, + size_t aio_threshold, + size_t mmap_threshold, + MMappedFileCache * mmap_cache) const +{ + auto wrapped_path = wrappedPath(path); + auto buffer = delegate->readFile(wrapped_path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + + String iv; + size_t offset = 0; + + if (exists(path) && getFileSize(path)) + { + iv = readIV(kIVSize, *buffer); + offset = kIVSize; + } + else + iv = randomString(kIVSize); + + return std::make_unique(buf_size, std::move(buffer), iv, key, offset); +} + +std::unique_ptr DiskEncrypted::writeFile(const String & path, size_t buf_size, WriteMode mode) +{ + String iv; + size_t start_offset = 0; + auto wrapped_path = wrappedPath(path); + + if (mode == WriteMode::Append && exists(path) && getFileSize(path)) + { + auto read_buffer = delegate->readFile(wrapped_path, kIVSize); + iv = readIV(kIVSize, *read_buffer); + start_offset = getFileSize(path); + } + else + iv = randomString(kIVSize); + + auto buffer = delegate->writeFile(wrapped_path, buf_size, mode); + return std::make_unique(buf_size, std::move(buffer), iv, key, start_offset); +} + + +size_t DiskEncrypted::getFileSize(const String & path) const +{ + auto wrapped_path = wrappedPath(path); + size_t size = delegate->getFileSize(wrapped_path); + return size > kIVSize ? (size - kIVSize) : 0; +} + +void DiskEncrypted::truncateFile(const String & path, size_t size) +{ + auto wrapped_path = wrappedPath(path); + delegate->truncateFile(wrapped_path, size ? (size + kIVSize) : 0); +} + +SyncGuardPtr DiskEncrypted::getDirectorySyncGuard(const String & path) const +{ + auto wrapped_path = wrappedPath(path); + return delegate->getDirectorySyncGuard(wrapped_path); +} + +void DiskEncrypted::applyNewSettings( + const Poco::Util::AbstractConfiguration & config, + ContextPtr /*context*/, + const String & config_prefix, + const DisksMap & map) +{ + String wrapped_disk_name = config.getString(config_prefix + ".disk", ""); + if (wrapped_disk_name.empty()) + throw Exception("The wrapped disk name can not be empty. An encrypted disk is a wrapper over another disk. " + "Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + + key = config.getString(config_prefix + ".key", ""); + if (key.empty()) + throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + + auto wrapped_disk = map.find(wrapped_disk_name); + if (wrapped_disk == map.end()) + throw Exception("The wrapped disk must have been announced earlier. No disk with name " + wrapped_disk_name + ". Disk " + name, + ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + delegate = wrapped_disk->second; + + disk_path = config.getString(config_prefix + ".path", ""); + initialize(); +} + +void registerDiskEncrypted(DiskFactory & factory) +{ + auto creator = [](const String & name, + const Poco::Util::AbstractConfiguration & config, + const String & config_prefix, + ContextPtr /*context*/, + const DisksMap & map) -> DiskPtr { + + String wrapped_disk_name = config.getString(config_prefix + ".disk", ""); + if (wrapped_disk_name.empty()) + throw Exception("The wrapped disk name can not be empty. An encrypted disk is a wrapper over another disk. " + "Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + + String key = config.getString(config_prefix + ".key", ""); + if (key.empty()) + throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + if (key.size() != cipherKeyLength(defaultCipher())) + throw Exception("Expected key with size " + std::to_string(cipherKeyLength(defaultCipher())) + ", got key with size " + std::to_string(key.size()), + ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + + auto wrapped_disk = map.find(wrapped_disk_name); + if (wrapped_disk == map.end()) + throw Exception("The wrapped disk must have been announced earlier. No disk with name " + wrapped_disk_name + ". Disk " + name, + ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + + String relative_path = config.getString(config_prefix + ".path", ""); + + return std::make_shared(name, wrapped_disk->second, key, relative_path); + }; + factory.registerDiskType("encrypted", creator); +} + +} + + +#endif diff --git a/src/Disks/DiskEncrypted.h b/src/Disks/DiskEncrypted.h new file mode 100644 index 00000000000..0a38765a791 --- /dev/null +++ b/src/Disks/DiskEncrypted.h @@ -0,0 +1,229 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include +#endif + +#if USE_SSL +#include +#include + + +namespace DB +{ +class ReadBufferFromFileBase; +class WriteBufferFromFileBase; + +class DiskEncrypted : public DiskDecorator +{ +public: + DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_); + + const String & getName() const override { return name; } + const String & getPath() const override { return disk_absolute_path; } + + ReservationPtr reserve(UInt64 bytes) override; + + bool exists(const String & path) const override + { + auto wrapped_path = wrappedPath(path); + return delegate->exists(wrapped_path); + } + + bool isFile(const String & path) const override + { + auto wrapped_path = wrappedPath(path); + return delegate->isFile(wrapped_path); + } + + bool isDirectory(const String & path) const override + { + auto wrapped_path = wrappedPath(path); + return delegate->isDirectory(wrapped_path); + } + + size_t getFileSize(const String & path) const override; + + void createDirectory(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->createDirectory(wrapped_path); + } + + void createDirectories(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->createDirectories(wrapped_path); + } + + + void clearDirectory(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->clearDirectory(wrapped_path); + } + + void moveDirectory(const String & from_path, const String & to_path) override + { + auto wrapped_from_path = wrappedPath(from_path); + auto wrapped_to_path = wrappedPath(to_path); + delegate->moveDirectory(wrapped_from_path, wrapped_to_path); + } + + DiskDirectoryIteratorPtr iterateDirectory(const String & path) override + { + auto wrapped_path = wrappedPath(path); + return delegate->iterateDirectory(wrapped_path); + } + + void createFile(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->createFile(wrapped_path); + } + + void moveFile(const String & from_path, const String & to_path) override + { + auto wrapped_from_path = wrappedPath(from_path); + auto wrapped_to_path = wrappedPath(to_path); + delegate->moveFile(wrapped_from_path, wrapped_to_path); + } + + void replaceFile(const String & from_path, const String & to_path) override + { + auto wrapped_from_path = wrappedPath(from_path); + auto wrapped_to_path = wrappedPath(to_path); + delegate->replaceFile(wrapped_from_path, wrapped_to_path); + } + + void listFiles(const String & path, std::vector & file_names) override + { + auto wrapped_path = wrappedPath(path); + delegate->listFiles(wrapped_path, file_names); + } + + void copy(const String & from_path, const std::shared_ptr & to_disk, const String & to_path) override + { + IDisk::copy(from_path, to_disk, to_path); + } + + std::unique_ptr readFile( + const String & path, + size_t buf_size, + size_t estimated_size, + size_t aio_threshold, + size_t mmap_threshold, + MMappedFileCache * mmap_cache) const override; + + std::unique_ptr writeFile( + const String & path, + size_t buf_size, + WriteMode mode) override; + + void removeFile(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->removeFile(wrapped_path); + } + + void removeFileIfExists(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->removeFileIfExists(wrapped_path); + } + + void removeDirectory(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->removeDirectory(wrapped_path); + } + + void removeRecursive(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->removeRecursive(wrapped_path); + } + + void removeSharedFile(const String & path, bool flag) override + { + auto wrapped_path = wrappedPath(path); + delegate->removeSharedFile(wrapped_path, flag); + } + + void removeSharedRecursive(const String & path, bool flag) override + { + auto wrapped_path = wrappedPath(path); + delegate->removeSharedRecursive(wrapped_path, flag); + } + + void removeSharedFileIfExists(const String & path, bool flag) override + { + auto wrapped_path = wrappedPath(path); + delegate->removeSharedFileIfExists(wrapped_path, flag); + } + + void setLastModified(const String & path, const Poco::Timestamp & timestamp) override + { + auto wrapped_path = wrappedPath(path); + delegate->setLastModified(wrapped_path, timestamp); + } + + Poco::Timestamp getLastModified(const String & path) override + { + auto wrapped_path = wrappedPath(path); + return delegate->getLastModified(wrapped_path); + } + + void setReadOnly(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->setReadOnly(wrapped_path); + } + + void createHardLink(const String & src_path, const String & dst_path) override + { + auto wrapped_src_path = wrappedPath(src_path); + auto wrapped_dst_path = wrappedPath(dst_path); + delegate->createHardLink(wrapped_src_path, wrapped_dst_path); + } + + void truncateFile(const String & path, size_t size) override; + + String getUniqueId(const String & path) const override + { + auto wrapped_path = wrappedPath(path); + return delegate->getUniqueId(wrapped_path); + } + + void onFreeze(const String & path) override + { + auto wrapped_path = wrappedPath(path); + delegate->onFreeze(wrapped_path); + } + + void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override; + + DiskType::Type getType() const override { return DiskType::Type::Encrypted; } + + SyncGuardPtr getDirectorySyncGuard(const String & path) const override; + +private: + void initialize(); + + String wrappedPath(const String & path) const + { + // if path starts_with disk_path -> got already wrapped path + if (!disk_path.empty() && path.starts_with(disk_path)) + return path; + return disk_path + path; + } + + String name; + String key; + String disk_path; + String disk_absolute_path; +}; + +} + +#endif diff --git a/src/Disks/DiskFactory.cpp b/src/Disks/DiskFactory.cpp index b0fb0bd7ca7..94175c92de6 100644 --- a/src/Disks/DiskFactory.cpp +++ b/src/Disks/DiskFactory.cpp @@ -24,7 +24,8 @@ DiskPtr DiskFactory::create( const String & name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - ContextPtr context) const + ContextPtr context, + const DisksMap & map) const { const auto disk_type = config.getString(config_prefix + ".type", "local"); @@ -33,7 +34,7 @@ DiskPtr DiskFactory::create( throw Exception{"DiskFactory: the disk '" + name + "' has unknown disk type: " + disk_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG}; const auto & disk_creator = found->second; - return disk_creator(name, config, config_prefix, context); + return disk_creator(name, config, config_prefix, context, map); } } diff --git a/src/Disks/DiskFactory.h b/src/Disks/DiskFactory.h index 1c05c8d0335..7fcac8928c8 100644 --- a/src/Disks/DiskFactory.h +++ b/src/Disks/DiskFactory.h @@ -8,12 +8,14 @@ #include #include +#include #include namespace DB { +using DisksMap = std::map; /** * Disk factory. Responsible for creating new disk objects. */ @@ -24,7 +26,8 @@ public: const String & name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - ContextPtr context)>; + ContextPtr context, + const DisksMap & map)>; static DiskFactory & instance(); @@ -34,7 +37,8 @@ public: const String & name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - ContextPtr context) const; + ContextPtr context, + const DisksMap & map) const; private: using DiskTypeRegistry = std::unordered_map; diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index 89c1514f5c8..a723803cd88 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -211,9 +211,9 @@ void DiskLocal::replaceFile(const String & from_path, const String & to_path) std::unique_ptr DiskLocal::readFile( - const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const + const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const { - return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, aio_threshold, mmap_threshold, mmap_cache, buf_size); + return createReadBufferFromFileBase(fs::path(disk_path) / path, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache, buf_size); } std::unique_ptr @@ -367,7 +367,8 @@ void registerDiskLocal(DiskFactory & factory) auto creator = [](const String & name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - ContextPtr context) -> DiskPtr { + ContextPtr context, + const DisksMap & /*map*/) -> DiskPtr { String path = config.getString(config_prefix + ".path", ""); if (name == "default") { diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index 47482ad8d67..3aa243b103b 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -74,7 +74,7 @@ public: const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const override; @@ -100,6 +100,8 @@ public: DiskType::Type getType() const override { return DiskType::Type::Local; } + bool supportZeroCopyReplication() const override { return false; } + SyncGuardPtr getDirectorySyncGuard(const String & path) const override; private: diff --git a/src/Disks/DiskMemory.cpp b/src/Disks/DiskMemory.cpp index 77926b4e375..337b9784080 100644 --- a/src/Disks/DiskMemory.cpp +++ b/src/Disks/DiskMemory.cpp @@ -450,7 +450,8 @@ void registerDiskMemory(DiskFactory & factory) auto creator = [](const String & name, const Poco::Util::AbstractConfiguration & /*config*/, const String & /*config_prefix*/, - ContextPtr /*context*/) -> DiskPtr { return std::make_shared(name); }; + ContextPtr /*context*/, + const DisksMap & /*map*/) -> DiskPtr { return std::make_shared(name); }; factory.registerDiskType("memory", creator); } diff --git a/src/Disks/DiskMemory.h b/src/Disks/DiskMemory.h index d5c57b20a4a..d168bc26ff3 100644 --- a/src/Disks/DiskMemory.h +++ b/src/Disks/DiskMemory.h @@ -66,7 +66,7 @@ public: const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const override; @@ -92,6 +92,8 @@ public: DiskType::Type getType() const override { return DiskType::Type::RAM; } + bool supportZeroCopyReplication() const override { return false; } + private: void createDirectoriesImpl(const String & path); void replaceFileImpl(const String & from_path, const String & to_path); diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index 2600dc5a1e1..1bd5b2acf50 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -187,11 +187,11 @@ void DiskRestartProxy::listFiles(const String & path, std::vector & file } std::unique_ptr DiskRestartProxy::readFile( - const String & path, size_t buf_size, size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) + const String & path, size_t buf_size, size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const { ReadLock lock (mutex); - auto impl = DiskDecorator::readFile(path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); + auto impl = DiskDecorator::readFile(path, buf_size, estimated_size, direct_io_threshold, mmap_threshold, mmap_cache); return std::make_unique(*this, std::move(impl)); } diff --git a/src/Disks/DiskRestartProxy.h b/src/Disks/DiskRestartProxy.h index f5502d9d68f..e6c94d9ad7b 100644 --- a/src/Disks/DiskRestartProxy.h +++ b/src/Disks/DiskRestartProxy.h @@ -47,7 +47,7 @@ public: const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const override; std::unique_ptr writeFile(const String & path, size_t buf_size, WriteMode mode) override; diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 0d36cadc349..bc7810479c5 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -37,7 +37,7 @@ DiskSelector::DiskSelector(const Poco::Util::AbstractConfiguration & config, con auto disk_config_prefix = config_prefix + "." + disk_name; - disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context)); + disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context, disks)); } if (!has_default_disk) disks.emplace(default_disk_name, std::make_shared(default_disk_name, context->getPath(), 0)); @@ -62,16 +62,16 @@ DiskSelectorPtr DiskSelector::updateFromConfig( if (!std::all_of(disk_name.begin(), disk_name.end(), isWordCharASCII)) throw Exception("Disk name can contain only alphanumeric and '_' (" + disk_name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG); + auto disk_config_prefix = config_prefix + "." + disk_name; if (result->getDisksMap().count(disk_name) == 0) { - auto disk_config_prefix = config_prefix + "." + disk_name; - result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context)); + result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context, result->getDisksMap())); } else { auto disk = old_disks_minus_new_disks[disk_name]; - disk->applyNewSettings(config, context); + disk->applyNewSettings(config, context, disk_config_prefix, result->getDisksMap()); old_disks_minus_new_disks.erase(disk_name); } diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index 4652cc40ea3..88cc6ee5197 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -12,7 +12,6 @@ namespace DB class DiskSelector; using DiskSelectorPtr = std::shared_ptr; -using DisksMap = std::map; /// Parse .xml configuration and store information about disks /// Mostly used for introspection. diff --git a/src/Disks/DiskType.h b/src/Disks/DiskType.h index a5c23fe2c2c..5eeeaaeb2e3 100644 --- a/src/Disks/DiskType.h +++ b/src/Disks/DiskType.h @@ -12,7 +12,8 @@ struct DiskType Local, RAM, S3, - HDFS + HDFS, + Encrypted }; static String toString(Type disk_type) { @@ -26,6 +27,8 @@ struct DiskType return "s3"; case Type::HDFS: return "hdfs"; + case Type::Encrypted: + return "encrypted"; } __builtin_unreachable(); } diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index dafd507ba1e..4eb43eaf7b5 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -115,7 +115,7 @@ std::unique_ptr DiskHDFS::writeFile(const String & path auto hdfs_path = remote_fs_root_path + file_name; LOG_DEBUG(log, "{} to file by path: {}. HDFS path: {}", mode == WriteMode::Rewrite ? "Write" : "Append", - backQuote(metadata_path + path), remote_fs_root_path + hdfs_path); + backQuote(metadata_path + path), hdfs_path); /// Single O_WRONLY in libhdfs adds O_TRUNC auto hdfs_buffer = std::make_unique(hdfs_path, @@ -153,6 +153,14 @@ void DiskHDFS::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) }); } +bool DiskHDFS::checkUniqueId(const String & hdfs_uri) const +{ + if (!boost::algorithm::starts_with(hdfs_uri, remote_fs_root_path)) + return false; + const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2); + const String remote_fs_object_path = hdfs_uri.substr(begin_of_path); + return (0 == hdfsExists(hdfs_fs.get(), remote_fs_object_path.c_str())); +} namespace { @@ -170,7 +178,8 @@ void registerDiskHDFS(DiskFactory & factory) auto creator = [](const String & name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - ContextPtr context_) -> DiskPtr + ContextPtr context_, + const DisksMap & /*map*/) -> DiskPtr { fs::path disk = fs::path(context_->getPath()) / "disks" / name; fs::create_directories(disk); diff --git a/src/Disks/HDFS/DiskHDFS.h b/src/Disks/HDFS/DiskHDFS.h index 49fdf44728b..068572daa62 100644 --- a/src/Disks/HDFS/DiskHDFS.h +++ b/src/Disks/HDFS/DiskHDFS.h @@ -44,11 +44,13 @@ public: DiskType::Type getType() const override { return DiskType::Type::HDFS; } + bool supportZeroCopyReplication() const override { return true; } + std::unique_ptr readFile( const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const override; @@ -58,6 +60,11 @@ public: RemoteFSPathKeeperPtr createFSPathKeeper() const override; + /// Check file exists and ClickHouse has an access to it + /// Overrode in remote disk + /// Required for remote disk to ensure that replica has access to data written by other node + bool checkUniqueId(const String & hdfs_uri) const override; + private: String getRandomName() { return toString(UUIDHelpers::generateV4()); } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index ecaf7d63fdc..a70b44a789f 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -156,7 +157,7 @@ public: const String & path, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, size_t estimated_size = 0, - size_t aio_threshold = 0, + size_t direct_io_threshold = 0, size_t mmap_threshold = 0, MMappedFileCache * mmap_cache = nullptr) const = 0; @@ -211,19 +212,23 @@ public: /// Return disk type - "local", "s3", etc. virtual DiskType::Type getType() const = 0; + /// Whether this disk support zero-copy replication. + /// Overrode in remote fs disks. + virtual bool supportZeroCopyReplication() const = 0; + /// Invoked when Global Context is shutdown. virtual void shutdown() {} /// Performs action on disk startup. virtual void startup() {} - /// Return some uniq string for file, overrode for S3 - /// Required for distinguish different copies of the same part on S3 + /// Return some uniq string for file, overrode for IDiskRemote + /// Required for distinguish different copies of the same part on remote disk virtual String getUniqueId(const String & path) const { return path; } /// Check file exists and ClickHouse has an access to it - /// Overrode in DiskS3 - /// Required for S3 to ensure that replica has access to data written by other node + /// Overrode in remote FS disks (s3/hdfs) + /// Required for remote disk to ensure that replica has access to data written by other node virtual bool checkUniqueId(const String & id) const { return exists(id); } /// Invoked on partitions freeze query. @@ -233,7 +238,7 @@ public: virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const; /// Applies new settings for disk in runtime. - virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr) {} + virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr, const String &, const DisksMap &) { } protected: friend class DiskDecorator; diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index a4dcc8037bc..398e617196d 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -344,17 +344,6 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path) } -void IDiskRemote::removeFileIfExists(const String & path) -{ - RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - if (fs::exists(fs::path(metadata_path) / path)) - { - removeMeta(path, fs_paths_keeper); - removeFromRemoteFS(fs_paths_keeper); - } -} - - void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); @@ -364,6 +353,18 @@ void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs) } +void IDiskRemote::removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) +{ + RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); + if (fs::exists(fs::path(metadata_path) / path)) + { + removeMeta(path, fs_paths_keeper); + if (!keep_in_remote_fs) + removeFromRemoteFS(fs_paths_keeper); + } +} + + void IDiskRemote::removeSharedRecursive(const String & path, bool keep_in_remote_fs) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); @@ -488,4 +489,13 @@ bool IDiskRemote::tryReserve(UInt64 bytes) return false; } +String IDiskRemote::getUniqueId(const String & path) const +{ + Metadata metadata(remote_fs_root_path, metadata_path, path); + String id; + if (!metadata.remote_fs_objects.empty()) + id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first; + return id; +} + } diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index 360d4e2de33..80b01c3c949 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -83,12 +83,14 @@ public: void removeFile(const String & path) override { removeSharedFile(path, false); } - void removeFileIfExists(const String & path) override; + void removeFileIfExists(const String & path) override { removeSharedFileIfExists(path, false); } void removeRecursive(const String & path) override { removeSharedRecursive(path, false); } void removeSharedFile(const String & path, bool keep_in_remote_fs) override; + void removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) override; + void removeSharedRecursive(const String & path, bool keep_in_remote_fs) override; void listFiles(const String & path, std::vector & file_names) override; @@ -117,6 +119,10 @@ public: ReservationPtr reserve(UInt64 bytes) override; + String getUniqueId(const String & path) const override; + + bool checkUniqueId(const String & id) const override = 0; + virtual void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) = 0; virtual RemoteFSPathKeeperPtr createFSPathKeeper() const = 0; diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index e52a19de99a..1f1c73c32c3 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -158,15 +158,6 @@ DiskS3::DiskS3( { } -String DiskS3::getUniqueId(const String & path) const -{ - Metadata metadata(remote_fs_root_path, metadata_path, path); - String id; - if (!metadata.remote_fs_objects.empty()) - id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first; - return id; -} - RemoteFSPathKeeperPtr DiskS3::createFSPathKeeper() const { auto settings = current_settings.get(); @@ -930,7 +921,7 @@ void DiskS3::onFreeze(const String & path) revision_file_buf.finalize(); } -void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) +void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &) { auto new_settings = settings_getter(config, "storage_configuration.disks." + name, context); diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index 21bf0d3867b..133488ad31f 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -77,7 +77,7 @@ public: const String & path, size_t buf_size, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache) const override; @@ -98,22 +98,21 @@ public: DiskType::Type getType() const override { return DiskType::Type::S3; } + bool supportZeroCopyReplication() const override { return true; } + void shutdown() override; void startup() override; - /// Return some uniq string for file - /// Required for distinguish different copies of the same part on S3 - String getUniqueId(const String & path) const override; - /// Check file exists and ClickHouse has an access to it - /// Required for S3 to ensure that replica has access to data wroten by other node + /// Overrode in remote disk + /// Required for remote disk to ensure that replica has access to data written by other node bool checkUniqueId(const String & id) const override; /// Dumps current revision counter into file 'revision.txt' at given path. void onFreeze(const String & path) override; - void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) override; + void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &) override; private: void createFileOperationObject(const String & operation_name, UInt64 revision, const ObjectMetadata & metadata); diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index 1e40f45b098..49a11b1dbb9 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -167,7 +167,8 @@ void registerDiskS3(DiskFactory & factory) auto creator = [](const String & name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, - ContextPtr context) -> DiskPtr { + ContextPtr context, + const DisksMap & /*map*/) -> DiskPtr { S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint"))); if (uri.key.back() != '/') throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); diff --git a/src/Disks/registerDisks.cpp b/src/Disks/registerDisks.cpp index 8f4901e49e5..bf2f09853fe 100644 --- a/src/Disks/registerDisks.cpp +++ b/src/Disks/registerDisks.cpp @@ -16,6 +16,10 @@ void registerDiskMemory(DiskFactory & factory); void registerDiskS3(DiskFactory & factory); #endif +#if USE_SSL +void registerDiskEncrypted(DiskFactory & factory); +#endif + #if USE_HDFS void registerDiskHDFS(DiskFactory & factory); #endif @@ -32,6 +36,10 @@ void registerDisks() registerDiskS3(factory); #endif +#if USE_SSL + registerDiskEncrypted(factory); +#endif + #if USE_HDFS registerDiskHDFS(factory); #endif diff --git a/src/Disks/ya.make b/src/Disks/ya.make index 2312dc96241..925dfd2a0ce 100644 --- a/src/Disks/ya.make +++ b/src/Disks/ya.make @@ -10,6 +10,7 @@ PEERDIR( SRCS( DiskCacheWrapper.cpp DiskDecorator.cpp + DiskEncrypted.cpp DiskFactory.cpp DiskLocal.cpp DiskMemory.cpp diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 8b7cf9635b4..a00839fc5f5 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -33,6 +33,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int FORMAT_IS_NOT_SUITABLE_FOR_INPUT; extern const int FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT; + extern const int UNSUPPORTED_METHOD; } const FormatFactory::Creators & FormatFactory::getCreators(const String & name) const @@ -207,6 +208,9 @@ BlockOutputStreamPtr FormatFactory::getOutputStreamParallelIfPossible( WriteCallback callback, const std::optional & _format_settings) const { + if (context->getMySQLProtocolContext() && name != "MySQLWire") + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats"); + const auto & output_getter = getCreators(name).output_processor_creator; const Settings & settings = context->getSettingsRef(); @@ -309,7 +313,10 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible( { const auto & output_getter = getCreators(name).output_processor_creator; if (!output_getter) - throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output (with processors)", name); + + if (context->getMySQLProtocolContext() && name != "MySQLWire") + throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats"); auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context); @@ -344,7 +351,7 @@ OutputFormatPtr FormatFactory::getOutputFormat( { const auto & output_getter = getCreators(name).output_processor_creator; if (!output_getter) - throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT); + throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output (with processors)", name); if (context->hasQueryContext() && context->getSettingsRef().log_queries) context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Format, name); diff --git a/src/Functions/CMakeLists.txt b/src/Functions/CMakeLists.txt index b20954c9652..04e5f80468b 100644 --- a/src/Functions/CMakeLists.txt +++ b/src/Functions/CMakeLists.txt @@ -124,3 +124,6 @@ endif() # Signed integer overflow on user-provided data inside boost::geometry - ignore. set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow) + +# target_link_libraries(clickhouse_functions PRIVATE ${S2_LIBRARY}) +target_include_directories(clickhouse_functions SYSTEM PUBLIC ${S2_GEOMETRY_INCLUDE_DIR}) diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index dcdd0e521eb..eac1a7ad1a1 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -49,47 +49,48 @@ Columns convertConstTupleToConstantElements(const ColumnConst & column) return res; } +ColumnWithTypeAndName columnGetNested(const ColumnWithTypeAndName & col) +{ + if (col.type->isNullable()) + { + const DataTypePtr & nested_type = static_cast(*col.type).getNestedType(); + + if (!col.column) + { + return ColumnWithTypeAndName{nullptr, nested_type, col.name}; + } + else if (const auto * nullable = checkAndGetColumn(*col.column)) + { + const auto & nested_col = nullable->getNestedColumnPtr(); + return ColumnWithTypeAndName{nested_col, nested_type, col.name}; + } + else if (const auto * const_column = checkAndGetColumn(*col.column)) + { + const auto * nullable_column = checkAndGetColumn(const_column->getDataColumn()); + + ColumnPtr nullable_res; + if (nullable_column) + { + const auto & nested_col = nullable_column->getNestedColumnPtr(); + nullable_res = ColumnConst::create(nested_col, col.column->size()); + } + else + { + nullable_res = makeNullable(col.column); + } + return ColumnWithTypeAndName{ nullable_res, nested_type, col.name }; + } + else + throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN); + } + return col; +} ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns) { ColumnsWithTypeAndName res; for (const auto & col : columns) - { - if (col.type->isNullable()) - { - const DataTypePtr & nested_type = static_cast(*col.type).getNestedType(); - - if (!col.column) - { - res.emplace_back(ColumnWithTypeAndName{nullptr, nested_type, col.name}); - } - else if (const auto * nullable = checkAndGetColumn(*col.column)) - { - const auto & nested_col = nullable->getNestedColumnPtr(); - res.emplace_back(ColumnWithTypeAndName{nested_col, nested_type, col.name}); - } - else if (const auto * const_column = checkAndGetColumn(*col.column)) - { - const auto * nullable_column = checkAndGetColumn(const_column->getDataColumn()); - - ColumnPtr nullable_res; - if (nullable_column) - { - const auto & nested_col = nullable_column->getNestedColumnPtr(); - nullable_res = ColumnConst::create(nested_col, col.column->size()); - } - else - { - nullable_res = makeNullable(col.column); - } - res.emplace_back(ColumnWithTypeAndName{ nullable_res, nested_type, col.name }); - } - else - throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN); - } - else - res.emplace_back(col); - } + res.emplace_back(columnGetNested(col)); return res; } diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h index 7ab008b8bea..5abe24f4e50 100644 --- a/src/Functions/FunctionHelpers.h +++ b/src/Functions/FunctionHelpers.h @@ -81,6 +81,8 @@ inline std::enable_if_t, Field> toField(const T & x, UInt32 s Columns convertConstTupleToConstantElements(const ColumnConst & column); +/// Returns nested column with corrected type if nullable +ColumnWithTypeAndName columnGetNested(const ColumnWithTypeAndName & col); /// Returns the copy of a given columns in which each column is replaced with its respective nested /// column if it is nullable. diff --git a/src/Functions/FunctionJoinGet.h b/src/Functions/FunctionJoinGet.h index 2250fa3ccf0..c701625e9cd 100644 --- a/src/Functions/FunctionJoinGet.h +++ b/src/Functions/FunctionJoinGet.h @@ -28,7 +28,7 @@ public: static constexpr auto name = or_null ? "joinGetOrNull" : "joinGet"; bool useDefaultImplementationForNulls() const override { return false; } - bool useDefaultImplementationForLowCardinalityColumns() const override { return true; } + bool useDefaultImplementationForLowCardinalityColumns() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override; diff --git a/src/Functions/FunctionsCoding.h b/src/Functions/FunctionsCoding.h index 72f2aa1be1c..00b09acea1f 100644 --- a/src/Functions/FunctionsCoding.h +++ b/src/Functions/FunctionsCoding.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -978,7 +979,8 @@ public: !which.isDateTime64() && !which.isUInt() && !which.isFloat() && - !which.isDecimal()) + !which.isDecimal() && + !which.isAggregateFunction()) throw Exception("Illegal type " + arguments[0]->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -990,6 +992,15 @@ public: const IColumn * column = arguments[0].column.get(); ColumnPtr res_column; + WhichDataType which(column->getDataType()); + if (which.isAggregateFunction()) + { + const ColumnPtr to_string = castColumn(arguments[0], std::make_shared()); + const auto * str_column = checkAndGetColumn(to_string.get()); + tryExecuteString(str_column, res_column); + return res_column; + } + if (tryExecuteUInt(column, res_column) || tryExecuteUInt(column, res_column) || tryExecuteUInt(column, res_column) || diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index ff97894d98e..4cb5b574475 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -579,36 +579,44 @@ template struct ConvertImpl struct FormatImpl { - static void execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType *, const DateLUTImpl *) + template + static ReturnType execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType *, const DateLUTImpl *) { writeText(x, wb); + return ReturnType(true); } }; template <> struct FormatImpl { - static void execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl *) + template + static ReturnType execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl *) { writeDateText(DayNum(x), wb); + return ReturnType(true); } }; template <> struct FormatImpl { - static void execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime *, const DateLUTImpl * time_zone) + template + static ReturnType execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime *, const DateLUTImpl * time_zone) { writeDateTimeText(x, wb, *time_zone); + return ReturnType(true); } }; template <> struct FormatImpl { - static void execute(const DataTypeDateTime64::FieldType x, WriteBuffer & wb, const DataTypeDateTime64 * type, const DateLUTImpl * time_zone) + template + static ReturnType execute(const DataTypeDateTime64::FieldType x, WriteBuffer & wb, const DataTypeDateTime64 * type, const DateLUTImpl * time_zone) { writeDateTimeText(DateTime64(x), type->getScale(), wb, *time_zone); + return ReturnType(true); } }; @@ -616,18 +624,34 @@ struct FormatImpl template struct FormatImpl> { - static void execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum * type, const DateLUTImpl *) + template + static ReturnType execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum * type, const DateLUTImpl *) { - writeString(type->getNameForValue(x), wb); + static constexpr bool throw_exception = std::is_same_v; + + if constexpr (throw_exception) + { + writeString(type->getNameForValue(x), wb); + } + else + { + StringRef res; + bool is_ok = type->getNameForValue(x, res); + if (is_ok) + writeString(res, wb); + return ReturnType(is_ok); + } } }; template struct FormatImpl> { - static void execute(const FieldType x, WriteBuffer & wb, const DataTypeDecimal * type, const DateLUTImpl *) + template + static ReturnType execute(const FieldType x, WriteBuffer & wb, const DataTypeDecimal * type, const DateLUTImpl *) { writeText(x, type->getScale(), wb); + return ReturnType(true); } }; @@ -642,6 +666,16 @@ struct ConvertImpl, DataTypeNumber, Name, Con } }; +static ColumnUInt8::MutablePtr copyNullMap(ColumnPtr col) +{ + ColumnUInt8::MutablePtr null_map = nullptr; + if (const auto * col_null = checkAndGetColumn(col.get())) + { + null_map = ColumnUInt8::create(); + null_map->insertRangeFrom(col_null->getNullMapColumn(), 0, col_null->size()); + } + return null_map; +} template struct ConvertImpl, DataTypeString>, Name, ConvertDefaultBehaviorTag> @@ -651,13 +685,18 @@ struct ConvertImpl(*col_with_type_and_name.type); const DateLUTImpl * time_zone = nullptr; /// For argument of DateTime type, second argument with time zone could be specified. if constexpr (std::is_same_v || std::is_same_v) - time_zone = &extractTimeZoneFromFunctionArguments(arguments, 1, 0); + { + auto non_null_args = createBlockWithNestedColumns(arguments); + time_zone = &extractTimeZoneFromFunctionArguments(non_null_args, 1, 0); + } if (const auto col_from = checkAndGetColumn(col_with_type_and_name.column.get())) { @@ -681,14 +720,30 @@ struct ConvertImpl write_buffer(data_to); - for (size_t i = 0; i < size; ++i) + if (null_map) { - FormatImpl::execute(vec_from[i], write_buffer, &type, time_zone); - writeChar(0, write_buffer); - offsets_to[i] = write_buffer.count(); + for (size_t i = 0; i < size; ++i) + { + bool is_ok = FormatImpl::template execute(vec_from[i], write_buffer, &type, time_zone); + null_map->getData()[i] |= !is_ok; + writeChar(0, write_buffer); + offsets_to[i] = write_buffer.count(); + } + } + else + { + for (size_t i = 0; i < size; ++i) + { + FormatImpl::template execute(vec_from[i], write_buffer, &type, time_zone); + writeChar(0, write_buffer); + offsets_to[i] = write_buffer.count(); + } } write_buffer.finalize(); + + if (null_map) + return ColumnNullable::create(std::move(col_to), std::move(null_map)); return col_to; } else @@ -702,9 +757,11 @@ struct ConvertImplisNullable() && null_map) + return ColumnNullable::create(std::move(col_to), std::move(null_map)); return col_to; } }; @@ -1398,7 +1458,11 @@ public: /// Function actually uses default implementation for nulls, /// but we need to know if return type is Nullable or not, /// so we use checked_return_type only to intercept the first call to getReturnTypeImpl(...). - bool useDefaultImplementationForNulls() const override { return checked_return_type; } + bool useDefaultImplementationForNulls() const override + { + bool to_nullable_string = to_nullable && std::is_same_v; + return checked_return_type && !to_nullable_string; + } bool useDefaultImplementationForConstants() const override { return true; } ColumnNumbers getArgumentsThatAreAlwaysConstant() const override @@ -1463,7 +1527,10 @@ private: throw Exception{"Function " + getName() + " expects at least 1 argument", ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION}; - const IDataType * from_type = arguments[0].type.get(); + if (result_type->onlyNull()) + return result_type->createColumnConstWithDefaultValue(input_rows_count); + + const DataTypePtr from_type = removeNullable(arguments[0].type); ColumnPtr result_column; auto call = [&](const auto & types, const auto & tag) -> bool @@ -1559,7 +1626,7 @@ private: /// Generic conversion of any type to String. if (std::is_same_v) { - return ConvertImplGenericToString::execute(arguments); + return ConvertImplGenericToString::execute(arguments, result_type); } else throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(), diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 381401be2c5..118855b4bf8 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -163,13 +163,6 @@ public: arguments[0]->getName(), getName()); - if (!WhichDataType(arguments[1]).isUInt64() && - !isTuple(arguments[1])) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of second argument of function {} must be UInt64 or tuple(...)", - arguments[1]->getName(), - getName()); - return std::make_shared(); } @@ -189,8 +182,8 @@ public: auto dictionary_key_type = dictionary->getKeyType(); const ColumnWithTypeAndName & key_column_with_type = arguments[1]; - const auto key_column = key_column_with_type.column; - const auto key_column_type = WhichDataType(key_column_with_type.type); + auto key_column = key_column_with_type.column; + auto key_column_type = key_column_with_type.type; ColumnPtr range_col = nullptr; DataTypePtr range_col_type = nullptr; @@ -214,7 +207,7 @@ public: if (dictionary_key_type == DictionaryKeyType::simple) { - if (!key_column_type.isUInt64()) + if (!WhichDataType(key_column_type).isUInt64()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument of function {} must be UInt64 when dictionary is simple. Actual type {}.", @@ -225,24 +218,39 @@ public: } else if (dictionary_key_type == DictionaryKeyType::complex) { - if (!key_column_type.isTuple()) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Second argument of function {} must be tuple when dictionary is complex. Actual type {}.", - getName(), - key_column_with_type.type->getName()); - /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys. - ColumnPtr key_column_full = key_column->convertToFullColumnIfConst(); + key_column = key_column->convertToFullColumnIfConst(); + size_t keys_size = dictionary->getStructure().getKeysSize(); - const auto & key_columns = typeid_cast(*key_column_full).getColumnsCopy(); - const auto & key_types = static_cast(*key_column_with_type.type).getElements(); + if (!isTuple(key_column_type)) + { + if (keys_size > 1) + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute." + "Actual type {}.", + getName(), + key_column_type->getName()); + } + else + { + Columns tuple_columns = {std::move(key_column)}; + key_column = ColumnTuple::create(tuple_columns); + + DataTypes tuple_types = {key_column_type}; + key_column_type = std::make_shared(tuple_types); + } + } + + const auto & key_columns = assert_cast(*key_column).getColumnsCopy(); + const auto & key_types = assert_cast(*key_column_type).getElements(); return dictionary->hasKeys(key_columns, key_types); } else { - if (!key_column_type.isUInt64()) + if (!WhichDataType(key_column_type).isUInt64()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument of function {} must be UInt64 when dictionary is range. Actual type {}.", @@ -346,13 +354,6 @@ public: Strings attribute_names = getAttributeNamesFromColumn(arguments[1].column, arguments[1].type); auto dictionary = helper.getDictionary(dictionary_name); - - if (!WhichDataType(arguments[2].type).isUInt64() && !isTuple(arguments[2].type)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of third argument of function {}, must be UInt64 or tuple(...).", - arguments[2].type->getName(), - getName()); - auto dictionary_key_type = dictionary->getKeyType(); size_t current_arguments_index = 3; @@ -446,18 +447,35 @@ public: } else if (dictionary_key_type == DictionaryKeyType::complex) { - if (!isTuple(key_col_with_type.type)) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Third argument of function {} must be tuple when dictionary is complex. Actual type {}.", - getName(), - key_col_with_type.type->getName()); - /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys. - ColumnPtr key_column_full = key_col_with_type.column->convertToFullColumnIfConst(); + ColumnPtr key_column = key_col_with_type.column->convertToFullColumnIfConst(); + DataTypePtr key_column_type = key_col_with_type.type; - const auto & key_columns = typeid_cast(*key_column_full).getColumnsCopy(); - const auto & key_types = static_cast(*key_col_with_type.type).getElements(); + size_t keys_size = dictionary->getStructure().getKeysSize(); + + if (!isTuple(key_column_type)) + { + if (keys_size > 1) + { + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Third argument of function {} must be tuple when dictionary is complex and key contains more than 1 attribute." + "Actual type {}.", + getName(), + key_col_with_type.type->getName()); + } + else + { + Columns tuple_columns = {std::move(key_column)}; + key_column = ColumnTuple::create(tuple_columns); + + DataTypes tuple_types = {key_column_type}; + key_column_type = std::make_shared(tuple_types); + } + } + + const auto & key_columns = assert_cast(*key_column).getColumnsCopy(); + const auto & key_types = assert_cast(*key_column_type).getElements(); result = executeDictionaryRequest( dictionary, diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 0da55343b9d..c00baf2850b 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -113,7 +113,8 @@ public: virtual ~IFunctionBase() = default; - virtual ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const + virtual ColumnPtr execute( + const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, bool dry_run = false) const { return prepare(arguments)->execute(arguments, result_type, input_rows_count, dry_run); } @@ -161,7 +162,8 @@ public: * Arguments are passed without modifications, useDefaultImplementationForNulls, useDefaultImplementationForConstants, * useDefaultImplementationForLowCardinality are not applied. */ - virtual ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; } + virtual ColumnPtr getConstantResultForNonConstArguments( + const ColumnsWithTypeAndName & /* arguments */, const DataTypePtr & /* result_type */) const { return nullptr; } /** Function is called "injective" if it returns different result for different values of arguments. * Example: hex, negate, tuple... @@ -358,6 +360,10 @@ public: */ virtual bool useDefaultImplementationForConstants() const { return false; } + /** Some arguments could remain constant during this implementation. + */ + virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; } + /** If function arguments has single low cardinality column and all other arguments are constants, call function on nested column. * Otherwise, convert all low cardinality columns to ordinary columns. * Returns ColumnLowCardinality if at least one argument is ColumnLowCardinality. @@ -367,10 +373,6 @@ public: /// If it isn't, will convert all ColumnLowCardinality arguments to full columns. virtual bool canBeExecutedOnLowCardinalityDictionary() const { return true; } - /** Some arguments could remain constant during this implementation. - */ - virtual ColumnNumbers getArgumentsThatAreAlwaysConstant() const { return {}; } - /** True if function can be called on default arguments (include Nullable's) and won't throw. * Counterexample: modulo(0, 0) */ diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h index 3367b52cc36..a050de2edb6 100644 --- a/src/Functions/PolygonUtils.h +++ b/src/Functions/PolygonUtils.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -40,6 +41,7 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int BAD_ARGUMENTS; } @@ -304,6 +306,13 @@ void PointInPolygonWithGrid::calcGridAttributes( y_scale = 1 / cell_height; x_shift = -min_corner.x(); y_shift = -min_corner.y(); + + if (!(isFinite(x_scale) + && isFinite(y_scale) + && isFinite(x_shift) + && isFinite(y_shift) + && isFinite(grid_size))) + throw Exception("Polygon is not valid: bounding box is unbounded", ErrorCodes::BAD_ARGUMENTS); } template @@ -358,7 +367,7 @@ bool PointInPolygonWithGrid::contains(CoordinateType x, Coordina if (has_empty_bound) return false; - if (std::isnan(x) || std::isnan(y)) + if (!isFinite(x) || !isFinite(y)) return false; CoordinateType float_row = (y + y_shift) * y_scale; diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h index 4670d610725..08576fe59ec 100644 --- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h +++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h @@ -41,6 +41,9 @@ public: String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } + bool useDefaultImplementationForConstants() const override { return true; } + ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; } + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { if (!isString(arguments[0].type)) @@ -65,9 +68,7 @@ public: const ColumnConst * column_tld_list_name = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get()); FirstSignificantSubdomainCustomLookup tld_lookup(column_tld_list_name->getValue()); - /// FIXME: convertToFullColumnIfConst() is suboptimal - auto column = arguments[0].column->convertToFullColumnIfConst(); - if (const ColumnString * col = checkAndGetColumn(*column)) + if (const ColumnString * col = checkAndGetColumn(*arguments[0].column)) { auto col_res = ColumnString::create(); vector(tld_lookup, col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets()); diff --git a/src/Functions/array/arrayElement.cpp b/src/Functions/array/arrayElement.cpp index f3d3f558d7b..d39a865133e 100644 --- a/src/Functions/array/arrayElement.cpp +++ b/src/Functions/array/arrayElement.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -110,6 +111,9 @@ private: static bool matchKeyToIndexString(const IColumn & data, const Offsets & offsets, const ColumnsWithTypeAndName & arguments, PaddedPODArray & matched_idxs); + static bool matchKeyToIndexFixedString(const IColumn & data, const Offsets & offsets, + const ColumnsWithTypeAndName & arguments, PaddedPODArray & matched_idxs); + static bool matchKeyToIndexStringConst(const IColumn & data, const Offsets & offsets, const Field & index, PaddedPODArray & matched_idxs); @@ -767,6 +771,19 @@ struct MatcherString } }; +struct MatcherFixedString +{ + const ColumnFixedString & data; + const ColumnFixedString & index; + + bool match(size_t row_data, size_t row_index) const + { + auto data_ref = data.getDataAt(row_data); + auto index_ref = index.getDataAt(row_index); + return memequalSmallAllowOverflow15(index_ref.data, index_ref.size, data_ref.data, data_ref.size); + } +}; + struct MatcherStringConst { const ColumnString & data; @@ -863,6 +880,23 @@ bool FunctionArrayElement::matchKeyToIndexString( return true; } +bool FunctionArrayElement::matchKeyToIndexFixedString( + const IColumn & data, const Offsets & offsets, + const ColumnsWithTypeAndName & arguments, PaddedPODArray & matched_idxs) +{ + const auto * index_string = checkAndGetColumn(arguments[1].column.get()); + if (!index_string) + return false; + + const auto * data_string = checkAndGetColumn(&data); + if (!data_string) + return false; + + MatcherFixedString matcher{*data_string, *index_string}; + executeMatchKeyToIndex(offsets, matched_idxs, matcher); + return true; +} + template bool FunctionArrayElement::matchKeyToIndexNumberConst( const IColumn & data, const Offsets & offsets, @@ -922,8 +956,10 @@ bool FunctionArrayElement::matchKeyToIndex( || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) + || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) || matchKeyToIndexNumber(data, offsets, arguments, matched_idxs) - || matchKeyToIndexString(data, offsets, arguments, matched_idxs); + || matchKeyToIndexString(data, offsets, arguments, matched_idxs) + || matchKeyToIndexFixedString(data, offsets, arguments, matched_idxs); } bool FunctionArrayElement::matchKeyToIndexConst( diff --git a/src/Functions/array/arrayIndex.h b/src/Functions/array/arrayIndex.h index f3b279faaef..a390abc4eaf 100644 --- a/src/Functions/array/arrayIndex.h +++ b/src/Functions/array/arrayIndex.h @@ -58,10 +58,10 @@ struct CountEqualAction namespace Impl { template < - class ConcreteAction, + typename ConcreteAction, bool RightArgIsConstant = false, - class IntegralInitial = UInt64, - class IntegralResult = UInt64> + typename IntegralInitial = UInt64, + typename IntegralResult = UInt64> struct Main { private: @@ -94,13 +94,13 @@ private: } /// LowCardinality - static bool compare(const IColumn & left, const Result& right, size_t i, size_t) + static bool compare(const IColumn & left, const Result & right, size_t i, size_t) { return left.getUInt(i) == right; } /// Generic - static bool compare(const IColumn& left, const IColumn& right, size_t i, size_t j) + static bool compare(const IColumn & left, const IColumn & right, size_t i, size_t j) { return 0 == left.compareAt(i, RightArgIsConstant ? 0 : j, right, 1); } @@ -109,7 +109,7 @@ private: static constexpr bool hasNull(const NullMap * const null_map, size_t i) noexcept { return (*null_map)[i]; } - template + template static void process( const Data & data, const ArrOffsets & offsets, const Target & target, ResultArr & result, [[maybe_unused]] const NullMap * const null_map_data, @@ -148,7 +148,7 @@ private: continue; } else if (!compare(data, target, current_offset + j, i)) - continue; + continue; ConcreteAction::apply(current, j); @@ -162,7 +162,7 @@ private: } public: - template + template static void vector( const Data & data, const ArrOffsets & offsets, @@ -183,7 +183,7 @@ public: }; /// When the 2nd function argument is a NULL value. -template +template struct Null { using ResultType = typename ConcreteAction::ResultType; @@ -227,7 +227,7 @@ struct Null } }; -template +template struct String { private: @@ -350,7 +350,7 @@ public: }; } -template +template class FunctionArrayIndex : public IFunction { public: @@ -565,7 +565,7 @@ private: * Integral s = {s1, s2, ...} * (s1, s1, s2, ...), (s2, s1, s2, ...), (s3, s1, s2, ...) */ - template + template static inline ColumnPtr executeIntegral(const ColumnsWithTypeAndName & arguments) { const ColumnArray * const left = checkAndGetColumn(arguments[0].column.get()); @@ -590,14 +590,14 @@ private: return nullptr; } - template + template static inline bool executeIntegral(ExecutionData& data) { return (executeIntegralExpanded(data) || ...); } /// Invoke executeIntegralImpl with such parameters: (A, other1), (A, other2), ... - template + template static inline bool executeIntegralExpanded(ExecutionData& data) { return (executeIntegralImpl(data) || ...); @@ -608,7 +608,7 @@ private: * second argument, namely, the @e value, so it's possible to invoke the has(Array(Int8), UInt64) e.g. * so we have to check all possible variants for #Initial and #Resulting types. */ - template + template static bool executeIntegralImpl(ExecutionData& data) { const ColumnVector * col_nested = checkAndGetColumn>(&data.left); @@ -647,7 +647,7 @@ private: } /** - * Catches arguments of type LC(T) (left) and U (right). + * Catches arguments of type LowCardinality(T) (left) and U (right). * * The perftests * https://clickhouse-test-reports.s3.yandex.net/12550/2d27fa0fa8c198a82bf1fe3625050ccf56695976/integration_tests_(release).html @@ -726,7 +726,7 @@ private: return col_result; } - else if (col_lc->nestedIsNullable()) // LC(Nullable(T)) and U + else if (col_lc->nestedIsNullable()) // LowCardinality(Nullable(T)) and U { const ColumnPtr left_casted = col_lc->convertToFullColumnIfLowCardinality(); // Nullable(T) const ColumnNullable& left_nullable = *checkAndGetColumn(left_casted.get()); @@ -746,16 +746,17 @@ private: ? right_nullable->getNestedColumn() : *right_casted.get(); - ExecutionData data = { + ExecutionData data = + { left_ptr, right_ptr, col_array->getOffsets(), nullptr, {null_map_left_casted, null_map_right_casted}}; - if (dispatchConvertedLCColumns(data)) + if (dispatchConvertedLowCardinalityColumns(data)) return data.result_column; } - else // LC(T) and U, T not Nullable + else // LowCardinality(T) and U, T not Nullable { if (col_arg.isNullable()) return nullptr; @@ -764,24 +765,25 @@ private: arg_lc && arg_lc->isNullable()) return nullptr; - // LC(T) and U (possibly LC(V)) + // LowCardinality(T) and U (possibly LowCardinality(V)) const ColumnPtr left_casted = col_lc->convertToFullColumnIfLowCardinality(); const ColumnPtr right_casted = col_arg.convertToFullColumnIfLowCardinality(); - ExecutionData data = { + ExecutionData data = + { *left_casted.get(), *right_casted.get(), col_array->getOffsets(), nullptr, {null_map_data, null_map_item} }; - if (dispatchConvertedLCColumns(data)) + if (dispatchConvertedLowCardinalityColumns(data)) return data.result_column; } return nullptr; } - static bool dispatchConvertedLCColumns(ExecutionData& data) + static bool dispatchConvertedLowCardinalityColumns(ExecutionData & data) { if (data.left.isNumeric() && data.right.isNumeric()) // ColumnArrays return executeIntegral(data); diff --git a/src/Functions/array/range.cpp b/src/Functions/array/range.cpp index 5b9886580dc..9eefc4f178d 100644 --- a/src/Functions/array/range.cpp +++ b/src/Functions/array/range.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include @@ -31,8 +32,10 @@ class FunctionRange : public IFunction { public: static constexpr auto name = "range"; - static constexpr size_t max_elements = 100'000'000; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + const size_t max_elements; + static FunctionPtr create(ContextPtr context_) { return std::make_shared(std::move(context_)); } + explicit FunctionRange(ContextPtr context) : max_elements(context->getSettingsRef().function_range_max_elements_in_block) {} private: String getName() const override { return name; } diff --git a/src/Functions/config_functions.h.in b/src/Functions/config_functions.h.in index eb96c13c355..3e1c862300c 100644 --- a/src/Functions/config_functions.h.in +++ b/src/Functions/config_functions.h.in @@ -6,4 +6,5 @@ #cmakedefine01 USE_SIMDJSON #cmakedefine01 USE_RAPIDJSON #cmakedefine01 USE_H3 +#cmakedefine01 USE_S2_GEOMETRY #cmakedefine01 USE_FASTOPS diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 7edb3faf62d..1b12e6c9ad3 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -21,6 +21,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; } namespace @@ -45,20 +46,23 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isFloat64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be Float64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), 1, getName()); arg = arguments[1].get(); if (!WhichDataType(arg).isFloat64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be Float64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), 2, getName()); arg = arguments[2].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(3) + " of function " + getName() + ". Must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 3, getName()); return std::make_shared(); } @@ -79,11 +83,14 @@ public: const double lat = col_lat->getFloat64(row); const UInt8 res = col_res->getUInt(row); - GeoCoord coord; - coord.lon = degsToRads(lon); + LatLng coord; + coord.lng = degsToRads(lon); coord.lat = degsToRads(lat); - H3Index hindex = geoToH3(&coord, res); + H3Index hindex; + H3Error err = latLngToCell(&coord, res, &hindex); + if (err) + throw Exception(ErrorCodes::INCORRECT_DATA, "Incorrect coordinates latitude: {}, longitude: {}, error: {}", coord.lat, coord.lng, err); dst_data[row] = hindex; } diff --git a/src/Functions/geoToS2.cpp b/src/Functions/geoToS2.cpp new file mode 100644 index 00000000000..c415cfade89 --- /dev/null +++ b/src/Functions/geoToS2.cpp @@ -0,0 +1,111 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +class S2CellId; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + +/** + * Accepts points of the form (longitude, latitude) + * Returns s2 identifier + */ +class FunctionGeoToS2 : public IFunction +{ +public: + static constexpr auto name = "geoToS2"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 2; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const auto * arg = arguments[i].get(); + if (!WhichDataType(arg).isFloat64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), i, getName()); + } + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_lon = arguments[0].column.get(); + const auto * col_lat = arguments[1].column.get(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.resize(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const Float64 lon = col_lon->getFloat64(row); + const Float64 lat = col_lat->getFloat64(row); + + if (isNaN(lon) || isNaN(lat)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments must not be NaN"); + + if (!(isFinite(lon) && isFinite(lat))) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Arguments must not be infinite"); + + /// S2 acceptes point as (latitude, longitude) + S2LatLng lat_lng = S2LatLng::FromDegrees(lat, lon); + S2CellId id(lat_lng); + + dst_data[row] = id.id(); + } + + return dst; + } + +}; + +} + +void registerFunctionGeoToS2(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/h3EdgeAngle.cpp b/src/Functions/h3EdgeAngle.cpp index 0fdafff9eed..aab8aeaf3a2 100644 --- a/src/Functions/h3EdgeAngle.cpp +++ b/src/Functions/h3EdgeAngle.cpp @@ -44,8 +44,9 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 1, getName()); return std::make_shared(); } @@ -62,11 +63,13 @@ public: { const int resolution = col_hindex->getUInt(row); if (resolution > MAX_H3_RES) - throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() - + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + resolution, getName(), MAX_H3_RES); // Numerical constant is 180 degrees / pi / Earth radius, Earth radius is from h3 sources - Float64 res = 8.99320592271288084e-6 * edgeLengthM(resolution); + Float64 res = 8.99320592271288084e-6 * getHexagonEdgeLengthAvgM(resolution); dst_data[row] = res; } diff --git a/src/Functions/h3EdgeLengthM.cpp b/src/Functions/h3EdgeLengthM.cpp index 5ec57510e54..3d745b21bd7 100644 --- a/src/Functions/h3EdgeLengthM.cpp +++ b/src/Functions/h3EdgeLengthM.cpp @@ -49,8 +49,9 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 1, getName()); return std::make_shared(); } @@ -67,10 +68,12 @@ public: { const UInt64 resolution = col_hindex->getUInt(row); if (resolution > MAX_H3_RES) - throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() - + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + resolution, getName(), MAX_H3_RES); - Float64 res = edgeLengthM(resolution); + Float64 res = getHexagonEdgeLengthAvgM(resolution); dst_data[row] = res; } diff --git a/src/Functions/h3GetBaseCell.cpp b/src/Functions/h3GetBaseCell.cpp index 7f3843ed792..4c424e4a1ab 100644 --- a/src/Functions/h3GetBaseCell.cpp +++ b/src/Functions/h3GetBaseCell.cpp @@ -41,8 +41,9 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); return std::make_shared(); } @@ -59,7 +60,7 @@ public: { const UInt64 hindex = col_hindex->getUInt(row); - UInt8 res = h3GetBaseCell(hindex); + UInt8 res = getBaseCellNumber(hindex); dst_data[row] = res; } diff --git a/src/Functions/h3GetResolution.cpp b/src/Functions/h3GetResolution.cpp index 074e07e4277..f387cdac2f0 100644 --- a/src/Functions/h3GetResolution.cpp +++ b/src/Functions/h3GetResolution.cpp @@ -41,8 +41,9 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); return std::make_shared(); } @@ -59,7 +60,7 @@ public: { const UInt64 hindex = col_hindex->getUInt(row); - UInt8 res = h3GetResolution(hindex); + UInt8 res = getResolution(hindex); dst_data[row] = res; } diff --git a/src/Functions/h3HexAreaM2.cpp b/src/Functions/h3HexAreaM2.cpp index e630fb7bd70..c4c6b5a57b2 100644 --- a/src/Functions/h3HexAreaM2.cpp +++ b/src/Functions/h3HexAreaM2.cpp @@ -44,8 +44,9 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 1, getName()); return std::make_shared(); } @@ -62,10 +63,12 @@ public: { const UInt64 resolution = col_hindex->getUInt(row); if (resolution > MAX_H3_RES) - throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() - + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + resolution, getName(), MAX_H3_RES); - Float64 res = hexAreaM2(resolution); + Float64 res = getHexagonAreaAvgM2(resolution); dst_data[row] = res; } diff --git a/src/Functions/h3IndexesAreNeighbors.cpp b/src/Functions/h3IndexesAreNeighbors.cpp index 3c03d3d1adb..2c9ceb9cc32 100644 --- a/src/Functions/h3IndexesAreNeighbors.cpp +++ b/src/Functions/h3IndexesAreNeighbors.cpp @@ -41,14 +41,16 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); arg = arguments[1].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 2, getName()); return std::make_shared(); } @@ -67,7 +69,7 @@ public: const UInt64 hindex_origin = col_hindex_origin->getUInt(row); const UInt64 hindex_dest = col_hindex_dest->getUInt(row); - UInt8 res = h3IndexesAreNeighbors(hindex_origin, hindex_dest); + UInt8 res = areNeighborCells(hindex_origin, hindex_dest); dst_data[row] = res; } diff --git a/src/Functions/h3IsValid.cpp b/src/Functions/h3IsValid.cpp index d7f5a2c0771..37ec2b99cd9 100644 --- a/src/Functions/h3IsValid.cpp +++ b/src/Functions/h3IsValid.cpp @@ -41,8 +41,9 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); return std::make_shared(); } @@ -59,7 +60,7 @@ public: { const UInt64 hindex = col_hindex->getUInt(row); - UInt8 is_valid = h3IsValid(hindex) == 0 ? 0 : 1; + UInt8 is_valid = isValidCell(hindex) == 0 ? 0 : 1; dst_data[row] = is_valid; } diff --git a/src/Functions/h3ToChildren.cpp b/src/Functions/h3ToChildren.cpp index d472c298432..d0d586cdf19 100644 --- a/src/Functions/h3ToChildren.cpp +++ b/src/Functions/h3ToChildren.cpp @@ -50,14 +50,16 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); arg = arguments[1].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 2, getName()); return std::make_shared(std::make_shared()); } @@ -81,17 +83,20 @@ public: const UInt8 child_resolution = col_resolution->getUInt(row); if (child_resolution > MAX_H3_RES) - throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName() - + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", + toString(child_resolution), getName(), toString(MAX_H3_RES)); - const size_t vec_size = maxH3ToChildrenSize(parent_hindex, child_resolution); + const size_t vec_size = cellToChildrenSize(parent_hindex, child_resolution); if (vec_size > MAX_ARRAY_SIZE) - throw Exception("The result of function" + getName() - + " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = " - + toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE); + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "The result of function {} (array of {} elements) will be too large with resolution argument = {}", + getName(), toString(vec_size), toString(child_resolution)); hindex_vec.resize(vec_size); - h3ToChildren(parent_hindex, child_resolution, hindex_vec.data()); + cellToChildren(parent_hindex, child_resolution, hindex_vec.data()); dst_data.reserve(dst_data.size() + vec_size); for (auto hindex : hindex_vec) diff --git a/src/Functions/h3ToParent.cpp b/src/Functions/h3ToParent.cpp index 6719d9f3456..0ec3df37e2e 100644 --- a/src/Functions/h3ToParent.cpp +++ b/src/Functions/h3ToParent.cpp @@ -44,14 +44,16 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); arg = arguments[1].get(); if (!WhichDataType(arg).isUInt8()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt8", + arg->getName(), 2, getName()); return std::make_shared(); } @@ -71,10 +73,12 @@ public: const UInt8 resolution = col_resolution->getUInt(row); if (resolution > MAX_H3_RES) - throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName() - + " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}", + toString(resolution), getName(), toString(MAX_H3_RES)); - UInt64 res = h3ToParent(hindex, resolution); + UInt64 res = cellToParent(hindex, resolution); dst_data[row] = res; } diff --git a/src/Functions/h3ToString.cpp b/src/Functions/h3ToString.cpp index dcd0951f67f..372afb97296 100644 --- a/src/Functions/h3ToString.cpp +++ b/src/Functions/h3ToString.cpp @@ -42,8 +42,9 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); return std::make_shared(); } @@ -66,17 +67,15 @@ public: { const UInt64 hindex = col_hindex->getUInt(i); - if (!h3IsValid(hindex)) - { - throw Exception("Invalid H3 index: " + std::to_string(hindex), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - } + if (!isValidCell(hindex)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Invalid H3 index: {}", hindex); + h3ToString(hindex, pos, H3_INDEX_STRING_LENGTH); // move to end of the index while (*pos != '\0') - { pos++; - } + vec_offsets[i] = ++pos - begin; } vec_res.resize(pos - begin); diff --git a/src/Functions/h3kRing.cpp b/src/Functions/h3kRing.cpp index b54ed48ef3f..583681e315e 100644 --- a/src/Functions/h3kRing.cpp +++ b/src/Functions/h3kRing.cpp @@ -47,14 +47,16 @@ public: const auto * arg = arguments[0].get(); if (!WhichDataType(arg).isUInt64()) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); arg = arguments[1].get(); if (!isInteger(arg)) throw Exception( - "Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be integer", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be integer", + arg->getName(), 2, getName()); return std::make_shared(std::make_shared()); } @@ -77,7 +79,7 @@ public: const H3Index origin_hindex = col_hindex->getUInt(row); const int k = col_k->getInt(row); - /// Overflow is possible. The function maxKringSize does not check for overflow. + /// Overflow is possible. The function maxGridDiskSize does not check for overflow. /// The calculation is similar to square of k but several times more. /// Let's use huge underestimation as the safe bound. We should not allow to generate too large arrays nevertheless. constexpr auto max_k = 10000; @@ -86,9 +88,9 @@ public: if (k < 0) throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Argument 'k' for {} function must be non negative", getName()); - const auto vec_size = maxKringSize(k); + const auto vec_size = maxGridDiskSize(k); hindex_vec.resize(vec_size); - kRing(origin_hindex, k, hindex_vec.data()); + gridDisk(origin_hindex, k, hindex_vec.data()); dst_data.reserve(dst_data.size() + vec_size); for (auto hindex : hindex_vec) diff --git a/src/Functions/h3toGeo.cpp b/src/Functions/h3toGeo.cpp new file mode 100644 index 00000000000..64facd1f010 --- /dev/null +++ b/src/Functions/h3toGeo.cpp @@ -0,0 +1,96 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_H3 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + +namespace +{ + +/// Implements the function h3ToGeo which takes a single argument (h3Index) +/// and returns the longitude and latitude that correspond to the provided h3 index +class FunctionH3ToGeo : public IFunction +{ +public: + static constexpr auto name = "h3ToGeo"; + + static FunctionPtr create(ContextPtr) { return std::make_shared(); } + + std::string getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 1; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), 1, getName()); + + return std::make_shared( + DataTypes{std::make_shared(), std::make_shared()}, + Strings{"longitude", "latitude"}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_index = arguments[0].column.get(); + + auto latitude = ColumnFloat64::create(input_rows_count); + auto longitude = ColumnFloat64::create(input_rows_count); + + ColumnFloat64::Container & lon_data = longitude->getData(); + ColumnFloat64::Container & lat_data = latitude->getData(); + + + for (size_t row = 0; row < input_rows_count; ++row) + { + H3Index h3index = col_index->getUInt(row); + LatLng coord{}; + + cellToLatLng(h3index,&coord); + lon_data[row] = radsToDegs(coord.lng); + lat_data[row] = radsToDegs(coord.lat); + } + + MutableColumns columns; + columns.emplace_back(std::move(longitude)); + columns.emplace_back(std::move(latitude)); + return ColumnTuple::create(std::move(columns)); + } +}; + +} + +void registerFunctionH3ToGeo(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/padString.cpp b/src/Functions/padString.cpp index 7711ab1a056..c03733a1198 100644 --- a/src/Functions/padString.cpp +++ b/src/Functions/padString.cpp @@ -89,6 +89,9 @@ namespace } /// Not necessary, but good for performance. + /// We repeat `pad_string` multiple times until it's length becomes 16 or more. + /// It speeds up the function appendTo() because it allows to copy padding characters by portions of at least + /// 16 bytes instead of single bytes. while (numCharsInPadString() < 16) { pad_string += pad_string; @@ -104,6 +107,12 @@ namespace } String pad_string; + + /// Offsets of code points in `pad_string`: + /// utf8_offsets[0] is the offset of the first code point in `pad_string`, it's always 0; + /// utf8_offsets[1] is the offset of the second code point in `pad_string`; + /// utf8_offsets[2] is the offset of the third code point in `pad_string`; + /// ... std::vector utf8_offsets; }; @@ -243,30 +252,32 @@ namespace const PaddingChars & padding_chars, StringSink & res_sink) const { - bool is_const_length = lengths.isConst(); - bool need_check_length = true; + bool is_const_new_length = lengths.isConst(); + size_t new_length = 0; + /// Insert padding characters to each string from `strings`, write the result strings into `res_sink`. + /// If for some input string its current length is greater than the specified new length then that string + /// will be trimmed to the specified new length instead of padding. for (; !res_sink.isEnd(); res_sink.next(), strings.next(), lengths.next()) { auto str = strings.getWhole(); size_t current_length = getLengthOfSlice(str); - auto new_length_slice = lengths.getWhole(); - size_t new_length = new_length_slice.elements->getUInt(new_length_slice.position); - - if (need_check_length) + if (!res_sink.rowNum() || !is_const_new_length) { + /// If `is_const_new_length` is true we can get and check the new length only once. + auto new_length_slice = lengths.getWhole(); + new_length = new_length_slice.elements->getUInt(new_length_slice.position); if (new_length > MAX_NEW_LENGTH) { throw Exception( "New padded length (" + std::to_string(new_length) + ") is too big, maximum is: " + std::to_string(MAX_NEW_LENGTH), ErrorCodes::TOO_LARGE_STRING_SIZE); } - if (is_const_length) + if (is_const_new_length) { size_t rows_count = res_sink.offsets.size(); res_sink.reserve((new_length + 1 /* zero terminator */) * rows_count); - need_check_length = false; } } diff --git a/src/Functions/registerFunctionsGeo.cpp b/src/Functions/registerFunctionsGeo.cpp index 605dd4dcba0..eb881870446 100644 --- a/src/Functions/registerFunctionsGeo.cpp +++ b/src/Functions/registerFunctionsGeo.cpp @@ -28,6 +28,7 @@ void registerFunctionSvg(FunctionFactory & factory); #if USE_H3 void registerFunctionGeoToH3(FunctionFactory &); +void registerFunctionH3ToGeo(FunctionFactory &); void registerFunctionH3EdgeAngle(FunctionFactory &); void registerFunctionH3EdgeLengthM(FunctionFactory &); void registerFunctionH3GetResolution(FunctionFactory &); @@ -42,6 +43,19 @@ void registerFunctionH3ToString(FunctionFactory &); void registerFunctionH3HexAreaM2(FunctionFactory &); #endif +#if USE_S2_GEOMETRY +void registerFunctionGeoToS2(FunctionFactory &); +void registerFunctionS2ToGeo(FunctionFactory &); +void registerFunctionS2GetNeighbors(FunctionFactory &); +void registerFunctionS2CellsIntersect(FunctionFactory &); +void registerFunctionS2CapContains(FunctionFactory &); +void registerFunctionS2CapUnion(FunctionFactory &); +void registerFunctionS2RectAdd(FunctionFactory &); +void registerFunctionS2RectContains(FunctionFactory &); +void registerFunctionS2RectUnion(FunctionFactory &); +void registerFunctionS2RectIntersection(FunctionFactory &); +#endif + void registerFunctionsGeo(FunctionFactory & factory) { @@ -66,6 +80,7 @@ void registerFunctionsGeo(FunctionFactory & factory) #if USE_H3 registerFunctionGeoToH3(factory); + registerFunctionH3ToGeo(factory); registerFunctionH3EdgeAngle(factory); registerFunctionH3EdgeLengthM(factory); registerFunctionH3GetResolution(factory); @@ -79,6 +94,19 @@ void registerFunctionsGeo(FunctionFactory & factory) registerFunctionH3ToString(factory); registerFunctionH3HexAreaM2(factory); #endif + +#if USE_S2_GEOMETRY + registerFunctionGeoToS2(factory); + registerFunctionS2ToGeo(factory); + registerFunctionS2GetNeighbors(factory); + registerFunctionS2CellsIntersect(factory); + registerFunctionS2CapContains(factory); + registerFunctionS2CapUnion(factory); + registerFunctionS2RectAdd(factory); + registerFunctionS2RectContains(factory); + registerFunctionS2RectUnion(factory); + registerFunctionS2RectIntersection(factory); +#endif } } diff --git a/src/Functions/s2CapContains.cpp b/src/Functions/s2CapContains.cpp new file mode 100644 index 00000000000..ce2abc14fad --- /dev/null +++ b/src/Functions/s2CapContains.cpp @@ -0,0 +1,132 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +/** + * The cap represents a portion of the sphere that has been cut off by a plane. + * It is defined by a point on a sphere and a radius in degrees. + * Imagine that we draw a line through the center of the sphere and our point. + * An infinite number of planes pass through this line, but any plane will intersect the cap in two points. + * Thus the angle is defined by one of this points and the entire line. + * So, the radius of Pi/2 defines a hemisphere and the radius of Pi defines a whole sphere. + * + * This function returns whether a cap contains a point. + */ +class FunctionS2CapContains : public IFunction +{ +public: + static constexpr auto name = "s2CapContains"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 3; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t index = 0; index < getNumberOfArguments(); ++index) + { + const auto * arg = arguments[index].get(); + + /// Radius + if (index == 1) + { + if (!WhichDataType(arg).isFloat64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), 2, getName()); + } + else if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), index + 1, getName()); + } + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_center = arguments[0].column.get(); + const auto * col_degrees = arguments[1].column.get(); + const auto * col_point = arguments[2].column.get(); + + auto dst = ColumnUInt8::create(); + auto & dst_data = dst->getData(); + dst_data.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const auto center = S2CellId(col_center->getUInt(row)); + const Float64 degrees = col_degrees->getFloat64(row); + const auto point = S2CellId(col_point->getUInt(row)); + + if (isNaN(degrees)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan"); + + if (std::isinf(degrees)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be infinite"); + + if (!center.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Center is not valid"); + + if (!point.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid"); + + S1Angle angle = S1Angle::Degrees(degrees); + S2Cap cap(center.ToPoint(), angle); + + dst_data.emplace_back(cap.Contains(point.ToPoint())); + } + + return dst; + } +}; + +} + +void registerFunctionS2CapContains(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2CapUnion.cpp b/src/Functions/s2CapUnion.cpp new file mode 100644 index 00000000000..4520f436161 --- /dev/null +++ b/src/Functions/s2CapUnion.cpp @@ -0,0 +1,141 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +/** + * The cap represents a portion of the sphere that has been cut off by a plane. + * See comment for s2CapContains function. + * This function returns the smallest cap that contains both of input caps. + * It is represented by identifier of the center and a radius. + */ +class FunctionS2CapUnion : public IFunction +{ +public: + static constexpr auto name = "s2CapUnion"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 4; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t index = 0; index < getNumberOfArguments(); ++index) + { + const auto * arg = arguments[index].get(); + if (index == 1 || index == 3) + { + if (!WhichDataType(arg).isFloat64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), index + 1, getName()); + } + else if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), index + 1, getName() + ); + } + + DataTypePtr center = std::make_shared(); + DataTypePtr radius = std::make_shared(); + + return std::make_shared(DataTypes{center, radius}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_center1 = arguments[0].column.get(); + const auto * col_radius1 = arguments[1].column.get(); + const auto * col_center2 = arguments[2].column.get(); + const auto * col_radius2 = arguments[3].column.get(); + + auto col_res_center = ColumnUInt64::create(); + auto col_res_radius = ColumnFloat64::create(); + + auto & vec_res_center = col_res_center->getData(); + vec_res_center.reserve(input_rows_count); + + auto & vec_res_radius = col_res_radius->getData(); + vec_res_radius.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const UInt64 first_center = col_center1->getUInt(row); + const Float64 first_radius = col_radius1->getFloat64(row); + const UInt64 second_center = col_center2->getUInt(row); + const Float64 second_radius = col_radius2->getFloat64(row); + + if (isNaN(first_radius) || isNaN(second_radius)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan"); + + if (std::isinf(first_radius) || std::isinf(second_radius)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be infinite"); + + auto first_center_cell = S2CellId(first_center); + auto second_center_cell = S2CellId(second_center); + + if (!first_center_cell.is_valid() || !second_center_cell.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Center of the cap is not valid"); + + S2Cap cap1(first_center_cell.ToPoint(), S1Angle::Degrees(first_radius)); + S2Cap cap2(second_center_cell.ToPoint(), S1Angle::Degrees(second_radius)); + + S2Cap cap_union = cap1.Union(cap2); + + vec_res_center.emplace_back(S2CellId(cap_union.center()).id()); + vec_res_radius.emplace_back(cap_union.GetRadius().degrees()); + } + + return ColumnTuple::create(Columns{std::move(col_res_center), std::move(col_res_radius)}); + } + +}; + +} + +void registerFunctionS2CapUnion(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2CellsIntersect.cpp b/src/Functions/s2CellsIntersect.cpp new file mode 100644 index 00000000000..3d25fdbe44d --- /dev/null +++ b/src/Functions/s2CellsIntersect.cpp @@ -0,0 +1,104 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +/** + * Each cell in s2 library is a quadrilateral bounded by four geodesics. + */ +class FunctionS2CellsIntersect : public IFunction +{ +public: + static constexpr auto name = "s2CellsIntersect"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 2; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const auto * arg = arguments[i].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), i, getName()); + } + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_id_first = arguments[0].column.get(); + const auto * col_id_second = arguments[1].column.get(); + + auto dst = ColumnUInt8::create(); + auto & dst_data = dst->getData(); + dst_data.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const UInt64 id_first = col_id_first->getInt(row); + const UInt64 id_second = col_id_second->getInt(row); + + auto first_cell = S2CellId(id_first); + auto second_cell = S2CellId(id_second); + + if (!first_cell.is_valid() || !second_cell.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cell is not valid"); + + dst_data.emplace_back(S2CellId(id_first).intersects(S2CellId(id_second))); + } + + return dst; + } + +}; + +} + +void registerFunctionS2CellsIntersect(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2GetNeighbors.cpp b/src/Functions/s2GetNeighbors.cpp new file mode 100644 index 00000000000..8da0777a4ef --- /dev/null +++ b/src/Functions/s2GetNeighbors.cpp @@ -0,0 +1,111 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +/** + * Each cell in s2 library is a quadrilateral bounded by four geodesics. + * So, each cell has 4 neighbors + */ +class FunctionS2GetNeighbors : public IFunction +{ +public: + static constexpr auto name = "s2GetNeighbors"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 1; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), 1, getName()); + + return std::make_shared(std::make_shared()); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_id = arguments[0].column.get(); + + auto dst = ColumnArray::create(ColumnUInt64::create()); + auto & dst_data = dst->getData(); + auto & dst_offsets = dst->getOffsets(); + dst_offsets.resize(input_rows_count); + size_t current_offset = 0; + + for (const auto row : collections::range(0, input_rows_count)) + { + const UInt64 id = col_id->getUInt(row); + + S2CellId cell_id(id); + + if (!cell_id.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cell is not valid"); + + S2CellId neighbors[4]; + cell_id.GetEdgeNeighbors(neighbors); + + dst_data.reserve(dst_data.size() + 4); + for (auto & neighbor : neighbors) + { + ++current_offset; + dst_data.insert(neighbor.id()); + } + dst_offsets[row] = current_offset; + } + + return dst; + } + +}; + +} + +void registerFunctionS2GetNeighbors(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2RectAdd.cpp b/src/Functions/s2RectAdd.cpp new file mode 100644 index 00000000000..ceceb11da05 --- /dev/null +++ b/src/Functions/s2RectAdd.cpp @@ -0,0 +1,115 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +class FunctionS2RectAdd : public IFunction +{ +public: + static constexpr auto name = "s2RectAdd"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 4; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t index = 0; index < getNumberOfArguments(); ++index) + { + const auto * arg = arguments[index].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), index, getName()); + } + + DataTypePtr element = std::make_shared(); + + return std::make_shared(DataTypes{element, element}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_lo = arguments[0].column.get(); + const auto * col_hi = arguments[1].column.get(); + const auto * col_point = arguments[2].column.get(); + + auto col_res_first = ColumnUInt64::create(); + auto col_res_second = ColumnUInt64::create(); + + auto & vec_res_first = col_res_first->getData(); + vec_res_first.reserve(input_rows_count); + + auto & vec_res_second = col_res_second->getData(); + vec_res_second.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const auto lo = S2CellId(col_lo->getUInt(row)); + const auto hi = S2CellId(col_hi->getUInt(row)); + const auto point = S2CellId(col_point->getUInt(row)); + + if (!lo.is_valid() || !hi.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid"); + + if (!point.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid"); + + S2LatLngRect rect(lo.ToLatLng(), hi.ToLatLng()); + + rect.AddPoint(point.ToPoint()); + + vec_res_first.emplace_back(S2CellId(rect.lo()).id()); + vec_res_second.emplace_back(S2CellId(rect.hi()).id()); + } + + return ColumnTuple::create(Columns{std::move(col_res_first), std::move(col_res_second)}); + } + +}; + +} + +void registerFunctionS2RectAdd(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2RectContains.cpp b/src/Functions/s2RectContains.cpp new file mode 100644 index 00000000000..2b4ae31a6b2 --- /dev/null +++ b/src/Functions/s2RectContains.cpp @@ -0,0 +1,105 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +class FunctionS2RectContains : public IFunction +{ +public: + static constexpr auto name = "s2RectContains"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 4; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const auto * arg = arguments[i].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), i, getName()); + } + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_lo = arguments[0].column.get(); + const auto * col_hi = arguments[1].column.get(); + const auto * col_point = arguments[2].column.get(); + + auto dst = ColumnVector::create(); + auto & dst_data = dst->getData(); + dst_data.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const auto lo = S2CellId(col_lo->getUInt(row)); + const auto hi = S2CellId(col_hi->getUInt(row)); + const auto point = S2CellId(col_point->getUInt(row)); + + if (!lo.is_valid() || !hi.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid"); + + if (!point.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid"); + + S2LatLngRect rect(lo.ToLatLng(), hi.ToLatLng()); + + dst_data.emplace_back(rect.Contains(point.ToLatLng())); + } + + return dst; + } + +}; + +} + +void registerFunctionS2RectContains(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2RectIntersection.cpp b/src/Functions/s2RectIntersection.cpp new file mode 100644 index 00000000000..f106167247b --- /dev/null +++ b/src/Functions/s2RectIntersection.cpp @@ -0,0 +1,121 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +class S2CellId; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + + +class FunctionS2RectIntersection : public IFunction +{ +public: + static constexpr auto name = "s2RectIntersection"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 4; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const auto * arg = arguments[i].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), i, getName()); + } + + DataTypePtr element = std::make_shared(); + + return std::make_shared(DataTypes{element, element}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_lo1 = arguments[0].column.get(); + const auto * col_hi1 = arguments[1].column.get(); + const auto * col_lo2 = arguments[2].column.get(); + const auto * col_hi2 = arguments[3].column.get(); + + auto col_res_first = ColumnUInt64::create(); + auto col_res_second = ColumnUInt64::create(); + + auto & vec_res_first = col_res_first->getData(); + vec_res_first.reserve(input_rows_count); + + auto & vec_res_second = col_res_second->getData(); + vec_res_second.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const auto lo1 = S2CellId(col_lo1->getUInt(row)); + const auto hi1 = S2CellId(col_hi1->getUInt(row)); + const auto lo2 = S2CellId(col_lo2->getUInt(row)); + const auto hi2 = S2CellId(col_hi2->getUInt(row)); + + if (!lo1.is_valid() || !hi1.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "First rectangle is not valid"); + + if (!lo2.is_valid() || !hi2.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second rectangle is not valid"); + + S2LatLngRect rect1(lo1.ToLatLng(), hi1.ToLatLng()); + S2LatLngRect rect2(lo2.ToLatLng(), hi2.ToLatLng()); + + S2LatLngRect rect_intersection = rect1.Intersection(rect2); + + vec_res_first.emplace_back(S2CellId(rect_intersection.lo()).id()); + vec_res_second.emplace_back(S2CellId(rect_intersection.hi()).id()); + } + + return ColumnTuple::create(Columns{std::move(col_res_first), std::move(col_res_second)}); + } + +}; + +} + +void registerFunctionS2RectIntersection(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2RectUnion.cpp b/src/Functions/s2RectUnion.cpp new file mode 100644 index 00000000000..387d8b25f29 --- /dev/null +++ b/src/Functions/s2RectUnion.cpp @@ -0,0 +1,119 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + + +class FunctionS2RectUnion : public IFunction +{ +public: + static constexpr auto name = "s2RectUnion"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 4; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + for (size_t i = 0; i < getNumberOfArguments(); ++i) + { + const auto * arg = arguments[i].get(); + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), i + 1, getName()); + } + + DataTypePtr element = std::make_shared(); + + return std::make_shared(DataTypes{element, element}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_lo1 = arguments[0].column.get(); + const auto * col_hi1 = arguments[1].column.get(); + const auto * col_lo2 = arguments[2].column.get(); + const auto * col_hi2 = arguments[3].column.get(); + + auto col_res_first = ColumnUInt64::create(); + auto col_res_second = ColumnUInt64::create(); + + auto & vec_res_first = col_res_first->getData(); + vec_res_first.reserve(input_rows_count); + + auto & vec_res_second = col_res_second->getData(); + vec_res_second.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const auto lo1 = S2CellId(col_lo1->getUInt(row)); + const auto hi1 = S2CellId(col_hi1->getUInt(row)); + const auto lo2 = S2CellId(col_lo2->getUInt(row)); + const auto hi2 = S2CellId(col_hi2->getUInt(row)); + + if (!lo1.is_valid() || !hi1.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "First rectangle is not valid"); + + if (!lo2.is_valid() || !hi2.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second rectangle is not valid"); + + S2LatLngRect rect1(lo1.ToLatLng(), hi1.ToLatLng()); + S2LatLngRect rect2(lo2.ToLatLng(), hi2.ToLatLng()); + + S2LatLngRect rect_union = rect1.Union(rect2); + + vec_res_first.emplace_back(S2CellId(rect_union.lo()).id()); + vec_res_second.emplace_back(S2CellId(rect_union.hi()).id()); + } + + return ColumnTuple::create(Columns{std::move(col_res_first), std::move(col_res_second)}); + } + +}; + +} + +void registerFunctionS2RectUnion(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2ToGeo.cpp b/src/Functions/s2ToGeo.cpp new file mode 100644 index 00000000000..98f71e898bd --- /dev/null +++ b/src/Functions/s2ToGeo.cpp @@ -0,0 +1,110 @@ +#if !defined(ARCADIA_BUILD) +# include "config_functions.h" +#endif + +#if USE_S2_GEOMETRY + +#include +#include +#include +#include +#include +#include +#include + +#include "s2_fwd.h" + +class S2CellId; + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +/** + * Returns a point (longitude, latitude) in degrees + */ +class FunctionS2ToGeo : public IFunction +{ +public: + static constexpr auto name = "s2ToGeo"; + + static FunctionPtr create(ContextPtr) + { + return std::make_shared(); + } + + std::string getName() const override + { + return name; + } + + size_t getNumberOfArguments() const override { return 1; } + + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override + { + const auto * arg = arguments[0].get(); + + if (!WhichDataType(arg).isUInt64()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Illegal type {} of argument {} of function {}. Must be Float64", + arg->getName(), 1, getName()); + + DataTypePtr element = std::make_shared(); + + return std::make_shared(DataTypes{element, element}); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto * col_id = arguments[0].column.get(); + + auto col_longitude = ColumnFloat64::create(); + auto col_latitude = ColumnFloat64::create(); + + auto & longitude = col_longitude->getData(); + longitude.reserve(input_rows_count); + + auto & latitude = col_latitude->getData(); + latitude.reserve(input_rows_count); + + for (const auto row : collections::range(0, input_rows_count)) + { + const auto id = S2CellId(col_id->getUInt(row)); + + if (!id.is_valid()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid"); + + S2Point point = id.ToPoint(); + S2LatLng ll(point); + + longitude.emplace_back(ll.lng().degrees()); + latitude.emplace_back(ll.lat().degrees()); + } + + return ColumnTuple::create(Columns{std::move(col_longitude), std::move(col_latitude)}); + } + +}; + +} + +void registerFunctionS2ToGeo(FunctionFactory & factory) +{ + factory.registerFunction(); +} + + +} + +#endif diff --git a/src/Functions/s2_fwd.h b/src/Functions/s2_fwd.h new file mode 100644 index 00000000000..e3f7026e48c --- /dev/null +++ b/src/Functions/s2_fwd.h @@ -0,0 +1,16 @@ +#pragma once +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wambiguous-reversed-operator" +#endif + +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE +#include // Y_IGNORE + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif diff --git a/src/Functions/toTimezone.cpp b/src/Functions/toTimezone.cpp index 551e07a8354..4bb5ab47659 100644 --- a/src/Functions/toTimezone.cpp +++ b/src/Functions/toTimezone.cpp @@ -19,20 +19,70 @@ namespace ErrorCodes namespace { +class ExecutableFunctionToTimeZone : public IExecutableFunction +{ +public: + explicit ExecutableFunctionToTimeZone() = default; + + String getName() const override { return "toTimezone"; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override + { + return arguments[0].column; + } +}; + +class FunctionBaseToTimeZone : public IFunctionBase +{ +public: + FunctionBaseToTimeZone( + bool is_constant_timezone_, + DataTypes argument_types_, + DataTypePtr return_type_) + : is_constant_timezone(is_constant_timezone_) + , argument_types(std::move(argument_types_)) + , return_type(std::move(return_type_)) {} + + String getName() const override { return "toTimezone"; } + + const DataTypes & getArgumentTypes() const override + { + return argument_types; + } + + const DataTypePtr & getResultType() const override + { + return return_type; + } + + ExecutableFunctionPtr prepare(const ColumnsWithTypeAndName & /*arguments*/) const override + { + return std::make_unique(); + } + + bool hasInformationAboutMonotonicity() const override { return is_constant_timezone; } + + Monotonicity getMonotonicityForRange(const IDataType & /*type*/, const Field & /*left*/, const Field & /*right*/) const override + { + return {is_constant_timezone, is_constant_timezone, is_constant_timezone}; + } + +private: + bool is_constant_timezone; + DataTypes argument_types; + DataTypePtr return_type; +}; /// Just changes time zone information for data type. The calculation is free. -class FunctionToTimezone : public IFunction +class ToTimeZoneOverloadResolver : public IFunctionOverloadResolver { public: static constexpr auto name = "toTimezone"; - static FunctionPtr create(ContextPtr) { return std::make_shared(); } - String getName() const override - { - return name; - } + String getName() const override { return name; } size_t getNumberOfArguments() const override { return 2; } + static FunctionOverloadResolverPtr create(ContextPtr) { return std::make_unique(); } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -54,9 +104,17 @@ public: return std::make_shared(date_time64->getScale(), time_zone_name); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override + FunctionBasePtr buildImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override { - return arguments[0].column; + bool is_constant_timezone = false; + if (arguments[1].column) + is_constant_timezone = isColumnConst(*arguments[1].column); + + DataTypes data_types(arguments.size()); + for (size_t i = 0; i < arguments.size(); ++i) + data_types[i] = arguments[i].type; + + return std::make_unique(is_constant_timezone, data_types, result_type); } }; @@ -64,7 +122,7 @@ public: void registerFunctionToTimeZone(FunctionFactory & factory) { - factory.registerFunction(); + factory.registerFunction(); factory.registerAlias("toTimeZone", "toTimezone"); } diff --git a/src/Functions/ya.make b/src/Functions/ya.make index 5f84511aa52..7955d4091e9 100644 --- a/src/Functions/ya.make +++ b/src/Functions/ya.make @@ -4,7 +4,7 @@ OWNER(g:clickhouse) LIBRARY() CFLAGS( - -DUSE_H3 -DUSE_SSL -DUSE_XXHASH + -DUSE_SSL -DUSE_XXHASH ) ADDINCL( @@ -277,6 +277,7 @@ SRCS( gcd.cpp generateUUIDv4.cpp geoToH3.cpp + geoToS2.cpp geohashDecode.cpp geohashEncode.cpp geohashesInBox.cpp @@ -300,6 +301,7 @@ SRCS( h3ToParent.cpp h3ToString.cpp h3kRing.cpp + h3toGeo.cpp hasColumnInTable.cpp hasThreadFuzzer.cpp hasToken.cpp @@ -455,6 +457,15 @@ SRCS( runningConcurrency.cpp runningDifference.cpp runningDifferenceStartingWithFirstValue.cpp + s2CapContains.cpp + s2CapUnion.cpp + s2CellsIntersect.cpp + s2GetNeighbors.cpp + s2RectAdd.cpp + s2RectContains.cpp + s2RectIntersection.cpp + s2RectUnion.cpp + s2ToGeo.cpp sigmoid.cpp sign.cpp sin.cpp diff --git a/src/Functions/ya.make.in b/src/Functions/ya.make.in index f75773fb47e..b21bf64304a 100644 --- a/src/Functions/ya.make.in +++ b/src/Functions/ya.make.in @@ -3,7 +3,7 @@ OWNER(g:clickhouse) LIBRARY() CFLAGS( - -DUSE_H3 -DUSE_SSL -DUSE_XXHASH + -DUSE_SSL -DUSE_XXHASH ) ADDINCL( diff --git a/src/IO/FileEncryptionCommon.cpp b/src/IO/FileEncryptionCommon.cpp new file mode 100644 index 00000000000..9cbc8ff0f3c --- /dev/null +++ b/src/IO/FileEncryptionCommon.cpp @@ -0,0 +1,269 @@ +#include + +#if USE_SSL +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int DATA_ENCRYPTION_ERROR; +} + +namespace FileEncryption +{ + +namespace +{ + String toBigEndianString(UInt128 value) + { + WriteBufferFromOwnString out; + writeBinaryBigEndian(value, out); + return std::move(out.str()); + } + + UInt128 fromBigEndianString(const String & str) + { + ReadBufferFromMemory in{str.data(), str.length()}; + UInt128 result; + readBinaryBigEndian(result, in); + return result; + } +} + +InitVector::InitVector(const String & iv_) : iv(fromBigEndianString(iv_)) {} + +const String & InitVector::str() const +{ + local = toBigEndianString(iv + counter); + return local; +} + +Encryption::Encryption(const String & iv_, const EncryptionKey & key_, size_t offset_) + : evp_cipher(defaultCipher()) + , init_vector(iv_) + , key(key_) + , block_size(cipherIVLength(evp_cipher)) +{ + if (iv_.size() != cipherIVLength(evp_cipher)) + throw DB::Exception("Expected iv with size " + std::to_string(cipherIVLength(evp_cipher)) + ", got iv with size " + std::to_string(iv_.size()), + DB::ErrorCodes::DATA_ENCRYPTION_ERROR); + if (key_.size() != cipherKeyLength(evp_cipher)) + throw DB::Exception("Expected key with size " + std::to_string(cipherKeyLength(evp_cipher)) + ", got iv with size " + std::to_string(key_.size()), + DB::ErrorCodes::DATA_ENCRYPTION_ERROR); + + offset = offset_; +} + +size_t Encryption::partBlockSize(size_t size, size_t off) const +{ + assert(off < block_size); + /// write the part as usual block + if (off == 0) + return 0; + return off + size <= block_size ? size : (block_size - off) % block_size; +} + +void Encryptor::encrypt(const char * plaintext, WriteBuffer & buf, size_t size) +{ + if (!size) + return; + + auto iv = InitVector(init_vector); + auto off = blockOffset(offset); + iv.set(blocks(offset)); + + size_t part_size = partBlockSize(size, off); + if (off) + { + buf.write(encryptPartialBlock(plaintext, part_size, iv, off).data(), part_size); + offset += part_size; + size -= part_size; + iv.inc(); + } + + if (size) + { + buf.write(encryptNBytes(plaintext + part_size, size, iv).data(), size); + offset += size; + } +} + +String Encryptor::encryptPartialBlock(const char * partial_block, size_t size, const InitVector & iv, size_t off) const +{ + if (size > block_size) + throw Exception("Expected partial block, got block with size > block_size: size = " + std::to_string(size) + " and offset = " + std::to_string(off), + ErrorCodes::DATA_ENCRYPTION_ERROR); + + String plaintext(block_size, '\0'); + for (size_t i = 0; i < size; ++i) + plaintext[i + off] = partial_block[i]; + + return String(encryptNBytes(plaintext.data(), block_size, iv), off, size); +} + +String Encryptor::encryptNBytes(const char * data, size_t bytes, const InitVector & iv) const +{ + String ciphertext(bytes, '\0'); + auto * ciphertext_ref = ciphertext.data(); + + auto evp_ctx_ptr = std::unique_ptr(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free); + auto * evp_ctx = evp_ctx_ptr.get(); + + if (EVP_EncryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) + throw Exception("Failed to initialize encryption context with cipher", ErrorCodes::DATA_ENCRYPTION_ERROR); + + if (EVP_EncryptInit_ex(evp_ctx, nullptr, nullptr, + reinterpret_cast(key.str().data()), + reinterpret_cast(iv.str().data())) != 1) + throw Exception("Failed to set key and IV for encryption", ErrorCodes::DATA_ENCRYPTION_ERROR); + + int output_len = 0; + if (EVP_EncryptUpdate(evp_ctx, + reinterpret_cast(ciphertext_ref), &output_len, + reinterpret_cast(data), static_cast(bytes)) != 1) + throw Exception("Failed to encrypt", ErrorCodes::DATA_ENCRYPTION_ERROR); + + ciphertext_ref += output_len; + + int final_output_len = 0; + if (EVP_EncryptFinal_ex(evp_ctx, + reinterpret_cast(ciphertext_ref), &final_output_len) != 1) + throw Exception("Failed to fetch ciphertext", ErrorCodes::DATA_ENCRYPTION_ERROR); + + if (output_len < 0 || final_output_len < 0 || static_cast(output_len) + static_cast(final_output_len) != bytes) + throw Exception("Only part of the data was encrypted", ErrorCodes::DATA_ENCRYPTION_ERROR); + + return ciphertext; +} + +void Decryptor::decrypt(const char * ciphertext, BufferBase::Position buf, size_t size, size_t off) +{ + if (!size) + return; + + auto iv = InitVector(init_vector); + iv.set(blocks(off)); + off = blockOffset(off); + + size_t part_size = partBlockSize(size, off); + if (off) + { + decryptPartialBlock(buf, ciphertext, part_size, iv, off); + size -= part_size; + if (part_size + off == block_size) + iv.inc(); + } + + if (size) + decryptNBytes(buf, ciphertext + part_size, size, iv); +} + +void Decryptor::decryptPartialBlock(BufferBase::Position & to, const char * partial_block, size_t size, const InitVector & iv, size_t off) const +{ + if (size > block_size) + throw Exception("Expecter partial block, got block with size > block_size: size = " + std::to_string(size) + " and offset = " + std::to_string(off), + ErrorCodes::DATA_ENCRYPTION_ERROR); + + String ciphertext(block_size, '\0'); + String plaintext(block_size, '\0'); + for (size_t i = 0; i < size; ++i) + ciphertext[i + off] = partial_block[i]; + + auto * plaintext_ref = plaintext.data(); + decryptNBytes(plaintext_ref, ciphertext.data(), off + size, iv); + + for (size_t i = 0; i < size; ++i) + *(to++) = plaintext[i + off]; +} + +void Decryptor::decryptNBytes(BufferBase::Position & to, const char * data, size_t bytes, const InitVector & iv) const +{ + auto evp_ctx_ptr = std::unique_ptr(EVP_CIPHER_CTX_new(), &EVP_CIPHER_CTX_free); + auto * evp_ctx = evp_ctx_ptr.get(); + + if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) + throw Exception("Failed to initialize decryption context with cipher", ErrorCodes::DATA_ENCRYPTION_ERROR); + + if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr, + reinterpret_cast(key.str().data()), + reinterpret_cast(iv.str().data())) != 1) + throw Exception("Failed to set key and IV for decryption", ErrorCodes::DATA_ENCRYPTION_ERROR); + + int output_len = 0; + if (EVP_DecryptUpdate(evp_ctx, + reinterpret_cast(to), &output_len, + reinterpret_cast(data), static_cast(bytes)) != 1) + throw Exception("Failed to decrypt", ErrorCodes::DATA_ENCRYPTION_ERROR); + + to += output_len; + + int final_output_len = 0; + if (EVP_DecryptFinal_ex(evp_ctx, + reinterpret_cast(to), &final_output_len) != 1) + throw Exception("Failed to fetch plaintext", ErrorCodes::DATA_ENCRYPTION_ERROR); + + if (output_len < 0 || final_output_len < 0 || static_cast(output_len) + static_cast(final_output_len) != bytes) + throw Exception("Only part of the data was decrypted", ErrorCodes::DATA_ENCRYPTION_ERROR); +} + +String readIV(size_t size, ReadBuffer & in) +{ + String iv(size, 0); + in.readStrict(reinterpret_cast(iv.data()), size); + return iv; +} + +String randomString(size_t size) +{ + String iv(size, 0); + + std::random_device rd; + std::mt19937 gen{rd()}; + std::uniform_int_distribution dis; + + char * ptr = iv.data(); + while (size) + { + auto value = dis(gen); + size_t n = std::min(size, sizeof(value)); + memcpy(ptr, &value, n); + ptr += n; + size -= n; + } + + return iv; +} + +void writeIV(const String & iv, WriteBuffer & out) +{ + out.write(iv.data(), iv.length()); +} + +size_t cipherKeyLength(const EVP_CIPHER * evp_cipher) +{ + return static_cast(EVP_CIPHER_key_length(evp_cipher)); +} + +size_t cipherIVLength(const EVP_CIPHER * evp_cipher) +{ + return static_cast(EVP_CIPHER_iv_length(evp_cipher)); +} + +const EVP_CIPHER * defaultCipher() +{ + return EVP_aes_128_ctr(); +} + +} +} + +#endif diff --git a/src/IO/FileEncryptionCommon.h b/src/IO/FileEncryptionCommon.h new file mode 100644 index 00000000000..f40de99faf6 --- /dev/null +++ b/src/IO/FileEncryptionCommon.h @@ -0,0 +1,104 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include +#endif + +#if USE_SSL +#include +#include + +namespace DB +{ +class ReadBuffer; +class WriteBuffer; + +namespace FileEncryption +{ + +constexpr size_t kIVSize = sizeof(UInt128); + +class InitVector +{ +public: + InitVector(const String & iv_); + const String & str() const; + void inc() { ++counter; } + void inc(size_t n) { counter += n; } + void set(size_t n) { counter = n; } + +private: + UInt128 iv; + UInt128 counter = 0; + mutable String local; +}; + + +class EncryptionKey +{ +public: + EncryptionKey(const String & key_) : key(key_) { } + size_t size() const { return key.size(); } + const String & str() const { return key; } + +private: + String key; +}; + + +class Encryption +{ +public: + Encryption(const String & iv_, const EncryptionKey & key_, size_t offset_); + +protected: + size_t blockOffset(size_t pos) const { return pos % block_size; } + size_t blocks(size_t pos) const { return pos / block_size; } + size_t partBlockSize(size_t size, size_t off) const; + const EVP_CIPHER * get() const { return evp_cipher; } + + const EVP_CIPHER * evp_cipher; + const String init_vector; + const EncryptionKey key; + size_t block_size; + + /// absolute offset + size_t offset = 0; +}; + + +class Encryptor : public Encryption +{ +public: + using Encryption::Encryption; + void encrypt(const char * plaintext, WriteBuffer & buf, size_t size); + +private: + String encryptPartialBlock(const char * partial_block, size_t size, const InitVector & iv, size_t off) const; + String encryptNBytes(const char * data, size_t bytes, const InitVector & iv) const; +}; + + +class Decryptor : public Encryption +{ +public: + Decryptor(const String & iv_, const EncryptionKey & key_) : Encryption(iv_, key_, 0) { } + void decrypt(const char * ciphertext, char * buf, size_t size, size_t off); + +private: + void decryptPartialBlock(char *& to, const char * partial_block, size_t size, const InitVector & iv, size_t off) const; + void decryptNBytes(char *& to, const char * data, size_t bytes, const InitVector & iv) const; +}; + + +String readIV(size_t size, ReadBuffer & in); +String randomString(size_t size); +void writeIV(const String & iv, WriteBuffer & out); +size_t cipherKeyLength(const EVP_CIPHER * evp_cipher); +size_t cipherIVLength(const EVP_CIPHER * evp_cipher); +const EVP_CIPHER * defaultCipher(); + +} +} + +#endif diff --git a/src/IO/HashingReadBuffer.h b/src/IO/HashingReadBuffer.h index 08b6de69dcb..5d42c64478c 100644 --- a/src/IO/HashingReadBuffer.h +++ b/src/IO/HashingReadBuffer.h @@ -34,7 +34,7 @@ private: working_buffer = in.buffer(); pos = in.position(); - // `pos` may be different from working_buffer.begin() when using AIO. + // `pos` may be different from working_buffer.begin() when using sophisticated ReadBuffers. calculateHash(pos, working_buffer.end() - pos); return res; diff --git a/src/IO/ReadBufferAIO.cpp b/src/IO/ReadBufferAIO.cpp deleted file mode 100644 index c064e0d4ed9..00000000000 --- a/src/IO/ReadBufferAIO.cpp +++ /dev/null @@ -1,312 +0,0 @@ -#if defined(OS_LINUX) || defined(__FreeBSD__) - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include - - -namespace ProfileEvents -{ - extern const Event FileOpen; - extern const Event ReadBufferAIORead; - extern const Event ReadBufferAIOReadBytes; -} - -namespace CurrentMetrics -{ - extern const Metric Read; -} - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int FILE_DOESNT_EXIST; - extern const int CANNOT_OPEN_FILE; - extern const int LOGICAL_ERROR; - extern const int ARGUMENT_OUT_OF_BOUND; - extern const int AIO_READ_ERROR; -} - - -/// Note: an additional page is allocated that will contain the data that -/// does not fit into the main buffer. -ReadBufferAIO::ReadBufferAIO(const std::string & filename_, size_t buffer_size_, int flags_, char * existing_memory_) - : ReadBufferFromFileBase(buffer_size_ + DEFAULT_AIO_FILE_BLOCK_SIZE, existing_memory_, DEFAULT_AIO_FILE_BLOCK_SIZE), - fill_buffer(BufferWithOwnMemory(internalBuffer().size(), nullptr, DEFAULT_AIO_FILE_BLOCK_SIZE)), - filename(filename_) -{ - ProfileEvents::increment(ProfileEvents::FileOpen); - - int open_flags = (flags_ == -1) ? O_RDONLY : flags_; - open_flags |= O_DIRECT; - open_flags |= O_CLOEXEC; - - fd = ::open(filename.c_str(), open_flags); - if (fd == -1) - { - auto error_code = (errno == ENOENT) ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE; - throwFromErrnoWithPath("Cannot open file " + filename, filename, error_code); - } -} - -ReadBufferAIO::~ReadBufferAIO() -{ - if (!aio_failed) - { - try - { - (void) waitForAIOCompletion(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - - if (fd != -1) - ::close(fd); -} - -void ReadBufferAIO::setMaxBytes(size_t max_bytes_read_) -{ - if (is_started) - throw Exception("Illegal attempt to set the maximum number of bytes to read from file " + filename, ErrorCodes::LOGICAL_ERROR); - max_bytes_read = max_bytes_read_; -} - -bool ReadBufferAIO::nextImpl() -{ - /// If the end of the file has already been reached by calling this function, - /// then the current call is wrong. - if (is_eof) - return false; - - std::optional watch; - if (profile_callback) - watch.emplace(clock_type); - - if (!is_pending_read) - synchronousRead(); - else - receive(); - - if (profile_callback) - { - ProfileInfo info; - info.bytes_requested = requested_byte_count; - info.bytes_read = bytes_read; - info.nanoseconds = watch->elapsed(); //-V1007 - profile_callback(info); - } - - is_started = true; - - /// If the end of the file is just reached, do nothing else. - if (is_eof) - return bytes_read != 0; - - /// Create an asynchronous request. - prepare(); - -#if defined(__FreeBSD__) - request.aio.aio_lio_opcode = LIO_READ; - request.aio.aio_fildes = fd; - request.aio.aio_buf = reinterpret_cast(buffer_begin); - request.aio.aio_nbytes = region_aligned_size; - request.aio.aio_offset = region_aligned_begin; -#else - request.aio_lio_opcode = IOCB_CMD_PREAD; - request.aio_fildes = fd; - request.aio_buf = reinterpret_cast(buffer_begin); - request.aio_nbytes = region_aligned_size; - request.aio_offset = region_aligned_begin; -#endif - - /// Send the request. - try - { - future_bytes_read = AIOContextPool::instance().post(request); - } - catch (...) - { - aio_failed = true; - throw; - } - - is_pending_read = true; - return true; -} - -off_t ReadBufferAIO::seek(off_t off, int whence) -{ - off_t new_pos_in_file; - - if (whence == SEEK_SET) - { - if (off < 0) - throw Exception("SEEK_SET underflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - new_pos_in_file = off; - } - else if (whence == SEEK_CUR) - { - if (off >= 0) - { - if (off > (std::numeric_limits::max() - getPosition())) - throw Exception("SEEK_CUR overflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - } - else if (off < -getPosition()) - throw Exception("SEEK_CUR underflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - new_pos_in_file = getPosition() + off; - } - else - throw Exception("ReadBufferAIO::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - - if (new_pos_in_file != getPosition()) - { - off_t first_read_pos_in_file = first_unread_pos_in_file - static_cast(working_buffer.size()); - if (hasPendingData() && (new_pos_in_file >= first_read_pos_in_file) && (new_pos_in_file <= first_unread_pos_in_file)) - { - /// Moved, but remained within the buffer. - pos = working_buffer.begin() + (new_pos_in_file - first_read_pos_in_file); - } - else - { - /// Moved past the buffer. - pos = working_buffer.end(); - first_unread_pos_in_file = new_pos_in_file; - - /// If we go back, than it's not eof - is_eof = false; - - /// We can not use the result of the current asynchronous request. - skip(); - } - } - - return new_pos_in_file; -} - -void ReadBufferAIO::synchronousRead() -{ - CurrentMetrics::Increment metric_increment_read{CurrentMetrics::Read}; - - prepare(); - bytes_read = ::pread(fd, buffer_begin, region_aligned_size, region_aligned_begin); - - ProfileEvents::increment(ProfileEvents::ReadBufferAIORead); - ProfileEvents::increment(ProfileEvents::ReadBufferAIOReadBytes, bytes_read); - - finalize(); -} - -void ReadBufferAIO::receive() -{ - if (!waitForAIOCompletion()) - { - throw Exception("Trying to receive data from AIO, but nothing was queued. It's a bug", ErrorCodes::LOGICAL_ERROR); - } - finalize(); -} - -void ReadBufferAIO::skip() -{ - if (!waitForAIOCompletion()) - return; - - /// @todo I presume this assignment is redundant since waitForAIOCompletion() performs a similar one -// bytes_read = future_bytes_read.get(); - if ((bytes_read < 0) || (static_cast(bytes_read) < region_left_padding)) - throw Exception("Asynchronous read error on file " + filename, ErrorCodes::AIO_READ_ERROR); -} - -bool ReadBufferAIO::waitForAIOCompletion() -{ - if (is_eof || !is_pending_read) - return false; - - CurrentMetrics::Increment metric_increment_read{CurrentMetrics::Read}; - - bytes_read = future_bytes_read.get(); - is_pending_read = false; - - ProfileEvents::increment(ProfileEvents::ReadBufferAIORead); - ProfileEvents::increment(ProfileEvents::ReadBufferAIOReadBytes, bytes_read); - - return true; -} - -void ReadBufferAIO::prepare() -{ - requested_byte_count = std::min(fill_buffer.internalBuffer().size() - DEFAULT_AIO_FILE_BLOCK_SIZE, max_bytes_read); - - /// Region of the disk from which we want to read data. - const off_t region_begin = first_unread_pos_in_file; - - if ((requested_byte_count > static_cast(std::numeric_limits::max())) || - (first_unread_pos_in_file > (std::numeric_limits::max() - static_cast(requested_byte_count)))) - throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR); - - const off_t region_end = first_unread_pos_in_file + requested_byte_count; - - /// The aligned region of the disk from which we will read the data. - region_left_padding = region_begin % DEFAULT_AIO_FILE_BLOCK_SIZE; - const size_t region_right_padding = (DEFAULT_AIO_FILE_BLOCK_SIZE - (region_end % DEFAULT_AIO_FILE_BLOCK_SIZE)) % DEFAULT_AIO_FILE_BLOCK_SIZE; - - region_aligned_begin = region_begin - region_left_padding; - - if (region_end > (std::numeric_limits::max() - static_cast(region_right_padding))) - throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR); - - const off_t region_aligned_end = region_end + region_right_padding; - region_aligned_size = region_aligned_end - region_aligned_begin; - - buffer_begin = fill_buffer.internalBuffer().begin(); - - /// Unpoison because msan doesn't instrument linux AIO - __msan_unpoison(buffer_begin, fill_buffer.internalBuffer().size()); -} - -void ReadBufferAIO::finalize() -{ - if ((bytes_read < 0) || (static_cast(bytes_read) < region_left_padding)) - throw Exception("Asynchronous read error on file " + filename, ErrorCodes::AIO_READ_ERROR); - - /// Ignore redundant bytes on the left. - bytes_read -= region_left_padding; - - /// Ignore redundant bytes on the right. - bytes_read = std::min(static_cast(bytes_read), static_cast(requested_byte_count)); - - if (bytes_read > 0) - fill_buffer.buffer().resize(region_left_padding + bytes_read); - if (static_cast(bytes_read) < requested_byte_count) - is_eof = true; - - if (first_unread_pos_in_file > (std::numeric_limits::max() - bytes_read)) - throw Exception("An overflow occurred during file operation", ErrorCodes::LOGICAL_ERROR); - - first_unread_pos_in_file += bytes_read; - total_bytes_read += bytes_read; - nextimpl_working_buffer_offset = region_left_padding; - - if (total_bytes_read == max_bytes_read) - is_eof = true; - - /// Swap the main and duplicate buffers. - swap(fill_buffer); -} - -} - -#endif diff --git a/src/IO/ReadBufferAIO.h b/src/IO/ReadBufferAIO.h deleted file mode 100644 index d476865747d..00000000000 --- a/src/IO/ReadBufferAIO.h +++ /dev/null @@ -1,111 +0,0 @@ -#pragma once - -#if defined(OS_LINUX) || defined(__FreeBSD__) - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace CurrentMetrics -{ - extern const Metric OpenFileForRead; -} - -namespace DB -{ - -/** Class for asynchronous data reading. - */ -class ReadBufferAIO final : public ReadBufferFromFileBase -{ -public: - ReadBufferAIO(const std::string & filename_, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, int flags_ = -1, - char * existing_memory_ = nullptr); - ~ReadBufferAIO() override; - - ReadBufferAIO(const ReadBufferAIO &) = delete; - ReadBufferAIO & operator=(const ReadBufferAIO &) = delete; - - void setMaxBytes(size_t max_bytes_read_); - off_t getPosition() override { return first_unread_pos_in_file - (working_buffer.end() - pos); } - std::string getFileName() const override { return filename; } - int getFD() const { return fd; } - - off_t seek(off_t off, int whence) override; - -private: - /// - bool nextImpl() override; - /// Synchronously read the data. - void synchronousRead(); - /// Get data from an asynchronous request. - void receive(); - /// Ignore data from an asynchronous request. - void skip(); - /// Wait for the end of the current asynchronous task. - bool waitForAIOCompletion(); - /// Prepare the request. - void prepare(); - /// Prepare for reading a duplicate buffer containing data from - /// of the last request. - void finalize(); - -private: - /// Buffer for asynchronous data read operations. - BufferWithOwnMemory fill_buffer; - - /// Description of the asynchronous read request. - iocb request{}; - std::future future_bytes_read; - - const std::string filename; - - /// The maximum number of bytes that can be read. - size_t max_bytes_read = std::numeric_limits::max(); - /// Number of bytes requested. - size_t requested_byte_count = 0; - /// The number of bytes read at the last request. - ssize_t bytes_read = 0; - /// The total number of bytes read. - size_t total_bytes_read = 0; - - /// The position of the first unread byte in the file. - off_t first_unread_pos_in_file = 0; - - /// The starting position of the aligned region of the disk from which the data is read. - off_t region_aligned_begin = 0; - /// Left offset to align the region of the disk. - size_t region_left_padding = 0; - /// The size of the aligned region of the disk. - size_t region_aligned_size = 0; - - /// The file descriptor for read. - int fd = -1; - - /// The buffer to which the received data is written. - Position buffer_begin = nullptr; - - /// The asynchronous read operation is not yet completed. - bool is_pending_read = false; - /// The end of the file is reached. - bool is_eof = false; - /// At least one read request was sent. - bool is_started = false; - /// Did the asynchronous operation fail? - bool aio_failed = false; - - CurrentMetrics::Increment metric_increment{CurrentMetrics::OpenFileForRead}; -}; - -} - -#endif diff --git a/src/IO/ReadBufferFromEncryptedFile.cpp b/src/IO/ReadBufferFromEncryptedFile.cpp new file mode 100644 index 00000000000..7a4d0e4ca14 --- /dev/null +++ b/src/IO/ReadBufferFromEncryptedFile.cpp @@ -0,0 +1,101 @@ +#include + +#if USE_SSL + +namespace DB +{ +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + +ReadBufferFromEncryptedFile::ReadBufferFromEncryptedFile( + size_t buf_size_, + std::unique_ptr in_, + const String & init_vector_, + const FileEncryption::EncryptionKey & key_, + const size_t iv_offset_) + : ReadBufferFromFileBase(buf_size_, nullptr, 0) + , in(std::move(in_)) + , buf_size(buf_size_) + , decryptor(FileEncryption::Decryptor(init_vector_, key_)) + , iv_offset(iv_offset_) +{ +} + +off_t ReadBufferFromEncryptedFile::seek(off_t off, int whence) +{ + if (whence == SEEK_CUR) + { + if (off < 0 && -off > getPosition()) + throw Exception("SEEK_CUR shift out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + if (!working_buffer.empty() && static_cast(offset() + off) < working_buffer.size()) + { + pos += off; + return getPosition(); + } + else + start_pos = off + getPosition(); + } + else if (whence == SEEK_SET) + { + if (off < 0) + throw Exception("SEEK_SET underflow: off = " + std::to_string(off), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + if (!working_buffer.empty() && static_cast(off) >= start_pos + && static_cast(off) < (start_pos + working_buffer.size())) + { + pos = working_buffer.begin() + (off - start_pos); + return getPosition(); + } + else + start_pos = off; + } + else + throw Exception("ReadBufferFromEncryptedFile::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + + initialize(); + return start_pos; +} + +bool ReadBufferFromEncryptedFile::nextImpl() +{ + if (in->eof()) + return false; + + if (initialized) + start_pos += working_buffer.size(); + initialize(); + return true; +} + +void ReadBufferFromEncryptedFile::initialize() +{ + size_t in_pos = start_pos + iv_offset; + + String data; + data.resize(buf_size); + size_t data_size = 0; + + in->seek(in_pos, SEEK_SET); + while (data_size < buf_size && !in->eof()) + { + auto size = in->read(data.data() + data_size, buf_size - data_size); + data_size += size; + in_pos += size; + in->seek(in_pos, SEEK_SET); + } + + data.resize(data_size); + working_buffer.resize(data_size); + + decryptor.decrypt(data.data(), working_buffer.begin(), data_size, start_pos); + + pos = working_buffer.begin(); + initialized = true; +} + +} + +#endif diff --git a/src/IO/ReadBufferFromEncryptedFile.h b/src/IO/ReadBufferFromEncryptedFile.h new file mode 100644 index 00000000000..b9c84537f17 --- /dev/null +++ b/src/IO/ReadBufferFromEncryptedFile.h @@ -0,0 +1,50 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include +#endif + +#if USE_SSL +#include +#include + + +namespace DB +{ + +class ReadBufferFromEncryptedFile : public ReadBufferFromFileBase +{ +public: + ReadBufferFromEncryptedFile( + size_t buf_size_, + std::unique_ptr in_, + const String & init_vector_, + const FileEncryption::EncryptionKey & key_, + const size_t iv_offset_); + + off_t seek(off_t off, int whence) override; + + off_t getPosition() override { return start_pos + offset(); } + + std::string getFileName() const override { return in->getFileName(); } + +private: + bool nextImpl() override; + + void initialize(); + + std::unique_ptr in; + size_t buf_size; + + FileEncryption::Decryptor decryptor; + bool initialized = false; + + // current working_buffer.begin() offset from decrypted file + size_t start_pos = 0; + size_t iv_offset = 0; +}; + +} + + +#endif diff --git a/src/IO/ReadBufferFromFile.h b/src/IO/ReadBufferFromFile.h index 33365bc7ceb..676f53afeb8 100644 --- a/src/IO/ReadBufferFromFile.h +++ b/src/IO/ReadBufferFromFile.h @@ -46,4 +46,18 @@ public: } }; + +/** Similar to ReadBufferFromFile but it is using 'pread' instead of 'read'. + */ +class ReadBufferFromFilePRead : public ReadBufferFromFile +{ +public: + ReadBufferFromFilePRead(const std::string & file_name_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, int flags = -1, + char * existing_memory = nullptr, size_t alignment = 0) + : ReadBufferFromFile(file_name_, buf_size, flags, existing_memory, alignment) + { + use_pread = true; + } +}; + } diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index 893c2bcb5d8..fdb538d4a49 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -59,7 +59,11 @@ bool ReadBufferFromFileDescriptor::nextImpl() ssize_t res = 0; { CurrentMetrics::Increment metric_increment{CurrentMetrics::Read}; - res = ::read(fd, internal_buffer.begin(), internal_buffer.size()); + + if (use_pread) + res = ::pread(fd, internal_buffer.begin(), internal_buffer.size(), file_offset_of_buffer_end); + else + res = ::read(fd, internal_buffer.begin(), internal_buffer.size()); } if (!res) break; @@ -128,12 +132,13 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence) if (new_pos + (working_buffer.end() - pos) == file_offset_of_buffer_end) return new_pos; - // file_offset_of_buffer_end corresponds to working_buffer.end(); it's a past-the-end pos, - // so the second inequality is strict. + /// file_offset_of_buffer_end corresponds to working_buffer.end(); it's a past-the-end pos, + /// so the second inequality is strict. if (file_offset_of_buffer_end - working_buffer.size() <= static_cast(new_pos) && new_pos < file_offset_of_buffer_end) { - /// Position is still inside buffer. + /// Position is still inside the buffer. + pos = working_buffer.end() - file_offset_of_buffer_end + new_pos; assert(pos >= working_buffer.begin()); assert(pos < working_buffer.end()); @@ -142,35 +147,66 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence) } else { - ProfileEvents::increment(ProfileEvents::Seek); - Stopwatch watch(profile_callback ? clock_type : CLOCK_MONOTONIC); + /// Position is out of the buffer, we need to do real seek. + off_t seek_pos = required_alignment > 1 + ? new_pos / required_alignment * required_alignment + : new_pos; + off_t offset_after_seek_pos = new_pos - seek_pos; + + /// First put position at the end of the buffer so the next read will fetch new data to the buffer. pos = working_buffer.end(); - off_t res = ::lseek(fd, new_pos, SEEK_SET); - if (-1 == res) - throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(), - ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - file_offset_of_buffer_end = new_pos; - watch.stop(); - ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + /// In case of using 'pread' we just update the info about the next position in file. + /// In case of using 'read' we call 'lseek'. - return res; + /// We account both cases as seek event as it leads to non-contiguous reads from file. + ProfileEvents::increment(ProfileEvents::Seek); + + if (!use_pread) + { + Stopwatch watch(profile_callback ? clock_type : CLOCK_MONOTONIC); + + off_t res = ::lseek(fd, seek_pos, SEEK_SET); + if (-1 == res) + throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(), + ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + + /// Also note that seeking past the file size is not allowed. + if (res != seek_pos) + throw Exception(ErrorCodes::CANNOT_SEEK_THROUGH_FILE, + "The 'lseek' syscall returned value ({}) that is not expected ({})", res, seek_pos); + + watch.stop(); + ProfileEvents::increment(ProfileEvents::DiskReadElapsedMicroseconds, watch.elapsedMicroseconds()); + } + + file_offset_of_buffer_end = seek_pos; + + if (offset_after_seek_pos > 0) + ignore(offset_after_seek_pos); + + return seek_pos; } } void ReadBufferFromFileDescriptor::rewind() { - ProfileEvents::increment(ProfileEvents::Seek); - off_t res = ::lseek(fd, 0, SEEK_SET); - if (-1 == res) - throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(), - ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + if (!use_pread) + { + ProfileEvents::increment(ProfileEvents::Seek); + off_t res = ::lseek(fd, 0, SEEK_SET); + if (-1 == res) + throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(), + ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + } + /// In case of pread, the ProfileEvents::Seek is not accounted, but it's Ok. /// Clearing the buffer with existing data. New data will be read on subsequent call to 'next'. working_buffer.resize(0); pos = working_buffer.begin(); + file_offset_of_buffer_end = 0; } diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h index 1883c6802bc..84970820abf 100644 --- a/src/IO/ReadBufferFromFileDescriptor.h +++ b/src/IO/ReadBufferFromFileDescriptor.h @@ -14,8 +14,11 @@ namespace DB class ReadBufferFromFileDescriptor : public ReadBufferFromFileBase { protected: + const size_t required_alignment = 0; /// For O_DIRECT both file offsets and memory addresses have to be aligned. + bool use_pread = false; /// To access one fd from multiple threads, use 'pread' syscall instead of 'read'. + + size_t file_offset_of_buffer_end = 0; /// What offset in file corresponds to working_buffer.end(). int fd; - size_t file_offset_of_buffer_end; /// What offset in file corresponds to working_buffer.end(). bool nextImpl() override; @@ -24,7 +27,9 @@ protected: public: ReadBufferFromFileDescriptor(int fd_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0) - : ReadBufferFromFileBase(buf_size, existing_memory, alignment), fd(fd_), file_offset_of_buffer_end(0) {} + : ReadBufferFromFileBase(buf_size, existing_memory, alignment), required_alignment(alignment), fd(fd_) + { + } int getFD() const { @@ -45,9 +50,23 @@ public: off_t size(); void setProgressCallback(ContextPtr context); + private: /// Assuming file descriptor supports 'select', check that we have data to read or wait until timeout. bool poll(size_t timeout_microseconds); }; + +/** Similar to ReadBufferFromFileDescriptor but it is using 'pread' allowing multiple concurrent reads from the same fd. + */ +class ReadBufferFromFileDescriptorPRead : public ReadBufferFromFileDescriptor +{ +public: + ReadBufferFromFileDescriptorPRead(int fd_, size_t buf_size = DBMS_DEFAULT_BUFFER_SIZE, char * existing_memory = nullptr, size_t alignment = 0) + : ReadBufferFromFileDescriptor(fd_, buf_size, existing_memory, alignment) + { + use_pread = true; + } +}; + } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index 9abdab11259..aa241322edf 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -45,20 +45,27 @@ bool ReadBufferFromS3::nextImpl() { Stopwatch watch; bool next_result = false; - auto sleep_time_with_backoff_milliseconds = std::chrono::milliseconds(100); - if (!impl) + if (impl) + { + /// `impl` has been initialized earlier and now we're at the end of the current portion of data. + impl->position() = position(); + assert(!impl->hasPendingData()); + } + else + { + /// `impl` is not initialized and we're about to read the first portion of data. impl = initialize(); + next_result = impl->hasPendingData(); + } - for (size_t attempt = 0; attempt < max_single_read_retries; ++attempt) + auto sleep_time_with_backoff_milliseconds = std::chrono::milliseconds(100); + for (size_t attempt = 0; (attempt < max_single_read_retries) && !next_result; ++attempt) { try { + /// Try to read a next portion of data. next_result = impl->next(); - /// FIXME. 1. Poco `istream` cannot read less than buffer_size or this state is being discarded during - /// istream <-> iostream conversion. `gcount` always contains 0, - /// that's why we always have error "Cannot read from istream at offset 0". - break; } catch (const Exception & e) @@ -68,24 +75,26 @@ bool ReadBufferFromS3::nextImpl() LOG_INFO(log, "Caught exception while reading S3 object. Bucket: {}, Key: {}, Offset: {}, Attempt: {}, Message: {}", bucket, key, getPosition(), attempt, e.message()); + /// Pause before next attempt. + std::this_thread::sleep_for(sleep_time_with_backoff_milliseconds); + sleep_time_with_backoff_milliseconds *= 2; + + /// Try to reinitialize `impl`. impl.reset(); impl = initialize(); + next_result = impl->hasPendingData(); } - - std::this_thread::sleep_for(sleep_time_with_backoff_milliseconds); - sleep_time_with_backoff_milliseconds *= 2; } watch.stop(); ProfileEvents::increment(ProfileEvents::S3ReadMicroseconds, watch.elapsedMicroseconds()); + if (!next_result) return false; - working_buffer = internal_buffer = impl->buffer(); - pos = working_buffer.begin(); - - ProfileEvents::increment(ProfileEvents::S3ReadBytes, internal_buffer.size()); + BufferBase::set(impl->buffer().begin(), impl->buffer().size(), impl->offset()); /// use the buffer returned by `impl` + ProfileEvents::increment(ProfileEvents::S3ReadBytes, working_buffer.size()); offset += working_buffer.size(); return true; diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 2a5594a6866..e695809871c 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -327,6 +327,7 @@ static void parseComplexEscapeSequence(Vector & s, ReadBuffer & buf) && decoded_char != '"' && decoded_char != '`' /// MySQL style identifiers && decoded_char != '/' /// JavaScript in HTML + && decoded_char != '=' /// Yandex's TSKV && !isControlASCII(decoded_char)) { s.push_back('\\'); diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 4e101aaaf63..d7bbab999ea 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -895,6 +895,17 @@ readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian archi x = __builtin_bswap64(x); } +template +inline std::enable_if_t, void> +readBinaryBigEndian(T & x, ReadBuffer & buf) /// Assuming little endian architecture. +{ + for (size_t i = 0; i != std::size(x.items); ++i) + { + auto & item = x.items[std::size(x.items) - i - 1]; + readBinaryBigEndian(item, buf); + } +} + /// Generic methods to read value in text tab-separated format. template diff --git a/src/IO/SeekableReadBuffer.h b/src/IO/SeekableReadBuffer.h index f8e6d817fb1..97620f0c03c 100644 --- a/src/IO/SeekableReadBuffer.h +++ b/src/IO/SeekableReadBuffer.h @@ -17,7 +17,7 @@ public: * Shifts buffer current position to given offset. * @param off Offset. * @param whence Seek mode (@see SEEK_SET, @see SEEK_CUR). - * @return New position from the begging of underlying buffer / file. + * @return New position from the beginning of underlying buffer / file. */ virtual off_t seek(off_t off, int whence) = 0; diff --git a/src/IO/WriteBufferFromEncryptedFile.cpp b/src/IO/WriteBufferFromEncryptedFile.cpp new file mode 100644 index 00000000000..ebc6b8610a1 --- /dev/null +++ b/src/IO/WriteBufferFromEncryptedFile.cpp @@ -0,0 +1,79 @@ +#include + +#if USE_SSL +#include + +namespace DB +{ + +WriteBufferFromEncryptedFile::WriteBufferFromEncryptedFile( + size_t buf_size_, + std::unique_ptr out_, + const String & init_vector_, + const FileEncryption::EncryptionKey & key_, + const size_t & file_size) + : WriteBufferFromFileBase(buf_size_, nullptr, 0) + , out(std::move(out_)) + , flush_iv(!file_size) + , iv(init_vector_) + , encryptor(FileEncryption::Encryptor(init_vector_, key_, file_size)) +{ +} + +WriteBufferFromEncryptedFile::~WriteBufferFromEncryptedFile() +{ + /// FIXME move final flush into the caller + MemoryTracker::LockExceptionInThread lock(VariableContext::Global); + finish(); +} + +void WriteBufferFromEncryptedFile::finish() +{ + if (finished) + return; + + try + { + finishImpl(); + out->finalize(); + finished = true; + } + catch (...) + { + /// Do not try to flush next time after exception. + out->position() = out->buffer().begin(); + finished = true; + throw; + } +} + +void WriteBufferFromEncryptedFile::finishImpl() +{ + /// If buffer has pending data - write it. + next(); + out->finalize(); +} + +void WriteBufferFromEncryptedFile::sync() +{ + /// If buffer has pending data - write it. + next(); + out->sync(); +} + +void WriteBufferFromEncryptedFile::nextImpl() +{ + if (!offset()) + return; + + if (flush_iv) + { + FileEncryption::writeIV(iv, *out); + flush_iv = false; + } + + encryptor.encrypt(working_buffer.begin(), *out, offset()); +} +} + +#endif diff --git a/src/IO/WriteBufferFromEncryptedFile.h b/src/IO/WriteBufferFromEncryptedFile.h new file mode 100644 index 00000000000..132b9886ef5 --- /dev/null +++ b/src/IO/WriteBufferFromEncryptedFile.h @@ -0,0 +1,47 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include +#endif + +#if USE_SSL +#include +#include + + +namespace DB +{ + +class WriteBufferFromEncryptedFile : public WriteBufferFromFileBase +{ +public: + WriteBufferFromEncryptedFile( + size_t buf_size_, + std::unique_ptr out_, + const String & init_vector_, + const FileEncryption::EncryptionKey & key_, + const size_t & file_size); + ~WriteBufferFromEncryptedFile() override; + + void sync() override; + void finalize() override { finish(); } + + std::string getFileName() const override { return out->getFileName(); } + +private: + void nextImpl() override; + + void finish(); + void finishImpl(); + + bool finished = false; + std::unique_ptr out; + + bool flush_iv; + String iv; + FileEncryption::Encryptor encryptor; +}; + +} + +#endif diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 7de6dbfa613..76ad0f77fcf 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -1094,6 +1094,17 @@ writeBinaryBigEndian(T x, WriteBuffer & buf) /// Assuming little endian archi writePODBinary(x, buf); } +template +inline std::enable_if_t, void> +writeBinaryBigEndian(const T & x, WriteBuffer & buf) /// Assuming little endian architecture. +{ + for (size_t i = 0; i != std::size(x.items); ++i) + { + const auto & item = x.items[std::size(x.items) - i - 1]; + writeBinaryBigEndian(item, buf); + } +} + struct PcgSerializer { static void serializePcg32(const pcg32_fast & rng, WriteBuffer & buf) diff --git a/src/IO/createReadBufferFromFileBase.cpp b/src/IO/createReadBufferFromFileBase.cpp index 230f049b2cb..11a0937ee48 100644 --- a/src/IO/createReadBufferFromFileBase.cpp +++ b/src/IO/createReadBufferFromFileBase.cpp @@ -1,8 +1,5 @@ #include #include -#if defined(OS_LINUX) || defined(__FreeBSD__) -#include -#endif #include #include @@ -10,8 +7,8 @@ namespace ProfileEvents { extern const Event CreatedReadBufferOrdinary; - extern const Event CreatedReadBufferAIO; - extern const Event CreatedReadBufferAIOFailed; + extern const Event CreatedReadBufferDirectIO; + extern const Event CreatedReadBufferDirectIOFailed; extern const Event CreatedReadBufferMMap; extern const Event CreatedReadBufferMMapFailed; } @@ -20,36 +17,15 @@ namespace DB { std::unique_ptr createReadBufferFromFileBase( - const std::string & filename_, - size_t estimated_size, size_t aio_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache, - size_t buffer_size_, int flags_, char * existing_memory_, size_t alignment) + const std::string & filename, + size_t estimated_size, size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache, + size_t buffer_size, int flags, char * existing_memory, size_t alignment) { -#if defined(OS_LINUX) || defined(__FreeBSD__) - if (aio_threshold && estimated_size >= aio_threshold) - { - /// Attempt to open a file with O_DIRECT - try - { - auto res = std::make_unique(filename_, buffer_size_, flags_, existing_memory_); - ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIO); - return res; - } - catch (const ErrnoException &) - { - /// Fallback to cached IO if O_DIRECT is not supported. - ProfileEvents::increment(ProfileEvents::CreatedReadBufferAIOFailed); - } - } -#else - (void)aio_threshold; - (void)estimated_size; -#endif - - if (!existing_memory_ && mmap_threshold && mmap_cache && estimated_size >= mmap_threshold) + if (!existing_memory && mmap_threshold && mmap_cache && estimated_size >= mmap_threshold) { try { - auto res = std::make_unique(*mmap_cache, filename_, 0); + auto res = std::make_unique(*mmap_cache, filename, 0); ProfileEvents::increment(ProfileEvents::CreatedReadBufferMMap); return res; } @@ -60,8 +36,63 @@ std::unique_ptr createReadBufferFromFileBase( } } +#if defined(OS_LINUX) || defined(__FreeBSD__) + if (direct_io_threshold && estimated_size >= direct_io_threshold) + { + /** O_DIRECT + * The O_DIRECT flag may impose alignment restrictions on the length and address of user-space buffers and the file offset of I/Os. + * In Linux alignment restrictions vary by filesystem and kernel version and might be absent entirely. + * However there is currently no filesystem-independent interface for an application to discover these restrictions + * for a given file or filesystem. Some filesystems provide their own interfaces for doing so, for example the + * XFS_IOC_DIOINFO operation in xfsctl(3). + * + * Under Linux 2.4, transfer sizes, and the alignment of the user buffer and the file offset must all be + * multiples of the logical block size of the filesystem. Since Linux 2.6.0, alignment to the logical block size + * of the underlying storage (typically 512 bytes) suffices. + * + * - man 2 open + */ + constexpr size_t min_alignment = DEFAULT_AIO_FILE_BLOCK_SIZE; + + auto align_up = [=](size_t value) { return (value + min_alignment - 1) / min_alignment * min_alignment; }; + + if (alignment == 0) + alignment = min_alignment; + else if (alignment % min_alignment) + alignment = align_up(alignment); + + if (buffer_size % min_alignment) + { + existing_memory = nullptr; /// Cannot reuse existing memory is it has unaligned size. + buffer_size = align_up(buffer_size); + } + + if (reinterpret_cast(existing_memory) % min_alignment) + { + existing_memory = nullptr; /// Cannot reuse existing memory is it has unaligned offset. + } + + /// Attempt to open a file with O_DIRECT + try + { + auto res = std::make_unique( + filename, buffer_size, (flags == -1 ? O_RDONLY | O_CLOEXEC : flags) | O_DIRECT, existing_memory, alignment); + ProfileEvents::increment(ProfileEvents::CreatedReadBufferDirectIO); + return res; + } + catch (const ErrnoException &) + { + /// Fallback to cached IO if O_DIRECT is not supported. + ProfileEvents::increment(ProfileEvents::CreatedReadBufferDirectIOFailed); + } + } +#else + (void)direct_io_threshold; + (void)estimated_size; +#endif + ProfileEvents::increment(ProfileEvents::CreatedReadBufferOrdinary); - return std::make_unique(filename_, buffer_size_, flags_, existing_memory_, alignment); + return std::make_unique(filename, buffer_size, flags, existing_memory, alignment); } } diff --git a/src/IO/createReadBufferFromFileBase.h b/src/IO/createReadBufferFromFileBase.h index 46d5b39ea44..dc2912ea752 100644 --- a/src/IO/createReadBufferFromFileBase.h +++ b/src/IO/createReadBufferFromFileBase.h @@ -13,20 +13,20 @@ class MMappedFileCache; /** Create an object to read data from a file. * estimated_size - the number of bytes to read - * aio_threshold - the minimum number of bytes for asynchronous reads + * direct_io_threshold - the minimum number of bytes for asynchronous reads * - * If aio_threshold = 0 or estimated_size < aio_threshold, read operations are executed synchronously. + * If direct_io_threshold = 0 or estimated_size < direct_io_threshold, read operations are executed synchronously. * Otherwise, the read operations are performed asynchronously. */ std::unique_ptr createReadBufferFromFileBase( - const std::string & filename_, + const std::string & filename, size_t estimated_size, - size_t aio_threshold, + size_t direct_io_threshold, size_t mmap_threshold, MMappedFileCache * mmap_cache, - size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, + size_t buffer_size = DBMS_DEFAULT_BUFFER_SIZE, int flags_ = -1, - char * existing_memory_ = nullptr, + char * existing_memory = nullptr, size_t alignment = 0); } diff --git a/src/IO/examples/CMakeLists.txt b/src/IO/examples/CMakeLists.txt index bcd0a8bba24..d5907bf67ad 100644 --- a/src/IO/examples/CMakeLists.txt +++ b/src/IO/examples/CMakeLists.txt @@ -49,11 +49,6 @@ target_link_libraries (io_operators PRIVATE clickhouse_common_io) add_executable (write_int write_int.cpp) target_link_libraries (write_int PRIVATE clickhouse_common_io) -if (OS_LINUX OR OS_FREEBSD) - add_executable(read_buffer_aio read_buffer_aio.cpp) - target_link_libraries (read_buffer_aio PRIVATE clickhouse_common_io) -endif () - add_executable (zlib_buffers zlib_buffers.cpp) target_link_libraries (zlib_buffers PRIVATE clickhouse_common_io) diff --git a/src/IO/examples/read_buffer_aio.cpp b/src/IO/examples/read_buffer_aio.cpp deleted file mode 100644 index 01ac9808cbb..00000000000 --- a/src/IO/examples/read_buffer_aio.cpp +++ /dev/null @@ -1,670 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include - - -namespace -{ - -void run(); -void prepare(std::string & filename, std::string & buf); -void prepare2(std::string & filename, std::string & buf); -void prepare3(std::string & filename, std::string & buf); -void prepare4(std::string & filename, std::string & buf); -std::string createTmpFile(); -[[noreturn]] void die(const std::string & msg); -void runTest(unsigned int num, const std::function & func); - -bool test1(const std::string & filename); -bool test2(const std::string & filename, const std::string & buf); -bool test3(const std::string & filename, const std::string & buf); -bool test4(const std::string & filename, const std::string & buf); -bool test5(const std::string & filename, const std::string & buf); -bool test6(const std::string & filename, const std::string & buf); -bool test7(const std::string & filename, const std::string & buf); -bool test8(const std::string & filename, const std::string & buf); -bool test9(const std::string & filename, const std::string & buf); -bool test10(const std::string & filename, const std::string & buf); -bool test11(const std::string & filename); -bool test12(const std::string & filename, const std::string & buf); -bool test13(const std::string & filename, const std::string & buf); -bool test14(const std::string & filename, const std::string & buf); -bool test15(const std::string & filename, const std::string & buf); -bool test16(const std::string & filename, const std::string & buf); -bool test17(const std::string & filename, const std::string & buf); -bool test18(const std::string & filename, const std::string & buf); -bool test19(const std::string & filename, const std::string & buf); -bool test20(const std::string & filename, const std::string & buf); - -void run() -{ - namespace fs = std::filesystem; - - std::string filename; - std::string buf; - prepare(filename, buf); - - std::string filename2; - std::string buf2; - prepare(filename2, buf2); - - std::string filename3; - std::string buf3; - prepare2(filename3, buf3); - - std::string filename4; - std::string buf4; - prepare3(filename4, buf4); - - std::string filename5; - std::string buf5; - prepare4(filename5, buf5); - - const std::vector> tests = - { - [&]{ return test1(filename); }, - [&]{ return test2(filename, buf); }, - [&]{ return test3(filename, buf); }, - [&]{ return test4(filename, buf); }, - [&]{ return test5(filename, buf); }, - [&]{ return test6(filename, buf); }, - [&]{ return test7(filename, buf); }, - [&]{ return test8(filename, buf); }, - [&]{ return test9(filename, buf); }, - [&]{ return test10(filename, buf); }, - [&]{ return test11(filename); }, - [&]{ return test12(filename, buf); }, - [&]{ return test13(filename2, buf2); }, - [&]{ return test14(filename, buf); }, - [&]{ return test15(filename3, buf3); }, - [&]{ return test16(filename3, buf3); }, - [&]{ return test17(filename4, buf4); }, - [&]{ return test18(filename5, buf5); }, - [&]{ return test19(filename, buf); }, - [&]{ return test20(filename, buf); } - }; - - unsigned int num = 0; - for (const auto & test : tests) - { - ++num; - runTest(num, test); - } - - fs::remove_all(fs::path(filename).parent_path().string()); - fs::remove_all(fs::path(filename2).parent_path().string()); - fs::remove_all(fs::path(filename3).parent_path().string()); - fs::remove_all(fs::path(filename4).parent_path().string()); - fs::remove_all(fs::path(filename5).parent_path().string()); -} - -void prepare(std::string & filename, std::string & buf) -{ - static const std::string symbols = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - filename = createTmpFile(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - buf.reserve(n); - - for (size_t i = 0; i < n; ++i) - buf += symbols[i % symbols.length()]; - - std::ofstream out(filename.c_str()); - if (!out.is_open()) - die("Could not open file"); - - out << buf; -} - -void prepare2(std::string & filename, std::string & buf) -{ - filename = createTmpFile(); - - buf = "122333444455555666666777777788888888999999999"; - - std::ofstream out(filename.c_str()); - if (!out.is_open()) - die("Could not open file"); - - out << buf; -} - -void prepare3(std::string & filename, std::string & buf) -{ - filename = createTmpFile(); - - buf = "122333444455555666666777777788888888999999999"; - - std::ofstream out(filename.c_str()); - if (!out.is_open()) - die("Could not open file"); - - out.seekp(7, std::ios_base::beg); - out << buf; -} - -void prepare4(std::string & filename, std::string & buf) -{ - static const std::string symbols = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - filename = createTmpFile(); - - std::ofstream out(filename.c_str()); - if (!out.is_open()) - die("Could not open file"); - - for (size_t i = 0; i < 1340; ++i) - buf += symbols[i % symbols.length()]; - - out.seekp(2984, std::ios_base::beg); - out << buf; -} - -std::string createTmpFile() -{ - char pattern[] = "/tmp/fileXXXXXX"; - char * dir = ::mkdtemp(pattern); - if (dir == nullptr) - die("Could not create directory"); - - return std::string(dir) + "/foo"; -} - -void die(const std::string & msg) -{ - std::cout << msg << "\n"; - ::exit(EXIT_FAILURE); -} - -void runTest(unsigned int num, const std::function & func) -{ - bool ok; - - try - { - ok = func(); - } - catch (const DB::Exception & ex) - { - ok = false; - std::cout << "Caught exception " << ex.displayText() << "\n"; - } - catch (const std::exception & ex) - { - ok = false; - std::cout << "Caught exception " << ex.what() << "\n"; - } - - if (ok) - std::cout << "Test " << num << " passed\n"; - else - std::cout << "Test " << num << " failed\n"; -} - -bool test1(const std::string & filename) -{ - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - if (in.getFileName() != filename) - return false; - if (in.getFD() == -1) - return false; - return true; -} - -bool test2(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(buf.length()); - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - size_t count = in.read(newbuf.data(), newbuf.length()); - if (count != newbuf.length()) - return false; - - return (newbuf == buf); -} - -bool test3(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(buf.length()); - - size_t requested = 9 * DEFAULT_AIO_FILE_BLOCK_SIZE; - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - in.setMaxBytes(requested); - size_t count = in.read(newbuf.data(), newbuf.length()); - - newbuf.resize(count); - return (newbuf == buf.substr(0, requested)); -} - -bool test4(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(buf.length()); - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - in.setMaxBytes(0); - size_t n_read = in.read(newbuf.data(), newbuf.length()); - - return n_read == 0; -} - -bool test5(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(1 + (DEFAULT_AIO_FILE_BLOCK_SIZE >> 1)); - - DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - in.setMaxBytes(1 + (DEFAULT_AIO_FILE_BLOCK_SIZE >> 1)); - - size_t count = in.read(newbuf.data(), newbuf.length()); - if (count != newbuf.length()) - return false; - - if (newbuf != buf.substr(0, newbuf.length())) - return false; - - return true; -} - -bool test6(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(buf.length()); - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - if (in.getPosition() != 0) - return false; - - size_t count = in.read(newbuf.data(), newbuf.length()); - if (count != newbuf.length()) - return false; - - if (static_cast(in.getPosition()) != buf.length()) - return false; - - return true; -} - -bool test7(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(buf.length() - DEFAULT_AIO_FILE_BLOCK_SIZE); - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - (void) in.seek(DEFAULT_AIO_FILE_BLOCK_SIZE, SEEK_SET); - size_t count = in.read(newbuf.data(), newbuf.length()); - if (count != (9 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - - return (newbuf == buf.substr(DEFAULT_AIO_FILE_BLOCK_SIZE)); -} - -bool test8(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(DEFAULT_AIO_FILE_BLOCK_SIZE - 1); - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - (void) in.seek(DEFAULT_AIO_FILE_BLOCK_SIZE + 1, SEEK_CUR); - size_t count = in.read(newbuf.data(), newbuf.length()); - - if (count != newbuf.length()) - return false; - - if (newbuf != buf.substr(DEFAULT_AIO_FILE_BLOCK_SIZE + 1, newbuf.length())) - return false; - - return true; -} - -bool test9(const std::string & filename, const std::string & buf) -{ - bool ok = false; - - try - { - std::string newbuf; - newbuf.resize(buf.length()); - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - size_t count = in.read(newbuf.data(), newbuf.length()); - if (count != newbuf.length()) - return false; - in.setMaxBytes(9 * DEFAULT_AIO_FILE_BLOCK_SIZE); - } - catch (const DB::Exception &) - { - ok = true; - } - - return ok; -} - -bool test10(const std::string & filename, const std::string & buf) -{ - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - { - std::string newbuf; - newbuf.resize(4 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - size_t count1 = in.read(newbuf.data(), newbuf.length()); - if (count1 != newbuf.length()) - return false; - - if (newbuf != buf.substr(0, 4 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - } - - (void) in.seek(2 * DEFAULT_AIO_FILE_BLOCK_SIZE, SEEK_CUR); - - { - std::string newbuf; - newbuf.resize(4 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - size_t count2 = in.read(newbuf.data(), newbuf.length()); - if (count2 != newbuf.length()) - return false; - - if (newbuf != buf.substr(6 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - } - - return true; -} - -bool test11(const std::string & filename) -{ - bool ok = false; - - try - { - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - (void) in.seek(-DEFAULT_AIO_FILE_BLOCK_SIZE, SEEK_SET); - } - catch (const DB::Exception &) - { - ok = true; - } - - return ok; -} - -bool test12(const std::string & filename, const std::string &) -{ - bool ok = false; - - try - { - std::string newbuf; - newbuf.resize(4 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - size_t count = in.read(newbuf.data(), newbuf.length()); - if (count != newbuf.length()) - return false; - - (void) in.seek(-(10 * DEFAULT_AIO_FILE_BLOCK_SIZE), SEEK_CUR); - } - catch (const DB::Exception &) - { - ok = true; - } - - return ok; -} - -bool test13(const std::string & filename, const std::string &) -{ - std::string newbuf; - newbuf.resize(2 * DEFAULT_AIO_FILE_BLOCK_SIZE - 3); - - DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - size_t count1 = in.read(newbuf.data(), newbuf.length()); - return count1 == newbuf.length(); -} - -bool test14(const std::string & filename, const std::string & buf) -{ - std::string newbuf; - newbuf.resize(1 + (DEFAULT_AIO_FILE_BLOCK_SIZE >> 1)); - - DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - (void) in.seek(2, SEEK_SET); - in.setMaxBytes(3 + (DEFAULT_AIO_FILE_BLOCK_SIZE >> 1)); - - size_t count = in.read(newbuf.data(), newbuf.length()); - if (count != newbuf.length()) - return false; - - if (newbuf != buf.substr(2, newbuf.length())) - return false; - - return true; -} - -bool test15(const std::string & filename, const std::string &) -{ - std::string newbuf; - newbuf.resize(1000); - - DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - - size_t count = in.read(newbuf.data(), 1); - if (count != 1) - return false; - if (newbuf[0] != '1') - return false; - return true; -} - -bool test16(const std::string & filename, const std::string &) -{ - DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - size_t count; - - { - std::string newbuf; - newbuf.resize(1); - count = in.read(newbuf.data(), 1); - if (count != 1) - return false; - if (newbuf[0] != '1') - return false; - } - - in.seek(2, SEEK_CUR); - - { - std::string newbuf; - newbuf.resize(3); - count = in.read(newbuf.data(), 3); - if (count != 3) - return false; - if (newbuf != "333") - return false; - } - - in.seek(4, SEEK_CUR); - - { - std::string newbuf; - newbuf.resize(5); - count = in.read(newbuf.data(), 5); - if (count != 5) - return false; - if (newbuf != "55555") - return false; - } - - in.seek(6, SEEK_CUR); - - { - std::string newbuf; - newbuf.resize(7); - count = in.read(newbuf.data(), 7); - if (count != 7) - return false; - if (newbuf != "7777777") - return false; - } - - in.seek(8, SEEK_CUR); - - { - std::string newbuf; - newbuf.resize(9); - count = in.read(newbuf.data(), 9); - if (count != 9) - return false; - if (newbuf != "999999999") - return false; - } - - return true; -} - -bool test17(const std::string & filename, const std::string & buf) -{ - DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - size_t count; - - { - std::string newbuf; - newbuf.resize(10); - count = in.read(newbuf.data(), 10); - - if (count != 10) - return false; - if (newbuf.substr(0, 7) != std::string(7, '\0')) - return false; - if (newbuf.substr(7) != "122") - return false; - } - - in.seek(7 + buf.length() - 2, SEEK_SET); - - { - std::string newbuf; - newbuf.resize(160); - count = in.read(newbuf.data(), 160); - - if (count != 2) - return false; - if (newbuf.substr(0, 2) != "99") - return false; - } - - in.seek(7 + buf.length() + DEFAULT_AIO_FILE_BLOCK_SIZE, SEEK_SET); - - { - std::string newbuf; - newbuf.resize(50); - count = in.read(newbuf.data(), 50); - if (count != 0) - return false; - } - - return true; -} - -bool test18(const std::string & filename, const std::string & buf) -{ - DB::ReadBufferAIO in(filename, DEFAULT_AIO_FILE_BLOCK_SIZE); - - std::string newbuf; - newbuf.resize(1340); - - in.seek(2984, SEEK_SET); - size_t count = in.read(newbuf.data(), 1340); - - if (count != 1340) - return false; - if (newbuf != buf) - return false; - - return true; -} - -bool test19(const std::string & filename, const std::string & buf) -{ - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - { - std::string newbuf; - newbuf.resize(5 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - size_t count1 = in.read(newbuf.data(), newbuf.length()); - if (count1 != newbuf.length()) - return false; - - if (newbuf != buf.substr(0, 5 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - } - - { - std::string newbuf; - newbuf.resize(5 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - size_t count2 = in.read(newbuf.data(), newbuf.length()); - if (count2 != newbuf.length()) - return false; - - if (newbuf != buf.substr(5 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - } - - return true; -} - -bool test20(const std::string & filename, const std::string & buf) -{ - DB::ReadBufferAIO in(filename, 3 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - { - std::string newbuf; - newbuf.resize(5 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - size_t count1 = in.read(newbuf.data(), newbuf.length()); - if (count1 != newbuf.length()) - return false; - - if (newbuf != buf.substr(0, 5 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - } - - (void) in.getPosition(); - - { - std::string newbuf; - newbuf.resize(5 * DEFAULT_AIO_FILE_BLOCK_SIZE); - - size_t count2 = in.read(newbuf.data(), newbuf.length()); - if (count2 != newbuf.length()) - return false; - - if (newbuf != buf.substr(5 * DEFAULT_AIO_FILE_BLOCK_SIZE)) - return false; - } - - return true; -} - -} - -int main() -{ - run(); - return 0; -} diff --git a/src/IO/tests/gtest_aio_seek_back_after_eof.cpp b/src/IO/tests/gtest_aio_seek_back_after_eof.cpp deleted file mode 100644 index 784f5479657..00000000000 --- a/src/IO/tests/gtest_aio_seek_back_after_eof.cpp +++ /dev/null @@ -1,91 +0,0 @@ -#if defined(__linux__) || defined(__FreeBSD__) - -#include - -#include -#include -#include -#include -#include -#include -#include - - -namespace -{ -std::string createTmpFileForEOFtest() -{ - char pattern[] = "./EOFtestFolderXXXXXX"; - if (char * dir = ::mkdtemp(pattern); dir) - { - return std::string(dir) + "/foo"; - } - else - { - /// We have no tmp in docker - /// So we have to use root - std::string almost_rand_dir = std::string{"/"} + std::to_string(randomSeed()) + "foo"; - return almost_rand_dir; - } - -} - -void prepareForEOF(std::string & filename, std::string & buf) -{ - static const std::string symbols = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - - filename = createTmpFileForEOFtest(); - - size_t n = 10 * DEFAULT_AIO_FILE_BLOCK_SIZE; - buf.reserve(n); - - for (size_t i = 0; i < n; ++i) - buf += symbols[i % symbols.length()]; - - std::ofstream out(filename); - out << buf; -} - - -} -TEST(ReadBufferAIOTest, TestReadAfterAIO) -{ - using namespace DB; - std::string data; - std::string file_path; - prepareForEOF(file_path, data); - ReadBufferAIO testbuf(file_path); - - std::string newdata; - newdata.resize(data.length()); - - size_t total_read = testbuf.read(newdata.data(), newdata.length()); - EXPECT_EQ(total_read, data.length()); - EXPECT_TRUE(testbuf.eof()); - - - testbuf.seek(data.length() - 100, SEEK_SET); - - std::string smalldata; - smalldata.resize(100); - size_t read_after_eof = testbuf.read(smalldata.data(), smalldata.size()); - EXPECT_EQ(read_after_eof, 100); - EXPECT_TRUE(testbuf.eof()); - - - testbuf.seek(0, SEEK_SET); - std::string repeatdata; - repeatdata.resize(data.length()); - size_t read_after_eof_big = testbuf.read(repeatdata.data(), repeatdata.size()); - EXPECT_EQ(read_after_eof_big, data.length()); - EXPECT_TRUE(testbuf.eof()); - - if (file_path[0] != '/') - { - const size_t last_slash = file_path.rfind('/'); - const std::string temp_dir = file_path.substr(0, last_slash); - std::filesystem::remove_all(temp_dir); - } -} - -#endif diff --git a/src/IO/tests/gtest_file_encryption.cpp b/src/IO/tests/gtest_file_encryption.cpp new file mode 100644 index 00000000000..1f6d793dc76 --- /dev/null +++ b/src/IO/tests/gtest_file_encryption.cpp @@ -0,0 +1,150 @@ +#if !defined(ARCADIA_BUILD) +#include +#endif + +#if USE_SSL +#include +#include +#include + + +using namespace DB; +using namespace DB::FileEncryption; + +struct InitVectorTestParam +{ + const std::string_view comment; + const String init; + UInt128 adder; + UInt128 setter; + const String after_inc; + const String after_add; + const String after_set; +}; + + +class InitVectorTest : public ::testing::TestWithParam {}; + + +String string_ends_with(size_t size, String str) +{ + String res(size, 0); + res.replace(size - str.size(), str.size(), str); + return res; +} + + +static std::ostream & operator << (std::ostream & ostr, const InitVectorTestParam & param) +{ + return ostr << param.comment; +} + + +TEST_P(InitVectorTest, InitVector) +{ + const auto & param = GetParam(); + + auto iv = InitVector(param.init); + ASSERT_EQ(param.init, iv.str()); + + iv.inc(); + ASSERT_EQ(param.after_inc, iv.str()); + + iv.inc(param.adder); + ASSERT_EQ(param.after_add, iv.str()); + + iv.set(param.setter); + ASSERT_EQ(param.after_set, iv.str()); + + iv.set(0); + ASSERT_EQ(param.init, iv.str()); +} + + +INSTANTIATE_TEST_SUITE_P(InitVectorInputs, + InitVectorTest, + ::testing::ValuesIn(std::initializer_list{ + { + "Basic init vector test. Get zero-string, add 0, set 0", + String(16, 0), + 0, + 0, + string_ends_with(16, "\x1"), + string_ends_with(16, "\x1"), + String(16, 0), + }, + { + "Init vector test. Get zero-string, add 85, set 1024", + String(16, 0), + 85, + 1024, + string_ends_with(16, "\x1"), + string_ends_with(16, "\x56"), + string_ends_with(16, String("\x4\0", 2)), + }, + { + "Long init vector test", + "\xa8\x65\x9c\x73\xf8\x5d\x83\xb4\x5c\xa6\x8c\x19\xf4\x77\x80\xe1", + 3349249125638641, + 1698923461902341, + "\xa8\x65\x9c\x73\xf8\x5d\x83\xb4\x5c\xa6\x8c\x19\xf4\x77\x80\xe2", + "\xa8\x65\x9c\x73\xf8\x5d\x83\xb4\x5c\xb2\x72\x39\xc8\xdd\x62\xd3", + String("\xa8\x65\x9c\x73\xf8\x5d\x83\xb4\x5c\xac\x95\x43\x65\xea\x00\xe6", 16) + }, + }) +); + + +TEST(FileEncryption, Encryption) +{ + String iv(16, 0); + EncryptionKey key("1234567812345678"); + String input = "abcd1234efgh5678ijkl"; + String expected = "\xfb\x8a\x9e\x66\x82\x72\x1b\xbe\x6b\x1d\xd8\x98\xc5\x8c\x63\xee\xcd\x36\x4a\x50"; + + String result(expected.size(), 0); + for (size_t i = 0; i <= expected.size(); ++i) + { + auto buf = WriteBufferFromString(result); + auto encryptor = Encryptor(iv, key, 0); + encryptor.encrypt(input.data(), buf, i); + ASSERT_EQ(expected.substr(0, i), result.substr(0, i)); + } + + size_t offset = 25; + String offset_expected = "\x6c\x67\xe4\xf5\x8f\x86\xb0\x19\xe5\xcd\x53\x59\xe0\xc6\x01\x5e\xc1\xfd\x60\x9d"; + for (size_t i = 0; i <= expected.size(); ++i) + { + auto buf = WriteBufferFromString(result); + auto encryptor = Encryptor(iv, key, offset); + encryptor.encrypt(input.data(), buf, i); + ASSERT_EQ(offset_expected.substr(0, i), result.substr(0, i)); + } +} + + +TEST(FileEncryption, Decryption) +{ + String iv(16, 0); + EncryptionKey key("1234567812345678"); + String expected = "abcd1234efgh5678ijkl"; + String input = "\xfb\x8a\x9e\x66\x82\x72\x1b\xbe\x6b\x1d\xd8\x98\xc5\x8c\x63\xee\xcd\x36\x4a\x50"; + auto decryptor = Decryptor(iv, key); + String result(expected.size(), 0); + + for (size_t i = 0; i <= expected.size(); ++i) + { + decryptor.decrypt(input.data(), result.data(), i, 0); + ASSERT_EQ(expected.substr(0, i), result.substr(0, i)); + } + + size_t offset = 25; + String offset_input = "\x6c\x67\xe4\xf5\x8f\x86\xb0\x19\xe5\xcd\x53\x59\xe0\xc6\x01\x5e\xc1\xfd\x60\x9d"; + for (size_t i = 0; i <= expected.size(); ++i) + { + decryptor.decrypt(offset_input.data(), result.data(), i, offset); + ASSERT_EQ(expected.substr(0, i), result.substr(0, i)); + } +} + +#endif diff --git a/src/IO/ya.make b/src/IO/ya.make index d8bdfa95295..3bd704ec6f0 100644 --- a/src/IO/ya.make +++ b/src/IO/ya.make @@ -26,6 +26,7 @@ SRCS( CascadeWriteBuffer.cpp CompressionMethod.cpp DoubleConverter.cpp + FileEncryptionCommon.cpp HTTPChunkedReadBuffer.cpp HTTPCommon.cpp HashingWriteBuffer.cpp @@ -44,7 +45,7 @@ SRCS( NullWriteBuffer.cpp PeekableReadBuffer.cpp Progress.cpp - ReadBufferAIO.cpp + ReadBufferFromEncryptedFile.cpp ReadBufferFromFile.cpp ReadBufferFromFileBase.cpp ReadBufferFromFileDecorator.cpp @@ -56,6 +57,7 @@ SRCS( SeekAvoidingReadBuffer.cpp TimeoutSetter.cpp UseSSL.cpp + WriteBufferFromEncryptedFile.cpp WriteBufferFromFile.cpp WriteBufferFromFileBase.cpp WriteBufferFromFileDecorator.cpp diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 1518706f0a6..63b0345b372 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -26,6 +26,7 @@ namespace ErrorCodes extern const int THERE_IS_NO_COLUMN; extern const int ILLEGAL_COLUMN; extern const int NOT_FOUND_COLUMN_IN_BLOCK; + extern const int BAD_ARGUMENTS; } const char * ActionsDAG::typeToString(ActionsDAG::ActionType type) @@ -202,6 +203,7 @@ const ActionsDAG::Node & ActionsDAG::addFunction( node.function_base = function->build(arguments); node.result_type = node.function_base->getResultType(); node.function = node.function_base->prepare(arguments); + node.is_deterministic = node.function_base->isDeterministic(); /// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function. if (node.function_base->isSuitableForConstantFolding()) @@ -426,6 +428,16 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs) { /// Constant folding. node->type = ActionsDAG::ActionType::COLUMN; + + for (const auto & child : node->children) + { + if (!child->is_deterministic) + { + node->is_deterministic = false; + break; + } + } + node->children.clear(); } @@ -981,6 +993,14 @@ bool ActionsDAG::trivial() const return true; } +void ActionsDAG::assertDeterministic() const +{ + for (const auto & node : nodes) + if (!node.is_deterministic) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Expression must be deterministic but it contains non-deterministic part `{}`", node.result_name); +} + void ActionsDAG::addMaterializingOutputActions() { for (auto & node : index) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 9cd0057bb1a..bfb5b177ac7 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -83,6 +83,9 @@ public: ExecutableFunctionPtr function; /// If function is a compiled statement. bool is_function_compiled = false; + /// It is deterministic (See IFunction::isDeterministic). + /// This property is kept after constant folding of non-deterministic functions like 'now', 'today'. + bool is_deterministic = true; /// For COLUMN node and propagated constants. ColumnPtr column; @@ -175,6 +178,7 @@ public: bool hasArrayJoin() const; bool hasStatefulFunctions() const; bool trivial() const; /// If actions has no functions or array join. + void assertDeterministic() const; /// Throw if not isDeterministic. #if USE_EMBEDDED_COMPILER void compileExpressions(size_t min_count_to_compile_expression); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 7aad11252cb..03fa756276e 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -686,7 +686,7 @@ ASTs ActionsMatcher::doUntuple(const ASTFunction * function, ActionsMatcher::Dat ASTs columns; size_t tid = 0; - for (const auto & name : tuple_type->getElementNames()) + for (const auto & name [[maybe_unused]] : tuple_type->getElementNames()) { auto tuple_ast = function->arguments->children[0]; if (tid != 0) @@ -697,11 +697,6 @@ ASTs ActionsMatcher::doUntuple(const ASTFunction * function, ActionsMatcher::Dat auto func = makeASTFunction("tupleElement", tuple_ast, literal); - if (tuple_type->haveExplicitNames()) - func->setAlias(name); - else - func->setAlias(data.getUniqueName("_ut_" + name)); - auto function_builder = FunctionFactory::instance().get(func->name, data.getContext()); data.addFunction(function_builder, {tuple_name_type->name, literal->getColumnName(data.getContext()->getSettingsRef())}, func->getColumnName(data.getContext()->getSettingsRef())); diff --git a/src/Interpreters/AsynchronousMetrics.cpp b/src/Interpreters/AsynchronousMetrics.cpp index aca92b8866d..da514759eb5 100644 --- a/src/Interpreters/AsynchronousMetrics.cpp +++ b/src/Interpreters/AsynchronousMetrics.cpp @@ -48,7 +48,7 @@ namespace ErrorCodes static constexpr size_t small_buffer_size = 4096; -static void openFileIfExists(const char * filename, std::optional & out) +static void openFileIfExists(const char * filename, std::optional & out) { /// Ignoring time of check is not time of use cases, as procfs/sysfs files are fairly persistent. @@ -57,11 +57,11 @@ static void openFileIfExists(const char * filename, std::optional openFileIfExists(const std::string & filename) +static std::unique_ptr openFileIfExists(const std::string & filename) { std::error_code ec; if (std::filesystem::is_regular_file(filename, ec)) - return std::make_unique(filename, small_buffer_size); + return std::make_unique(filename, small_buffer_size); return {}; } @@ -89,7 +89,7 @@ AsynchronousMetrics::AsynchronousMetrics( for (size_t thermal_device_index = 0;; ++thermal_device_index) { - std::unique_ptr file = openFileIfExists(fmt::format("/sys/class/thermal/thermal_zone{}/temp", thermal_device_index)); + std::unique_ptr file = openFileIfExists(fmt::format("/sys/class/thermal/thermal_zone{}/temp", thermal_device_index)); if (!file) { /// Sometimes indices are from zero sometimes from one. @@ -113,7 +113,7 @@ AsynchronousMetrics::AsynchronousMetrics( } String hwmon_name; - ReadBufferFromFile hwmon_name_in(hwmon_name_file, small_buffer_size); + ReadBufferFromFilePRead hwmon_name_in(hwmon_name_file, small_buffer_size); readText(hwmon_name, hwmon_name_in); std::replace(hwmon_name.begin(), hwmon_name.end(), ' ', '_'); @@ -134,14 +134,14 @@ AsynchronousMetrics::AsynchronousMetrics( break; } - std::unique_ptr file = openFileIfExists(sensor_value_file); + std::unique_ptr file = openFileIfExists(sensor_value_file); if (!file) continue; String sensor_name; if (sensor_name_file_exists) { - ReadBufferFromFile sensor_name_in(sensor_name_file, small_buffer_size); + ReadBufferFromFilePRead sensor_name_in(sensor_name_file, small_buffer_size); readText(sensor_name, sensor_name_in); std::replace(sensor_name.begin(), sensor_name.end(), ' ', '_'); } @@ -184,7 +184,7 @@ AsynchronousMetrics::AsynchronousMetrics( if (device_name.starts_with("loop")) continue; - std::unique_ptr file = openFileIfExists(device_dir.path() / "stat"); + std::unique_ptr file = openFileIfExists(device_dir.path() / "stat"); if (!file) continue; @@ -1021,7 +1021,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti { try { - ReadBufferFromFile & in = *thermal[i]; + ReadBufferFromFilePRead & in = *thermal[i]; in.rewind(); Int64 temperature = 0; @@ -1065,7 +1065,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti { if (edac[i].first) { - ReadBufferFromFile & in = *edac[i].first; + ReadBufferFromFilePRead & in = *edac[i].first; in.rewind(); uint64_t errors = 0; readText(errors, in); @@ -1074,7 +1074,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti if (edac[i].second) { - ReadBufferFromFile & in = *edac[i].second; + ReadBufferFromFilePRead & in = *edac[i].second; in.rewind(); uint64_t errors = 0; readText(errors, in); @@ -1179,7 +1179,7 @@ void AsynchronousMetrics::update(std::chrono::system_clock::time_point update_ti total_number_of_parts += table_merge_tree->getPartsCount(); } - if (StorageReplicatedMergeTree * table_replicated_merge_tree = dynamic_cast(table.get())) + if (StorageReplicatedMergeTree * table_replicated_merge_tree = typeid_cast(table.get())) { StorageReplicatedMergeTree::Status status; table_replicated_merge_tree->getStatus(status, false); diff --git a/src/Interpreters/AsynchronousMetrics.h b/src/Interpreters/AsynchronousMetrics.h index 606d117e605..07e117c4dd9 100644 --- a/src/Interpreters/AsynchronousMetrics.h +++ b/src/Interpreters/AsynchronousMetrics.h @@ -82,25 +82,25 @@ private: #if defined(OS_LINUX) MemoryStatisticsOS memory_stat; - std::optional meminfo; - std::optional loadavg; - std::optional proc_stat; - std::optional cpuinfo; - std::optional file_nr; - std::optional uptime; - std::optional net_dev; + std::optional meminfo; + std::optional loadavg; + std::optional proc_stat; + std::optional cpuinfo; + std::optional file_nr; + std::optional uptime; + std::optional net_dev; - std::vector> thermal; + std::vector> thermal; std::unordered_map>> hwmon_devices; + std::unique_ptr>> hwmon_devices; std::vector /* correctable errors */, - std::unique_ptr /* uncorrectable errors */>> edac; + std::unique_ptr /* correctable errors */, + std::unique_ptr /* uncorrectable errors */>> edac; - std::unordered_map> block_devs; + std::unordered_map> block_devs; /// TODO: socket statistics. diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index a857bf81f95..59d8942538c 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -149,6 +149,7 @@ void executeQuery( OptimizeShardingKeyRewriteInVisitor::Data visitor_data{ sharding_key_expr, + sharding_key_expr->getSampleBlock().getByPosition(0).type, sharding_key_column_name, shard_info, not_optimized_cluster->getSlotToShard(), diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 9b204f12ab2..cbf2c0820f5 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -386,6 +386,7 @@ struct ContextSharedPart ActionLocksManagerPtr action_locks_manager; /// Set of storages' action lockers std::unique_ptr system_logs; /// Used to log queries and operations on parts std::optional storage_s3_settings; /// Settings of S3 storage + std::vector warnings; /// Store warning messages about server configuration. RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml @@ -514,6 +515,13 @@ struct ContextSharedPart trace_collector.emplace(std::move(trace_log)); } + + void addWarningMessage(const String & message) + { + /// A warning goes both: into server's log; stored to be placed in `system.warnings` table. + log->warning(message); + warnings.push_back(message); + } }; @@ -635,6 +643,12 @@ String Context::getDictionariesLibPath() const return shared->dictionaries_lib_path; } +std::vector Context::getWarnings() const +{ + auto lock = getLock(); + return shared->warnings; +} + VolumePtr Context::getTemporaryVolume() const { auto lock = getLock(); @@ -706,6 +720,12 @@ void Context::setDictionariesLibPath(const String & path) shared->dictionaries_lib_path = path; } +void Context::addWarningMessage(const String & msg) +{ + auto lock = getLock(); + shared->addWarningMessage(msg); +} + void Context::setConfig(const ConfigurationPtr & config) { auto lock = getLock(); @@ -2335,11 +2355,6 @@ OutputFormatPtr Context::getOutputFormatParallelIfPossible(const String & name, return FormatFactory::instance().getOutputFormatParallelIfPossible(name, buf, sample, shared_from_this()); } -OutputFormatPtr Context::getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const -{ - return FormatFactory::instance().getOutputFormat(name, buf, sample, shared_from_this()); -} - time_t Context::getUptimeSeconds() const { @@ -2712,4 +2727,18 @@ PartUUIDsPtr Context::getIgnoredPartUUIDs() const return ignored_part_uuids; } +void Context::setMySQLProtocolContext(MySQLWireContext * mysql_context) +{ + assert(session_context.lock().get() == this); + assert(!mysql_protocol_context); + assert(mysql_context); + mysql_protocol_context = mysql_context; +} + +MySQLWireContext * Context::getMySQLProtocolContext() const +{ + assert(!mysql_protocol_context || session_context.lock().get()); + return mysql_protocol_context; +} + } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 2b53c737915..05eab209eff 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -119,6 +119,8 @@ using ThrottlerPtr = std::shared_ptr; class ZooKeeperMetadataTransaction; using ZooKeeperMetadataTransactionPtr = std::shared_ptr; +struct MySQLWireContext; + /// Callback for external tables initializer using ExternalTablesInitializer = std::function; @@ -298,6 +300,8 @@ private: /// thousands of signatures. /// And I hope it will be replaced with more common Transaction sometime. + MySQLWireContext * mysql_protocol_context = nullptr; + Context(); Context(const Context &); Context & operator=(const Context &); @@ -319,6 +323,9 @@ public: String getUserFilesPath() const; String getDictionariesLibPath() const; + /// A list of warnings about server configuration to place in `system.warnings` table. + std::vector getWarnings() const; + VolumePtr getTemporaryVolume() const; void setPath(const String & path); @@ -326,6 +333,8 @@ public: void setUserFilesPath(const String & path); void setDictionariesLibPath(const String & path); + void addWarningMessage(const String & msg); + VolumePtr setTemporaryStorage(const String & path, const String & policy_name = ""); using ConfigurationPtr = Poco::AutoPtr; @@ -533,7 +542,6 @@ public: BlockOutputStreamPtr getOutputStream(const String & name, WriteBuffer & buf, const Block & sample) const; OutputFormatPtr getOutputFormatParallelIfPossible(const String & name, WriteBuffer & buf, const Block & sample) const; - OutputFormatPtr getOutputFormat(const String & name, WriteBuffer & buf, const Block & sample) const; InterserverIOHandler & getInterserverIOHandler(); @@ -789,14 +797,10 @@ public: /// Returns context of current distributed DDL query or nullptr. ZooKeeperMetadataTransactionPtr getZooKeeperMetadataTransaction() const; - struct MySQLWireContext - { - uint8_t sequence_id = 0; - uint32_t client_capabilities = 0; - size_t max_packet_size = 0; - }; - - MySQLWireContext mysql; + /// Caller is responsible for lifetime of mysql_context. + /// Used in MySQLHandler for session context. + void setMySQLProtocolContext(MySQLWireContext * mysql_context); + MySQLWireContext * getMySQLProtocolContext() const; PartUUIDsPtr getPartUUIDs() const; PartUUIDsPtr getIgnoredPartUUIDs() const; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index bd06c753319..905fcf0331c 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -531,11 +531,12 @@ Names ExpressionActions::getRequiredColumns() const bool ExpressionActions::hasArrayJoin() const { - for (const auto & action : actions) - if (action.node->type == ActionsDAG::ActionType::ARRAY_JOIN) - return true; + return getActionsDAG().hasArrayJoin(); +} - return false; +void ExpressionActions::assertDeterministic() const +{ + getActionsDAG().assertDeterministic(); } diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h index 7699e82a73b..4fddd1fd27e 100644 --- a/src/Interpreters/ExpressionActions.h +++ b/src/Interpreters/ExpressionActions.h @@ -103,6 +103,7 @@ public: void execute(Block & block, bool dry_run = false) const; bool hasArrayJoin() const; + void assertDeterministic() const; /// Obtain a sample block that contains the names and types of result columns. const Block & getSampleBlock() const { return sample_block; } diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 1496ea3dc61..c780f7bcea6 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1478,12 +1478,6 @@ ExpressionAnalysisResult::ExpressionAnalysisResult( chain.clear(); }; - if (storage) - { - query_analyzer.makeSetsForIndex(query.where()); - query_analyzer.makeSetsForIndex(query.prewhere()); - } - { ExpressionActionsChain chain(context); Names additional_required_columns_after_prewhere; diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index ac5d281f337..94a0b24d41a 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -326,15 +326,15 @@ public: /// Deletes all columns except mentioned by SELECT, arranges the remaining columns and renames them to aliases. ActionsDAGPtr appendProjectResult(ExpressionActionsChain & chain) const; + /// Create Set-s that we make from IN section to use index on them. + void makeSetsForIndex(const ASTPtr & node); + private: StorageMetadataPtr metadata_snapshot; /// If non-empty, ignore all expressions not from this list. NameSet required_result_columns; SelectQueryOptions query_options; - /// Create Set-s that we make from IN section to use index on them. - void makeSetsForIndex(const ASTPtr & node); - JoinPtr makeTableJoin( const ASTTablesInSelectQueryElement & join_element, const ColumnsWithTypeAndName & left_sample_columns); diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h index 5d92f4f8b6f..a9c7cb61a0a 100644 --- a/src/Interpreters/GlobalSubqueriesVisitor.h +++ b/src/Interpreters/GlobalSubqueriesVisitor.h @@ -198,8 +198,9 @@ private: { ASTPtr & ast = func.arguments->children[1]; - /// Literal can use regular IN - if (ast->as()) + /// Literal or function can use regular IN. + /// NOTE: We don't support passing table functions to IN. + if (ast->as() || ast->as()) { if (func.name == "globalIn") func.name = "in"; diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 6e5f7df99bd..56ad13511ac 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -1368,18 +1368,6 @@ void HashJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) throw Exception("Logical error: unknown combination of JOIN", ErrorCodes::LOGICAL_ERROR); } - -void HashJoin::joinTotals(Block & block) const -{ - Block sample_right_block = sample_block_with_columns_to_add.cloneEmpty(); - /// For StorageJoin column names isn't qualified in sample_block_with_columns_to_add - for (auto & col : sample_right_block) - col.name = getTableJoin().renamedRightColumnName(col.name); - - JoinCommon::joinTotals(totals, sample_right_block, *table_join, block); -} - - template struct AdderNonJoined { diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 84c447d875e..86c53081059 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -155,9 +155,7 @@ public: /** Keep "totals" (separate part of dataset, see WITH TOTALS) to use later. */ void setTotals(const Block & block) override { totals = block; } - bool hasTotals() const override { return totals; } - - void joinTotals(Block & block) const override; + const Block & getTotals() const override { return totals; } bool isFilled() const override { return from_storage_join || data->type == Type::DICT; } diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index 0f486fbe523..c2cf007d823 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -31,11 +31,9 @@ public: /// Could be called from different threads in parallel. virtual void joinBlock(Block & block, std::shared_ptr & not_processed) = 0; - virtual bool hasTotals() const = 0; - /// Set totals for right table + /// Set/Get totals for right table virtual void setTotals(const Block & block) = 0; - /// Add totals to block from left table - virtual void joinTotals(Block & block) const = 0; + virtual const Block & getTotals() const = 0; virtual size_t getTotalRowCount() const = 0; virtual size_t getTotalByteCount() const = 0; diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index d820cbbae45..ea44dbb86ca 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -283,6 +283,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( checkStackSize(); query_info.ignore_projections = options.ignore_projections; + query_info.is_projection_query = options.is_projection_query; initSettings(); const Settings & settings = context->getSettingsRef(); @@ -399,7 +400,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( view = nullptr; } - if (try_move_to_prewhere && storage && query.where() && !query.prewhere()) + if (try_move_to_prewhere && storage && storage->supportsPrewhere() && query.where() && !query.prewhere()) { /// PREWHERE optimization: transfer some condition from WHERE to PREWHERE if enabled and viable if (const auto & column_sizes = storage->getColumnSizes(); !column_sizes.empty()) @@ -575,9 +576,9 @@ void InterpreterSelectQuery::buildQueryPlan(QueryPlan & query_plan) /// We must guarantee that result structure is the same as in getSampleBlock() /// - /// But if we ignore aggregation, plan header does not match result_header. + /// But if it's a projection query, plan header does not match result_header. /// TODO: add special stage for InterpreterSelectQuery? - if (!options.ignore_aggregation && !blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, result_header)) + if (!options.is_projection_query && !blocksHaveEqualStructure(query_plan.getCurrentDataStream().header, result_header)) { auto convert_actions_dag = ActionsDAG::makeConvertingActions( query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName(), @@ -608,17 +609,17 @@ Block InterpreterSelectQuery::getSampleBlockImpl() query_info.query = query_ptr; query_info.has_window = query_analyzer->hasWindow(); + if (storage) + { + auto & query = getSelectQuery(); + query_analyzer->makeSetsForIndex(query.where()); + query_analyzer->makeSetsForIndex(query.prewhere()); + query_info.sets = query_analyzer->getPreparedSets(); + } if (storage && !options.only_analyze) - { from_stage = storage->getQueryProcessingStage(context, options.to_stage, metadata_snapshot, query_info); - /// TODO how can we make IN index work if we cache parts before selecting a projection? - /// XXX Used for IN set index analysis. Is this a proper way? - if (query_info.projection) - metadata_snapshot->selected_projection = query_info.projection->desc; - } - /// Do I need to perform the first part of the pipeline? /// Running on remote servers during distributed processing or if query is not distributed. /// @@ -1881,8 +1882,6 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (max_streams > 1 && !is_remote) max_streams *= settings.max_streams_to_max_threads_ratio; - // TODO figure out how to make set for projections - query_info.sets = query_analyzer->getPreparedSets(); auto & prewhere_info = analysis_result.prewhere_info; if (prewhere_info) @@ -2013,7 +2012,7 @@ void InterpreterSelectQuery::executeAggregation(QueryPlan & query_plan, const Ac expression_before_aggregation->setStepDescription("Before GROUP BY"); query_plan.addStep(std::move(expression_before_aggregation)); - if (options.ignore_aggregation) + if (options.is_projection_query) return; const auto & header_before_aggregation = query_plan.getCurrentDataStream().header; diff --git a/src/Interpreters/JoinSwitcher.h b/src/Interpreters/JoinSwitcher.h index 75ff7bb9b2c..a89ac6d5d98 100644 --- a/src/Interpreters/JoinSwitcher.h +++ b/src/Interpreters/JoinSwitcher.h @@ -31,9 +31,9 @@ public: join->joinBlock(block, not_processed); } - bool hasTotals() const override + const Block & getTotals() const override { - return join->hasTotals(); + return join->getTotals(); } void setTotals(const Block & block) override @@ -41,11 +41,6 @@ public: join->setTotals(block); } - void joinTotals(Block & block) const override - { - join->joinTotals(block); - } - size_t getTotalRowCount() const override { return join->getTotalRowCount(); diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 26463c8c6ed..46833f9769e 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -503,11 +503,6 @@ void MergeJoin::setTotals(const Block & totals_block) used_rows_bitmap = std::make_shared(getRightBlocksCount()); } -void MergeJoin::joinTotals(Block & block) const -{ - JoinCommon::joinTotals(totals, right_columns_to_add, *table_join, block); -} - void MergeJoin::mergeRightBlocks() { if (is_in_memory) diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index b6bde8fb131..8c829569a41 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -26,9 +26,10 @@ public: const TableJoin & getTableJoin() const override { return *table_join; } bool addJoinedBlock(const Block & block, bool check_limits) override; void joinBlock(Block &, ExtraBlockPtr & not_processed) override; - void joinTotals(Block &) const override; + void setTotals(const Block &) override; - bool hasTotals() const override { return totals; } + const Block & getTotals() const override { return totals; } + size_t getTotalRowCount() const override { return right_blocks.row_count; } size_t getTotalByteCount() const override { return right_blocks.bytes; } diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp index 399def00006..ecfda4cd0c1 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include @@ -13,12 +12,12 @@ namespace using namespace DB; Field executeFunctionOnField( - const Field & field, const std::string & name, + const Field & field, + const std::string & name, const ExpressionActionsPtr & sharding_expr, + const DataTypePtr & type, const std::string & sharding_key_column_name) { - DataTypePtr type = applyVisitor(FieldToDataType{}, field); - ColumnWithTypeAndName column; column.column = type->createColumnConst(1, field); column.name = name; @@ -34,25 +33,26 @@ Field executeFunctionOnField( /// @param sharding_column_value - one of values from IN /// @param sharding_column_name - name of that column -/// @param sharding_expr - expression of sharding_key for the Distributed() table -/// @param sharding_key_column_name - name of the column for sharding_expr -/// @param shard_info - info for the current shard (to compare shard_num with calculated) -/// @param slots - weight -> shard mapping /// @return true if shard may contain such value (or it is unknown), otherwise false. bool shardContains( - const Field & sharding_column_value, + Field sharding_column_value, const std::string & sharding_column_name, - const ExpressionActionsPtr & sharding_expr, - const std::string & sharding_key_column_name, - const Cluster::ShardInfo & shard_info, - const Cluster::SlotToShard & slots) + const OptimizeShardingKeyRewriteInMatcher::Data & data) { + UInt64 field_value; + /// Convert value to numeric (if required). + if (!sharding_column_value.tryGet(field_value)) + sharding_column_value = convertFieldToType(sharding_column_value, *data.sharding_key_type); + /// NULL is not allowed in sharding key, /// so it should be safe to assume that shard cannot contain it. if (sharding_column_value.isNull()) return false; - Field sharding_value = executeFunctionOnField(sharding_column_value, sharding_column_name, sharding_expr, sharding_key_column_name); + Field sharding_value = executeFunctionOnField( + sharding_column_value, sharding_column_name, + data.sharding_key_expr, data.sharding_key_type, + data.sharding_key_column_name); /// The value from IN can be non-numeric, /// but in this case it should be convertible to numeric type, let's try. sharding_value = convertFieldToType(sharding_value, DataTypeUInt64()); @@ -61,8 +61,8 @@ bool shardContains( return false; UInt64 value = sharding_value.get(); - const auto shard_num = slots[value % slots.size()] + 1; - return shard_info.shard_num == shard_num; + const auto shard_num = data.slots[value % data.slots.size()] + 1; + return data.shard_info.shard_num == shard_num; } } @@ -92,10 +92,7 @@ void OptimizeShardingKeyRewriteInMatcher::visit(ASTFunction & function, Data & d if (!identifier) return; - const auto & sharding_expr = data.sharding_key_expr; - const auto & sharding_key_column_name = data.sharding_key_column_name; - - if (!sharding_expr->getRequiredColumnsWithTypes().contains(identifier->name())) + if (!data.sharding_key_expr->getRequiredColumnsWithTypes().contains(identifier->name())) return; /// NOTE: that we should not take care about empty tuple, @@ -107,7 +104,7 @@ void OptimizeShardingKeyRewriteInMatcher::visit(ASTFunction & function, Data & d std::erase_if(tuple_elements->children, [&](auto & child) { auto * literal = child->template as(); - return literal && !shardContains(literal->value, identifier->name(), sharding_expr, sharding_key_column_name, data.shard_info, data.slots); + return literal && !shardContains(literal->value, identifier->name(), data); }); } else if (auto * tuple_literal = right->as(); @@ -116,7 +113,7 @@ void OptimizeShardingKeyRewriteInMatcher::visit(ASTFunction & function, Data & d auto & tuple = tuple_literal->value.get(); std::erase_if(tuple, [&](auto & child) { - return !shardContains(child, identifier->name(), sharding_expr, sharding_key_column_name, data.shard_info, data.slots); + return !shardContains(child, identifier->name(), data); }); } } diff --git a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h index 3087fb844ed..d546db40df7 100644 --- a/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h +++ b/src/Interpreters/OptimizeShardingKeyRewriteInVisitor.h @@ -25,9 +25,15 @@ struct OptimizeShardingKeyRewriteInMatcher struct Data { + /// Expression of sharding_key for the Distributed() table const ExpressionActionsPtr & sharding_key_expr; + /// Type of sharding_key column. + const DataTypePtr & sharding_key_type; + /// Name of the column for sharding_expr const std::string & sharding_key_column_name; + /// Info for the current shard (to compare shard_num with calculated) const Cluster::ShardInfo & shard_info; + /// weight -> shard mapping const Cluster::SlotToShard & slots; }; diff --git a/src/Interpreters/SelectQueryOptions.h b/src/Interpreters/SelectQueryOptions.h index 1a1f0267ab0..52ce7c83741 100644 --- a/src/Interpreters/SelectQueryOptions.h +++ b/src/Interpreters/SelectQueryOptions.h @@ -32,13 +32,14 @@ struct SelectQueryOptions bool remove_duplicates = false; bool ignore_quota = false; bool ignore_limits = false; - /// This is a temporary flag to avoid adding aggregating step. Used for projections. - /// TODO: we need more stages for InterpreterSelectQuery - bool ignore_aggregation = false; /// This flag is needed to analyze query ignoring table projections. /// It is needed because we build another one InterpreterSelectQuery while analyzing projections. /// It helps to avoid infinite recursion. bool ignore_projections = false; + /// This flag is also used for projection analysis. + /// It is needed because lazy normal projections require special planning in FetchColumns stage, such as adding WHERE transform. + /// It is also used to avoid adding aggregating step when aggregate projection is chosen. + bool is_projection_query = false; bool ignore_alias = false; bool is_internal = false; bool is_subquery = false; // non-subquery can also have subquery_depth > 0, e.g. insert select @@ -100,9 +101,9 @@ struct SelectQueryOptions return *this; } - SelectQueryOptions & ignoreAggregation(bool value = true) + SelectQueryOptions & projectionQuery(bool value = true) { - ignore_aggregation = value; + is_projection_query = value; return *this; } diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 5b55754f00a..99c08c70b7c 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -948,7 +948,8 @@ void executeQuery( WriteBuffer & ostr, bool allow_into_outfile, ContextMutablePtr context, - std::function set_result_details) + std::function set_result_details, + std::function before_finalize_callback) { PODArray parse_buf; const char * begin; @@ -1079,6 +1080,8 @@ void executeQuery( out->onProgress(progress); }); + out->setBeforeFinalizeCallback(before_finalize_callback); + if (set_result_details) set_result_details( context->getClientInfo().current_query_id, out->getContentType(), format_name, DateLUT::instance().getTimeZone()); diff --git a/src/Interpreters/executeQuery.h b/src/Interpreters/executeQuery.h index 6448b26a652..77f142de121 100644 --- a/src/Interpreters/executeQuery.h +++ b/src/Interpreters/executeQuery.h @@ -17,7 +17,8 @@ void executeQuery( WriteBuffer & ostr, /// Where to write query output to. bool allow_into_outfile, /// If true and the query contains INTO OUTFILE section, redirect output to that file. ContextMutablePtr context, /// DB, tables, data types, storage engines, functions, aggregate functions... - std::function set_result_details /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone. + std::function set_result_details, /// If a non-empty callback is passed, it will be called with the query id, the content-type, the format, and the timezone. + std::function before_finalize_callback = {} /// Will be set in output format to be called before finalize. ); diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 5548667e1a7..74f2c26a2ef 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -322,46 +322,26 @@ void createMissedColumns(Block & block) } /// Append totals from right to left block, correct types if needed -void joinTotals(const Block & totals, const Block & columns_to_add, const TableJoin & table_join, Block & block) +void joinTotals(Block left_totals, Block right_totals, const TableJoin & table_join, Block & out_block) { if (table_join.forceNullableLeft()) - convertColumnsToNullable(block); + JoinCommon::convertColumnsToNullable(left_totals); - if (Block totals_without_keys = totals) + if (table_join.forceNullableRight()) + JoinCommon::convertColumnsToNullable(right_totals); + + for (auto & col : out_block) { - for (const auto & name : table_join.keyNamesRight()) - totals_without_keys.erase(totals_without_keys.getPositionByName(name)); + if (const auto * left_col = left_totals.findByName(col.name)) + col = *left_col; + else if (const auto * right_col = right_totals.findByName(col.name)) + col = *right_col; + else + col.column = col.type->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst(); - for (auto & col : totals_without_keys) - { - if (table_join.rightBecomeNullable(col.type)) - JoinCommon::convertColumnToNullable(col); - - /// In case of arrayJoin it can be not one row - if (col.column->size() != 1) - col.column = col.column->cloneResized(1); - } - - for (size_t i = 0; i < totals_without_keys.columns(); ++i) - block.insert(totals_without_keys.safeGetByPosition(i)); - } - else - { - /// We will join empty `totals` - from one row with the default values. - - for (size_t i = 0; i < columns_to_add.columns(); ++i) - { - const auto & col = columns_to_add.getByPosition(i); - if (block.has(col.name)) - { - /// For StorageJoin we discarded table qualifiers, so some names may clash - continue; - } - block.insert({ - col.type->createColumnConstWithDefaultValue(1)->convertToFullColumnIfConst(), - col.type, - col.name}); - } + /// In case of using `arrayJoin` we can get more or less rows than one + if (col.column->size() != 1) + col.column = col.column->cloneResized(1); } } diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 9334b9d672f..2da795d0d4c 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -35,7 +35,7 @@ ColumnRawPtrs extractKeysForJoin(const Block & block_keys, const Names & key_nam void checkTypesOfKeys(const Block & block_left, const Names & key_names_left, const Block & block_right, const Names & key_names_right); void createMissedColumns(Block & block); -void joinTotals(const Block & totals, const Block & columns_to_add, const TableJoin & table_join, Block & block); +void joinTotals(Block left_totals, Block right_totals, const TableJoin & table_join, Block & out_block); void addDefaultValues(IColumn & column, const DataTypePtr & type, size_t count); diff --git a/src/Processors/Formats/IOutputFormat.cpp b/src/Processors/Formats/IOutputFormat.cpp index 88649d9ca25..7d82c267f36 100644 --- a/src/Processors/Formats/IOutputFormat.cpp +++ b/src/Processors/Formats/IOutputFormat.cpp @@ -76,6 +76,9 @@ void IOutputFormat::work() if (rows_before_limit_counter && rows_before_limit_counter->hasAppliedLimit()) setRowsBeforeLimit(rows_before_limit_counter->get()); + if (before_finalize_callback) + before_finalize_callback(); + finalize(); finalized = true; return; @@ -117,4 +120,3 @@ void IOutputFormat::write(const Block & block) } } - diff --git a/src/Processors/Formats/IOutputFormat.h b/src/Processors/Formats/IOutputFormat.h index 4c2b3f30070..4d86d18f70e 100644 --- a/src/Processors/Formats/IOutputFormat.h +++ b/src/Processors/Formats/IOutputFormat.h @@ -67,6 +67,9 @@ public: /// Passed value are delta, that must be summarized. virtual void onProgress(const Progress & /*progress*/) {} + /// Set callback, which will be called before call to finalize(). + void setBeforeFinalizeCallback(std::function callback) { before_finalize_callback = callback; } + /// Content-Type to set when sending HTTP response. virtual std::string getContentType() const { return "text/plain; charset=UTF-8"; } @@ -91,6 +94,7 @@ private: size_t result_bytes = 0; bool prefix_written = false; + + std::function before_finalize_callback; }; } - diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index 0f73349c271..0f6d90b720e 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -17,6 +17,22 @@ MySQLOutputFormat::MySQLOutputFormat(WriteBuffer & out_, const Block & header_, { } +void MySQLOutputFormat::setContext(ContextPtr context_) +{ + context = context_; + /// MySQlWire is a special format that is usually used as output format for MySQL protocol connections. + /// In this case we have to use the corresponding session context to set correct sequence_id. + mysql_context = getContext()->getMySQLProtocolContext(); + if (!mysql_context) + { + /// But it's also possible to specify MySQLWire as output format for clickhouse-client or clickhouse-local. + /// There is no MySQL protocol context in this case, so we create dummy one. + own_mysql_context.emplace(); + mysql_context = &own_mysql_context.value(); + } + packet_endpoint = mysql_context->makeEndpoint(out); +} + void MySQLOutputFormat::initialize() { if (initialized) @@ -40,7 +56,7 @@ void MySQLOutputFormat::initialize() packet_endpoint->sendPacket(getColumnDefinition(column_name, data_types[i]->getTypeId())); } - if (!(getContext()->mysql.client_capabilities & Capability::CLIENT_DEPRECATE_EOF)) + if (!(mysql_context->client_capabilities & Capability::CLIENT_DEPRECATE_EOF)) { packet_endpoint->sendPacket(EOFPacket(0, 0)); } @@ -79,10 +95,10 @@ void MySQLOutputFormat::finalize() const auto & header = getPort(PortKind::Main).getHeader(); if (header.columns() == 0) packet_endpoint->sendPacket( - OKPacket(0x0, getContext()->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); - else if (getContext()->mysql.client_capabilities & CLIENT_DEPRECATE_EOF) + OKPacket(0x0, mysql_context->client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); + else if (mysql_context->client_capabilities & CLIENT_DEPRECATE_EOF) packet_endpoint->sendPacket( - OKPacket(0xfe, getContext()->mysql.client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); + OKPacket(0xfe, mysql_context->client_capabilities, affected_rows, 0, 0, "", human_readable_info), true); else packet_endpoint->sendPacket(EOFPacket(0, 0), true); } diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.h b/src/Processors/Formats/Impl/MySQLOutputFormat.h index 7d67df3015e..fed2a431860 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.h +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.h @@ -25,11 +25,7 @@ public: String getName() const override { return "MySQLOutputFormat"; } - void setContext(ContextPtr context_) - { - context = context_; - packet_endpoint = std::make_unique(out, const_cast(getContext()->mysql.sequence_id)); /// TODO: fix it - } + void setContext(ContextPtr context_); void consume(Chunk) override; void finalize() override; @@ -41,7 +37,9 @@ public: private: bool initialized = false; - std::unique_ptr packet_endpoint; + std::optional own_mysql_context; + MySQLWireContext * mysql_context = nullptr; + MySQLProtocol::PacketEndpointPtr packet_endpoint; FormatSettings format_settings; DataTypes data_types; Serializations serializations; diff --git a/src/Processors/QueryPlan/JoinStep.cpp b/src/Processors/QueryPlan/JoinStep.cpp index b06d6628dcb..736d7eb37c1 100644 --- a/src/Processors/QueryPlan/JoinStep.cpp +++ b/src/Processors/QueryPlan/JoinStep.cpp @@ -70,7 +70,7 @@ FilledJoinStep::FilledJoinStep(const DataStream & input_stream_, JoinPtr join_, void FilledJoinStep::transformPipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) { bool default_totals = false; - if (!pipeline.hasTotals() && join->hasTotals()) + if (!pipeline.hasTotals() && join->getTotals()) { pipeline.addDefaultTotals(); default_totals = true; diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 2dc8246cde7..3cf1bdcb4d8 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -829,7 +829,8 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre log, requested_num_streams, result.index_stats, - true); + true /* use_skip_indexes */, + true /* check_limits */); size_t sum_marks_pk = total_marks_pk; for (const auto & stat : result.index_stats) diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index 31b2da46ab3..87f22319bba 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include #include @@ -159,19 +159,16 @@ void JoiningTransform::transform(Chunk & chunk) Block block; if (on_totals) { - /// We have to make chunk empty before return - /// In case of using `arrayJoin` we can get more or less rows than one - auto cols = chunk.detachColumns(); - for (auto & col : cols) - col = col->cloneResized(1); - block = inputs.front().getHeader().cloneWithColumns(std::move(cols)); + const auto & left_totals = inputs.front().getHeader().cloneWithColumns(chunk.detachColumns()); + const auto & right_totals = join->getTotals(); /// Drop totals if both out stream and joined stream doesn't have ones. /// See comment in ExpressionTransform.h - if (default_totals && !join->hasTotals()) + if (default_totals && !right_totals) return; - join->joinTotals(block); + block = outputs.front().getHeader().cloneEmpty(); + JoinCommon::joinTotals(left_totals, right_totals, join->getTableJoin(), block); } else block = readExecute(chunk); diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index beace5dd576..b8913f5e64f 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -95,10 +95,11 @@ void MySQLHandler::run() connection_context->getClientInfo().interface = ClientInfo::Interface::MYSQL; connection_context->setDefaultFormat("MySQLWire"); connection_context->getClientInfo().connection_id = connection_id; + connection_context->setMySQLProtocolContext(&connection_context_mysql); in = std::make_shared(socket()); out = std::make_shared(socket()); - packet_endpoint = std::make_shared(*in, *out, connection_context->mysql.sequence_id); + packet_endpoint = connection_context_mysql.makeEndpoint(*in, *out); try { @@ -110,11 +111,11 @@ void MySQLHandler::run() HandshakeResponse handshake_response; finishHandshake(handshake_response); - connection_context->mysql.client_capabilities = handshake_response.capability_flags; + connection_context_mysql.client_capabilities = handshake_response.capability_flags; if (handshake_response.max_packet_size) - connection_context->mysql.max_packet_size = handshake_response.max_packet_size; - if (!connection_context->mysql.max_packet_size) - connection_context->mysql.max_packet_size = MAX_PACKET_LENGTH; + connection_context_mysql.max_packet_size = handshake_response.max_packet_size; + if (!connection_context_mysql.max_packet_size) + connection_context_mysql.max_packet_size = MAX_PACKET_LENGTH; LOG_TRACE(log, "Capabilities: {}, max_packet_size: {}, character_set: {}, user: {}, auth_response length: {}, database: {}, auth_plugin_name: {}", @@ -395,14 +396,14 @@ void MySQLHandlerSSL::finishHandshakeSSL( ReadBufferFromMemory payload(buf, pos); payload.ignore(PACKET_HEADER_SIZE); ssl_request.readPayloadWithUnpacked(payload); - connection_context->mysql.client_capabilities = ssl_request.capability_flags; - connection_context->mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH; + connection_context_mysql.client_capabilities = ssl_request.capability_flags; + connection_context_mysql.max_packet_size = ssl_request.max_packet_size ? ssl_request.max_packet_size : MAX_PACKET_LENGTH; secure_connection = true; ss = std::make_shared(SecureStreamSocket::attach(socket(), SSLManager::instance().defaultServerContext())); in = std::make_shared(*ss); out = std::make_shared(*ss); - connection_context->mysql.sequence_id = 2; - packet_endpoint = std::make_shared(*in, *out, connection_context->mysql.sequence_id); + connection_context_mysql.sequence_id = 2; + packet_endpoint = connection_context_mysql.makeEndpoint(*in, *out); packet_endpoint->receivePacket(packet); /// Reading HandshakeResponse from secure socket. } diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index e681ad2e6f6..2ea5695a0a6 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -56,9 +56,10 @@ private: protected: Poco::Logger * log; + MySQLWireContext connection_context_mysql; ContextMutablePtr connection_context; - std::shared_ptr packet_endpoint; + MySQLProtocol::PacketEndpointPtr packet_endpoint; private: UInt64 connection_id = 0; diff --git a/src/Server/PrometheusMetricsWriter.cpp b/src/Server/PrometheusMetricsWriter.cpp index 787f0fcd95e..30ae6f6fe42 100644 --- a/src/Server/PrometheusMetricsWriter.cpp +++ b/src/Server/PrometheusMetricsWriter.cpp @@ -4,7 +4,7 @@ #include #include -#include +#include namespace { @@ -24,9 +24,13 @@ void writeOutLine(DB::WriteBuffer & wb, T && val, TArgs &&... args) writeOutLine(wb, std::forward(args)...); } -void replaceInvalidChars(std::string & metric_name) +/// Returns false if name is not valid +bool replaceInvalidChars(std::string & metric_name) { - std::replace(metric_name.begin(), metric_name.end(), '.', '_'); + /// dirty solution + metric_name = std::regex_replace(metric_name, std::regex("[^a-zA-Z0-9_:]"), "_"); + metric_name = std::regex_replace(metric_name, std::regex("^[^a-zA-Z]*"), ""); + return !metric_name.empty(); } } @@ -57,7 +61,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const std::string metric_name{ProfileEvents::getName(static_cast(i))}; std::string metric_doc{ProfileEvents::getDocumentation(static_cast(i))}; - replaceInvalidChars(metric_name); + if (!replaceInvalidChars(metric_name)) + continue; std::string key{profile_events_prefix + metric_name}; writeOutLine(wb, "# HELP", key, metric_doc); @@ -75,7 +80,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const std::string metric_name{CurrentMetrics::getName(static_cast(i))}; std::string metric_doc{CurrentMetrics::getDocumentation(static_cast(i))}; - replaceInvalidChars(metric_name); + if (!replaceInvalidChars(metric_name)) + continue; std::string key{current_metrics_prefix + metric_name}; writeOutLine(wb, "# HELP", key, metric_doc); @@ -91,7 +97,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const { std::string key{asynchronous_metrics_prefix + name_value.first}; - replaceInvalidChars(key); + if (!replaceInvalidChars(key)) + continue; auto value = name_value.second; // TODO: add HELP section? asynchronous_metrics contains only key and value @@ -108,7 +115,8 @@ void PrometheusMetricsWriter::write(WriteBuffer & wb) const std::string metric_name{CurrentStatusInfo::getName(static_cast(i))}; std::string metric_doc{CurrentStatusInfo::getDocumentation(static_cast(i))}; - replaceInvalidChars(metric_name); + if (!replaceInvalidChars(metric_name)) + continue; std::string key{current_status_prefix + metric_name}; writeOutLine(wb, "# HELP", key, metric_doc); diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 9a50cec5986..c0d7541eacc 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -111,6 +111,7 @@ DistributedBlockOutputStream::DistributedBlockOutputStream( if (settings.max_distributed_depth && context->getClientInfo().distributed_depth > settings.max_distributed_depth) throw Exception("Maximum distributed depth exceeded", ErrorCodes::TOO_LARGE_DISTRIBUTED_DEPTH); context->getClientInfo().distributed_depth += 1; + random_shard_insert = settings.insert_distributed_one_random_shard && !storage.has_sharding_key; } @@ -156,9 +157,6 @@ void DistributedBlockOutputStream::write(const Block & block) void DistributedBlockOutputStream::writeAsync(const Block & block) { - const Settings & settings = context->getSettingsRef(); - bool random_shard_insert = settings.insert_distributed_one_random_shard && !storage.has_sharding_key; - if (random_shard_insert) { writeAsyncImpl(block, storage.getRandomShardIndex(cluster->getShardsInfo())); @@ -264,11 +262,19 @@ void DistributedBlockOutputStream::waitForJobs() } } - size_t jobs_count = remote_jobs_count + local_jobs_count; size_t num_finished_jobs = finished_jobs_count; + if (random_shard_insert) + { + if (finished_jobs_count != 1) + LOG_WARNING(log, "Expected 1 writing jobs when doing random shard insert, but finished {}", num_finished_jobs); + } + else + { + size_t jobs_count = remote_jobs_count + local_jobs_count; - if (num_finished_jobs < jobs_count) - LOG_WARNING(log, "Expected {} writing jobs, but finished only {}", jobs_count, num_finished_jobs); + if (num_finished_jobs < jobs_count) + LOG_WARNING(log, "Expected {} writing jobs, but finished only {}", jobs_count, num_finished_jobs); + } } @@ -401,7 +407,6 @@ void DistributedBlockOutputStream::writeSync(const Block & block) { const Settings & settings = context->getSettingsRef(); const auto & shards_info = cluster->getShardsInfo(); - bool random_shard_insert = settings.insert_distributed_one_random_shard && !storage.has_sharding_key; size_t start = 0; size_t end = shards_info.size(); @@ -410,20 +415,13 @@ void DistributedBlockOutputStream::writeSync(const Block & block) start = settings.insert_shard_id - 1; end = settings.insert_shard_id; } - else if (random_shard_insert) - { - start = storage.getRandomShardIndex(shards_info); - end = start + 1; - } - - size_t num_shards = end - start; if (!pool) { /// Deferred initialization. Only for sync insertion. initWritingJobs(block, start, end); - size_t jobs_count = remote_jobs_count + local_jobs_count; + size_t jobs_count = random_shard_insert ? 1 : (remote_jobs_count + local_jobs_count); size_t max_threads = std::min(settings.max_distributed_connections, jobs_count); pool.emplace(/* max_threads_= */ max_threads, /* max_free_threads_= */ max_threads, @@ -440,12 +438,20 @@ void DistributedBlockOutputStream::writeSync(const Block & block) watch_current_block.restart(); + if (random_shard_insert) + { + start = storage.getRandomShardIndex(shards_info); + end = start + 1; + } + + size_t num_shards = end - start; + if (num_shards > 1) { auto current_selector = createSelector(block); - /// Prepare row numbers for each shard - for (size_t shard_index : collections::range(0, num_shards)) + /// Prepare row numbers for needed shards + for (size_t shard_index : collections::range(start, end)) per_shard_jobs[shard_index].shard_current_block_permutation.resize(0); for (size_t i = 0; i < block.rows(); ++i) @@ -456,7 +462,7 @@ void DistributedBlockOutputStream::writeSync(const Block & block) { /// Run jobs in parallel for each block and wait them finished_jobs_count = 0; - for (size_t shard_index : collections::range(0, shards_info.size())) + for (size_t shard_index : collections::range(start, end)) for (JobReplica & job : per_shard_jobs[shard_index].replicas_jobs) pool->scheduleOrThrowOnError(runWritingJob(job, block, num_shards)); } diff --git a/src/Storages/Distributed/DistributedBlockOutputStream.h b/src/Storages/Distributed/DistributedBlockOutputStream.h index 0ae57ce053d..8e6e914cb29 100644 --- a/src/Storages/Distributed/DistributedBlockOutputStream.h +++ b/src/Storages/Distributed/DistributedBlockOutputStream.h @@ -94,6 +94,7 @@ private: size_t inserted_rows = 0; bool insert_sync; + bool random_shard_insert; bool allow_materialized; /// Sync-related stuff diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index ff6101fce07..e30da82416d 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -15,7 +16,8 @@ #include #include #include - +#include +#include namespace fs = std::filesystem; @@ -40,6 +42,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int S3_ERROR; extern const int INCORRECT_PART_TYPE; + extern const int ZERO_COPY_REPLICATION_ERROR; } namespace DataPartsExchange @@ -52,7 +55,7 @@ constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE_AND_TTL_INFOS = 2; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_TYPE = 3; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_DEFAULT_COMPRESSION = 4; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID = 5; -constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY = 6; +constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY = 6; constexpr auto REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION = 7; @@ -168,27 +171,27 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) writeUUIDText(part->uuid, out); - bool try_use_s3_copy = false; + String remote_fs_metadata = parse(params.get("remote_fs_metadata", "")); + std::regex re("\\s*,\\s*"); + Strings capability( + std::sregex_token_iterator(remote_fs_metadata.begin(), remote_fs_metadata.end(), re, -1), + std::sregex_token_iterator()); - if (data_settings->allow_s3_zero_copy_replication - && client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY) - { /// if source and destination are in the same S3 storage we try to use S3 CopyObject request first - int send_s3_metadata = parse(params.get("send_s3_metadata", "0")); - if (send_s3_metadata == 1) + if (data_settings->allow_remote_fs_zero_copy_replication && + client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY) + { + auto disk = part->volume->getDisk(); + auto disk_type = DiskType::toString(disk->getType()); + if (disk->supportZeroCopyReplication() && std::find(capability.begin(), capability.end(), disk_type) != capability.end()) { - auto disk = part->volume->getDisk(); - if (disk->getType() == DB::DiskType::Type::S3) - { - try_use_s3_copy = true; - } + /// Send metadata if the receiver's capability covers the source disk type. + response.addCookie({"remote_fs_metadata", disk_type}); + sendPartFromDiskRemoteMeta(part, out); + return; } } - if (try_use_s3_copy) - { - response.addCookie({"send_s3_metadata", "1"}); - sendPartS3Metadata(part, out); - } - else if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION) + + if (client_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION) { const auto & projections = part->getProjectionParts(); writeBinary(projections.size(), out); @@ -323,7 +326,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( return data_checksums; } -void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteBuffer & out) +void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part, WriteBuffer & out) { /// We'll take a list of files from the list of checksums. MergeTreeData::DataPart::Checksums checksums = part->checksums; @@ -333,8 +336,8 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB checksums.files[file_name] = {}; auto disk = part->volume->getDisk(); - if (disk->getType() != DB::DiskType::Type::S3) - throw Exception("S3 disk is not S3 anymore", ErrorCodes::LOGICAL_ERROR); + if (!disk->supportZeroCopyReplication()) + throw Exception(fmt::format("disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR); part->storage.lockSharedData(*part); @@ -351,9 +354,9 @@ void Service::sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteB fs::path metadata(metadata_file); if (!fs::exists(metadata)) - throw Exception("S3 metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); + throw Exception("Remote metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); if (!fs::is_regular_file(metadata)) - throw Exception("S3 metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); + throw Exception("Remote metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); UInt64 file_size = fs::file_size(metadata); writeStringBinary(it.first, out); @@ -399,8 +402,8 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( bool to_detached, const String & tmp_prefix_, std::optional * tagger_ptr, - bool try_use_s3_copy, - const DiskPtr disk_s3) + bool try_zero_copy, + DiskPtr disk) { if (blocker.isCancelled()) throw Exception("Fetching of part was cancelled", ErrorCodes::ABORTED); @@ -421,30 +424,34 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( {"compress", "false"} }); - if (try_use_s3_copy && disk_s3 && disk_s3->getType() != DB::DiskType::Type::S3) - throw Exception("Try to fetch shared s3 part on non-s3 disk", ErrorCodes::LOGICAL_ERROR); - - Disks disks_s3; - - if (!data_settings->allow_s3_zero_copy_replication) - try_use_s3_copy = false; - - if (try_use_s3_copy) + Strings capability; + if (try_zero_copy && data_settings->allow_remote_fs_zero_copy_replication) { - if (disk_s3) - disks_s3.push_back(disk_s3); - else + if (!disk) { - disks_s3 = data.getDisksByType(DiskType::Type::S3); - - if (disks_s3.empty()) - try_use_s3_copy = false; + DiskType::Type zero_copy_disk_types[] = {DiskType::Type::S3, DiskType::Type::HDFS}; + for (auto disk_type: zero_copy_disk_types) + { + Disks disks = data.getDisksByType(disk_type); + if (!disks.empty()) + { + capability.push_back(DiskType::toString(disk_type)); + } + } + } + else if (disk->supportZeroCopyReplication()) + { + capability.push_back(DiskType::toString(disk->getType())); } } - - if (try_use_s3_copy) + if (!capability.empty()) { - uri.addQueryParameter("send_s3_metadata", "1"); + const String & remote_fs_metadata = boost::algorithm::join(capability, ", "); + uri.addQueryParameter("remote_fs_metadata", remote_fs_metadata); + } + else + { + try_zero_copy = false; } Poco::Net::HTTPBasicCredentials creds{}; @@ -467,73 +474,6 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( int server_protocol_version = parse(in.getResponseCookie("server_protocol_version", "0")); - int send_s3 = parse(in.getResponseCookie("send_s3_metadata", "0")); - - if (send_s3 == 1) - { - if (server_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_S3_COPY) - throw Exception("Got 'send_s3_metadata' cookie with old protocol version", ErrorCodes::LOGICAL_ERROR); - if (!try_use_s3_copy) - throw Exception("Got 'send_s3_metadata' cookie when was not requested", ErrorCodes::LOGICAL_ERROR); - - size_t sum_files_size = 0; - readBinary(sum_files_size, in); - IMergeTreeDataPart::TTLInfos ttl_infos; - String ttl_infos_string; - readBinary(ttl_infos_string, in); - ReadBufferFromString ttl_infos_buffer(ttl_infos_string); - assertString("ttl format version: 1\n", ttl_infos_buffer); - ttl_infos.read(ttl_infos_buffer); - - ReservationPtr reservation - = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, &ttl_infos, true); - if (!reservation) - reservation - = data.reserveSpacePreferringTTLRules(metadata_snapshot, sum_files_size, ttl_infos, std::time(nullptr), 0, true); - if (reservation) - { - /// When we have multi-volume storage, one of them was chosen, depends on TTL, free space, etc. - /// Chosen one may be S3 or not. - DiskPtr disk = reservation->getDisk(); - if (disk && disk->getType() == DiskType::Type::S3) - { - for (const auto & d : disks_s3) - { - if (d->getPath() == disk->getPath()) - { - Disks disks_tmp = { disk }; - disks_s3.swap(disks_tmp); - break; - } - } - } - } - - String part_type = "Wide"; - readStringBinary(part_type, in); - if (part_type == "InMemory") - throw Exception("Got 'send_s3_metadata' cookie for in-memory part", ErrorCodes::INCORRECT_PART_TYPE); - - UUID part_uuid = UUIDHelpers::Nil; - - /// Always true due to values of constants. But we keep this condition just in case. - if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) //-V547 - readUUIDText(part_uuid, in); - - try - { - return downloadPartToS3(part_name, replica_path, to_detached, tmp_prefix_, std::move(disks_s3), in, throttler); - } - catch (const Exception & e) - { - if (e.code() != ErrorCodes::S3_ERROR) - throw; - /// Try again but without S3 copy - return fetchPart(metadata_snapshot, context, part_name, replica_path, host, port, timeouts, - user, password, interserver_scheme, throttler, to_detached, tmp_prefix_, nullptr, false); - } - } - ReservationPtr reservation; size_t sum_files_size = 0; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_SIZE) @@ -547,24 +487,29 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( ReadBufferFromString ttl_infos_buffer(ttl_infos_string); assertString("ttl format version: 1\n", ttl_infos_buffer); ttl_infos.read(ttl_infos_buffer); - reservation - = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, &ttl_infos, true); - if (!reservation) + if (!disk) + { reservation - = data.reserveSpacePreferringTTLRules(metadata_snapshot, sum_files_size, ttl_infos, std::time(nullptr), 0, true); + = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, &ttl_infos, true); + if (!reservation) + reservation + = data.reserveSpacePreferringTTLRules(metadata_snapshot, sum_files_size, ttl_infos, std::time(nullptr), 0, true); + } } - else + else if (!disk) { reservation = data.balancedReservation(metadata_snapshot, sum_files_size, 0, part_name, part_info, {}, tagger_ptr, nullptr); if (!reservation) reservation = data.reserveSpace(sum_files_size); } } - else + else if (!disk) { /// We don't know real size of part because sender server version is too old reservation = data.makeEmptyReservationOnLargestDisk(); } + if (!disk) + disk = reservation->getDisk(); bool sync = (data_settings->min_compressed_bytes_to_fsync_after_fetch && sum_files_size >= data_settings->min_compressed_bytes_to_fsync_after_fetch); @@ -577,8 +522,35 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_UUID) readUUIDText(part_uuid, in); + String remote_fs_metadata = parse(in.getResponseCookie("remote_fs_metadata", "")); + if (!remote_fs_metadata.empty()) + { + if (!try_zero_copy) + throw Exception("Got unexpected 'remote_fs_metadata' cookie", ErrorCodes::LOGICAL_ERROR); + if (std::find(capability.begin(), capability.end(), remote_fs_metadata) == capability.end()) + throw Exception(fmt::format("Got 'remote_fs_metadata' cookie {}, expect one from {}", remote_fs_metadata, fmt::join(capability, ", ")), ErrorCodes::LOGICAL_ERROR); + if (server_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY) + throw Exception(fmt::format("Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version), ErrorCodes::LOGICAL_ERROR); + if (part_type == "InMemory") + throw Exception("Got 'remote_fs_metadata' cookie for in-memory part", ErrorCodes::INCORRECT_PART_TYPE); + + try + { + return downloadPartToDiskRemoteMeta(part_name, replica_path, to_detached, tmp_prefix_, disk, in, throttler); + } + catch (const Exception & e) + { + if (e.code() != ErrorCodes::S3_ERROR && e.code() != ErrorCodes::ZERO_COPY_REPLICATION_ERROR) + throw; + LOG_WARNING(log, e.message() + " Will retry fetching part without zero-copy."); + /// Try again but without zero-copy + return fetchPart(metadata_snapshot, context, part_name, replica_path, host, port, timeouts, + user, password, interserver_scheme, throttler, to_detached, tmp_prefix_, nullptr, false, disk); + } + } + auto storage_id = data.getStorageID(); - String new_part_path = part_type == "InMemory" ? "memory" : fs::path(data.getFullPathOnDisk(reservation->getDisk())) / part_name / ""; + String new_part_path = part_type == "InMemory" ? "memory" : fs::path(data.getFullPathOnDisk(disk)) / part_name / ""; auto entry = data.getContext()->getReplicatedFetchList().insert( storage_id.getDatabaseName(), storage_id.getTableName(), part_info.partition_id, part_name, new_part_path, @@ -586,15 +558,14 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( in.setNextCallback(ReplicatedFetchReadCallback(*entry)); - size_t projections = 0; if (server_protocol_version >= REPLICATION_PROTOCOL_VERSION_WITH_PARTS_PROJECTION) readBinary(projections, in); MergeTreeData::DataPart::Checksums checksums; return part_type == "InMemory" - ? downloadPartToMemory(part_name, part_uuid, metadata_snapshot, context, std::move(reservation), in, projections, throttler) - : downloadPartToDisk(part_name, replica_path, to_detached, tmp_prefix_, sync, reservation->getDisk(), in, projections, checksums, throttler); + ? downloadPartToMemory(part_name, part_uuid, metadata_snapshot, context, disk, in, projections, throttler) + : downloadPartToDisk(part_name, replica_path, to_detached, tmp_prefix_, sync, disk, in, projections, checksums, throttler); } MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( @@ -602,12 +573,12 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( const UUID & part_uuid, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, - ReservationPtr reservation, + DiskPtr disk, PooledReadWriteBufferFromHTTP & in, size_t projections, ThrottlerPtr throttler) { - auto volume = std::make_shared("volume_" + part_name, reservation->getDisk(), 0); + auto volume = std::make_shared("volume_" + part_name, disk, 0); MergeTreeData::MutableDataPartPtr new_data_part = std::make_shared(data, part_name, volume); @@ -794,31 +765,24 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDisk( return new_data_part; } -MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToS3( +MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( const String & part_name, const String & replica_path, bool to_detached, const String & tmp_prefix_, - const Disks & disks_s3, + DiskPtr disk, PooledReadWriteBufferFromHTTP & in, ThrottlerPtr throttler) { - if (disks_s3.empty()) - throw Exception("No S3 disks anymore", ErrorCodes::LOGICAL_ERROR); - String part_id; readStringBinary(part_id, in); - DiskPtr disk = disks_s3[0]; - - for (const auto & disk_s3 : disks_s3) + if (!disk->supportZeroCopyReplication() || !disk->checkUniqueId(part_id)) { - if (disk_s3->checkUniqueId(part_id)) - { - disk = disk_s3; - break; - } + throw Exception(fmt::format("Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName()), ErrorCodes::ZERO_COPY_REPLICATION_ERROR); } + LOG_DEBUG(log, "Downloading Part {} unique id {} metadata onto disk {}.", + part_name, part_id, disk->getName()); static const String TMP_PREFIX = "tmp_fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index eb776c33f0f..0c12cc51cc7 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -50,7 +50,7 @@ private: int client_protocol_version, const std::map> & projections = {}); - void sendPartS3Metadata(const MergeTreeData::DataPartPtr & part, WriteBuffer & out); + void sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part, WriteBuffer & out); /// StorageReplicatedMergeTree::shutdown() waits for all parts exchange handlers to finish, /// so Service will never access dangling reference to storage @@ -81,8 +81,8 @@ public: bool to_detached = false, const String & tmp_prefix_ = "", std::optional * tagger_ptr = nullptr, - bool try_use_s3_copy = true, - const DiskPtr disk_s3 = nullptr); + bool try_zero_copy = true, + DiskPtr dest_disk = nullptr); /// You need to stop the data transfer. ActionBlocker blocker; @@ -115,17 +115,17 @@ private: const UUID & part_uuid, const StorageMetadataPtr & metadata_snapshot, ContextPtr context, - ReservationPtr reservation, + DiskPtr disk, PooledReadWriteBufferFromHTTP & in, size_t projections, ThrottlerPtr throttler); - MergeTreeData::MutableDataPartPtr downloadPartToS3( + MergeTreeData::MutableDataPartPtr downloadPartToDiskRemoteMeta( const String & part_name, const String & replica_path, bool to_detached, const String & tmp_prefix_, - const Disks & disks_s3, + DiskPtr disk, PooledReadWriteBufferFromHTTP & in, ThrottlerPtr throttler); diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 8fe6a0a484b..70e72c85e79 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1103,7 +1103,7 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ std::optional IMergeTreeDataPart::keepSharedDataInDecoupledStorage() const { - /// NOTE: It's needed for S3 zero-copy replication + /// NOTE: It's needed for zero-copy replication if (force_keep_shared_data) return true; @@ -1501,16 +1501,11 @@ SerializationPtr IMergeTreeDataPart::getSerializationForColumn(const NameAndType String IMergeTreeDataPart::getUniqueId() const { - String id; - auto disk = volume->getDisk(); + if (!disk->supportZeroCopyReplication()) + throw Exception(fmt::format("Disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR); - if (disk->getType() == DB::DiskType::Type::S3) - id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt"); - - if (id.empty()) - throw Exception("Can't get unique S3 object", ErrorCodes::LOGICAL_ERROR); - + String id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt"); return id; } diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 54fcfc1adc9..3c2c2d44271 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -374,7 +374,7 @@ public: void loadProjections(bool require_columns_checksums, bool check_consistency); - /// Return set of metadat file names without checksums. For example, + /// Return set of metadata file names without checksums. For example, /// columns.txt or checksums.txt itself. NameSet getFileNamesWithoutChecksums() const; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index f311d58b7af..ac0cd939646 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -104,6 +104,7 @@ namespace ErrorCodes extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int LOGICAL_ERROR; extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER; extern const int CORRUPTED_DATA; extern const int BAD_TYPE_OF_FIELD; extern const int BAD_ARGUMENTS; @@ -125,12 +126,36 @@ namespace ErrorCodes extern const int TOO_MANY_SIMULTANEOUS_QUERIES; } - -static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key) +static void checkSampleExpression(const StorageInMemoryMetadata & metadata, bool allow_sampling_expression_not_in_primary_key, bool check_sample_column_is_correct) { const auto & pk_sample_block = metadata.getPrimaryKey().sample_block; if (!pk_sample_block.has(metadata.sampling_key.column_names[0]) && !allow_sampling_expression_not_in_primary_key) throw Exception("Sampling expression must be present in the primary key", ErrorCodes::BAD_ARGUMENTS); + + if (!check_sample_column_is_correct) + return; + + const auto & sampling_key = metadata.getSamplingKey(); + DataTypePtr sampling_column_type = sampling_key.data_types[0]; + + bool is_correct_sample_condition = false; + if (sampling_key.data_types.size() == 1) + { + if (typeid_cast(sampling_column_type.get())) + is_correct_sample_condition = true; + else if (typeid_cast(sampling_column_type.get())) + is_correct_sample_condition = true; + else if (typeid_cast(sampling_column_type.get())) + is_correct_sample_condition = true; + else if (typeid_cast(sampling_column_type.get())) + is_correct_sample_condition = true; + } + + if (!is_correct_sample_condition) + throw Exception( + "Invalid sampling column type in storage parameters: " + sampling_column_type->getName() + + ". Must be one unsigned integer type", + ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER); } MergeTreeData::MergeTreeData( @@ -200,7 +225,8 @@ MergeTreeData::MergeTreeData( if (metadata_.sampling_key.definition_ast != nullptr) { /// This is for backward compatibility. - checkSampleExpression(metadata_, attach || settings->compatibility_allow_sampling_expression_not_in_primary_key); + checkSampleExpression(metadata_, attach || settings->compatibility_allow_sampling_expression_not_in_primary_key, + settings->check_sample_column_is_correct); } checkTTLExpressions(metadata_, metadata_); @@ -270,19 +296,17 @@ StoragePolicyPtr MergeTreeData::getStoragePolicy() const static void checkKeyExpression(const ExpressionActions & expr, const Block & sample_block, const String & key_name, bool allow_nullable_key) { - for (const auto & action : expr.getActions()) - { - if (action.node->type == ActionsDAG::ActionType::ARRAY_JOIN) - throw Exception(key_name + " key cannot contain array joins", ErrorCodes::ILLEGAL_COLUMN); + if (expr.hasArrayJoin()) + throw Exception(key_name + " key cannot contain array joins", ErrorCodes::ILLEGAL_COLUMN); - if (action.node->type == ActionsDAG::ActionType::FUNCTION) - { - IFunctionBase & func = *action.node->function_base; - if (!func.isDeterministic()) - throw Exception(key_name + " key cannot contain non-deterministic functions, " - "but contains function " + func.getName(), - ErrorCodes::BAD_ARGUMENTS); - } + try + { + expr.assertDeterministic(); + } + catch (Exception & e) + { + e.addMessage(fmt::format("for {} key", key_name)); + throw; } for (const ColumnWithTypeAndName & element : sample_block) @@ -418,7 +442,6 @@ void MergeTreeData::checkProperties( } checkKeyExpression(*new_sorting_key.expression, new_sorting_key.sample_block, "Sorting", allow_nullable_key); - } void MergeTreeData::setProperties(const StorageInMemoryMetadata & new_metadata, const StorageInMemoryMetadata & old_metadata, bool attach) @@ -1088,7 +1111,7 @@ static bool isOldPartDirectory(const DiskPtr & disk, const String & directory_pa } -void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_lifetime_seconds) +void MergeTreeData::clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds) { /// If the method is already called from another thread, then we don't need to do anything. std::unique_lock lock(clear_old_temporary_directories_mutex, std::defer_lock); @@ -1097,9 +1120,7 @@ void MergeTreeData::clearOldTemporaryDirectories(ssize_t custom_directories_life const auto settings = getSettings(); time_t current_time = time(nullptr); - ssize_t deadline = (custom_directories_lifetime_seconds >= 0) - ? current_time - custom_directories_lifetime_seconds - : current_time - settings->temporary_directories_lifetime.totalSeconds(); + ssize_t deadline = current_time - custom_directories_lifetime_seconds; /// Delete temporary directories older than a day. for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) @@ -1675,7 +1696,8 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context "ALTER MODIFY SAMPLE BY is not supported for default-partitioned tables created with the old syntax", ErrorCodes::BAD_ARGUMENTS); - checkSampleExpression(new_metadata, getSettings()->compatibility_allow_sampling_expression_not_in_primary_key); + checkSampleExpression(new_metadata, getSettings()->compatibility_allow_sampling_expression_not_in_primary_key, + getSettings()->check_sample_column_is_correct); } if (command.type == AlterCommand::ADD_INDEX && !is_custom_partitioned) { @@ -2750,19 +2772,17 @@ void MergeTreeData::swapActivePart(MergeTreeData::DataPartPtr part_copy) if (active_part_it == data_parts_by_info.end()) throw Exception("Cannot swap part '" + part_copy->name + "', no such active part.", ErrorCodes::NO_SUCH_DATA_PART); - /// We do not check allow_s3_zero_copy_replication here because data may be shared - /// when allow_s3_zero_copy_replication turned on and off again + /// We do not check allow_remote_fs_zero_copy_replication here because data may be shared + /// when allow_remote_fs_zero_copy_replication turned on and off again original_active_part->force_keep_shared_data = false; - if (original_active_part->volume->getDisk()->getType() == DiskType::Type::S3) + if (original_active_part->volume->getDisk()->supportZeroCopyReplication() && + part_copy->volume->getDisk()->supportZeroCopyReplication() && + original_active_part->getUniqueId() == part_copy->getUniqueId()) { - if (part_copy->volume->getDisk()->getType() == DiskType::Type::S3 - && original_active_part->getUniqueId() == part_copy->getUniqueId()) - { - /// May be when several volumes use the same S3 storage - original_active_part->force_keep_shared_data = true; - } + /// May be when several volumes use the same S3/HDFS storage + original_active_part->force_keep_shared_data = true; } modifyPartState(original_active_part, DataPartState::DeleteOnDestroy); @@ -3337,20 +3357,25 @@ MergeTreeData::getAllDataPartsVector(MergeTreeData::DataPartStateVector * out_st return res; } -std::vector -MergeTreeData::getDetachedParts() const +std::vector MergeTreeData::getDetachedParts() const { std::vector res; for (const auto & [path, disk] : getRelativeDataPathsWithDisks()) { - for (auto it = disk->iterateDirectory(fs::path(path) / MergeTreeData::DETACHED_DIR_NAME); it->isValid(); it->next()) - { - res.emplace_back(); - auto & part = res.back(); + String detached_path = fs::path(path) / MergeTreeData::DETACHED_DIR_NAME; - DetachedPartInfo::tryParseDetachedPartName(it->name(), part, format_version); - part.disk = disk->getName(); + /// Note: we don't care about TOCTOU issue here. + if (disk->exists(detached_path)) + { + for (auto it = disk->iterateDirectory(detached_path); it->isValid(); it->next()) + { + res.emplace_back(); + auto & part = res.back(); + + DetachedPartInfo::tryParseDetachedPartName(it->name(), part, format_version); + part.disk = disk->getName(); + } } } return res; @@ -3828,16 +3853,20 @@ bool MergeTreeData::mayBenefitFromIndexForIn( for (const auto & index : metadata_snapshot->getSecondaryIndices()) if (index_wrapper_factory.get(index)->mayBenefitFromIndexForIn(item)) return true; - if (metadata_snapshot->selected_projection - && metadata_snapshot->selected_projection->isPrimaryKeyColumnPossiblyWrappedInFunctions(item)) - return true; + for (const auto & projection : metadata_snapshot->getProjections()) + { + if (projection.isPrimaryKeyColumnPossiblyWrappedInFunctions(item)) + return true; + } } /// The tuple itself may be part of the primary key, so check that as a last resort. if (isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(left_in_operand, metadata_snapshot)) return true; - if (metadata_snapshot->selected_projection - && metadata_snapshot->selected_projection->isPrimaryKeyColumnPossiblyWrappedInFunctions(left_in_operand)) - return true; + for (const auto & projection : metadata_snapshot->getProjections()) + { + if (projection.isPrimaryKeyColumnPossiblyWrappedInFunctions(left_in_operand)) + return true; + } return false; } else @@ -3846,10 +3875,11 @@ bool MergeTreeData::mayBenefitFromIndexForIn( if (index_wrapper_factory.get(index)->mayBenefitFromIndexForIn(left_in_operand)) return true; - if (metadata_snapshot->selected_projection - && metadata_snapshot->selected_projection->isPrimaryKeyColumnPossiblyWrappedInFunctions(left_in_operand)) - return true; - + for (const auto & projection : metadata_snapshot->getProjections()) + { + if (projection.isPrimaryKeyColumnPossiblyWrappedInFunctions(left_in_operand)) + return true; + } return isPrimaryOrMinMaxKeyColumnPossiblyWrappedInFunctions(left_in_operand, metadata_snapshot); } } @@ -3889,7 +3919,7 @@ static void selectBestProjection( candidate.required_columns, metadata_snapshot, candidate.desc->metadata, - query_info, // TODO syntax_analysis_result set in index + query_info, query_context, settings.max_threads, max_added_blocks); @@ -3907,7 +3937,7 @@ static void selectBestProjection( required_columns, metadata_snapshot, metadata_snapshot, - query_info, // TODO syntax_analysis_result set in index + query_info, query_context, settings.max_threads, max_added_blocks); @@ -3926,7 +3956,7 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot, SelectQueryInfo & query_info) const { const auto & settings = query_context->getSettingsRef(); - if (!settings.allow_experimental_projection_optimization || query_info.ignore_projections) + if (!settings.allow_experimental_projection_optimization || query_info.ignore_projections || query_info.is_projection_query) return false; const auto & query_ptr = query_info.query; @@ -4165,7 +4195,7 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( analysis_result.required_columns, metadata_snapshot, metadata_snapshot, - query_info, // TODO syntax_analysis_result set in index + query_info, query_context, settings.max_threads, max_added_blocks); diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index a6ece4a7a98..02d1f5e264e 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -526,9 +526,8 @@ public: void clearOldWriteAheadLogs(); /// Delete all directories which names begin with "tmp" - /// Set non-negative parameter value to override MergeTreeSettings temporary_directories_lifetime - /// Must be called with locked lockForShare() because use relative_data_path. - void clearOldTemporaryDirectories(ssize_t custom_directories_lifetime_seconds = -1); + /// Must be called with locked lockForShare() because it's using relative_data_path. + void clearOldTemporaryDirectories(size_t custom_directories_lifetime_seconds); void clearEmptyParts(); @@ -814,11 +813,11 @@ public: bool scheduleDataMovingJob(IBackgroundJobExecutor & executor); bool areBackgroundMovesNeeded() const; - /// Lock part in zookeeper for use common S3 data in several nodes + /// Lock part in zookeeper for shared data in several nodes /// Overridden in StorageReplicatedMergeTree virtual void lockSharedData(const IMergeTreeDataPart &) const {} - /// Unlock common S3 data part in zookeeper + /// Unlock shared data part in zookeeper /// Overridden in StorageReplicatedMergeTree virtual bool unlockSharedData(const IMergeTreeDataPart &) const { return true; } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index f9ed30ed4ed..a777c244426 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -828,7 +828,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataMergerMutator::mergePartsToTempor UInt64 watch_prev_elapsed = 0; /// We count total amount of bytes in parts - /// and use direct_io + aio if there is more than min_merge_bytes_to_use_direct_io + /// and use direct_io if there is more than min_merge_bytes_to_use_direct_io bool read_with_direct_io = false; if (data_settings->min_merge_bytes_to_use_direct_io != 0) { diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index 2666ba1518f..8fccfbb1f90 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -395,10 +395,10 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const String & name, auto disk = data_part->volume->getDisk(); String escaped_name = escapeForFileName(name); - String mrk_path = fullPath(disk, part_path + escaped_name + marks_file_extension); - String bin_path = fullPath(disk, part_path + escaped_name + DATA_FILE_EXTENSION); - DB::ReadBufferFromFile mrk_in(mrk_path); - DB::CompressedReadBufferFromFile bin_in(bin_path, 0, 0, 0, nullptr); + String mrk_path = part_path + escaped_name + marks_file_extension; + String bin_path = part_path + escaped_name + DATA_FILE_EXTENSION; + auto mrk_in = disk->readFile(mrk_path); + DB::CompressedReadBufferFromFile bin_in(disk->readFile(bin_path)); bool must_be_last = false; UInt64 offset_in_compressed_file = 0; UInt64 offset_in_decompressed_block = 0; @@ -407,15 +407,15 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const String & name, size_t mark_num; const auto & serialization = serializations[name]; - for (mark_num = 0; !mrk_in.eof(); ++mark_num) + for (mark_num = 0; !mrk_in->eof(); ++mark_num) { if (mark_num > index_granularity.getMarksCount()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect number of marks in memory {}, on disk (at least) {}", index_granularity.getMarksCount(), mark_num + 1); - DB::readBinary(offset_in_compressed_file, mrk_in); - DB::readBinary(offset_in_decompressed_block, mrk_in); + DB::readBinary(offset_in_compressed_file, *mrk_in); + DB::readBinary(offset_in_decompressed_block, *mrk_in); if (settings.can_use_adaptive_granularity) - DB::readBinary(index_granularity_rows, mrk_in); + DB::readBinary(index_granularity_rows, *mrk_in); else index_granularity_rows = data_part->index_granularity_info.fixed_index_granularity; @@ -424,7 +424,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const String & name, if (index_granularity_rows != 0) throw Exception(ErrorCodes::LOGICAL_ERROR, "We ran out of binary data but still have non empty mark #{} with rows number {}", mark_num, index_granularity_rows); - if (!mrk_in.eof()) + if (!mrk_in->eof()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mark #{} must be last, but we still have some to read", mark_num); break; @@ -486,7 +486,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const String & name, } } - if (!mrk_in.eof()) + if (!mrk_in->eof()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Still have something in marks stream, last mark #{} index granularity size {}, last rows {}", mark_num, index_granularity.getMarksCount(), index_granularity_rows); if (!bin_in.eof()) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index 0a05eeb966e..916850ab3d0 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -178,7 +178,6 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( Pipe projection_pipe; Pipe ordinary_pipe; - const auto & given_select = query_info.query->as(); if (!projection_parts.empty()) { LOG_DEBUG(log, "projection required columns: {}", fmt::join(query_info.projection->required_columns, ", ")); @@ -226,22 +225,28 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( if (!normal_parts.empty()) { auto storage_from_base_parts_of_projection = StorageFromMergeTreeDataPart::create(std::move(normal_parts)); - auto ast = query_info.projection->desc->query_ast->clone(); - auto & select = ast->as(); - if (given_select.where()) - select.setExpression(ASTSelectQuery::Expression::WHERE, given_select.where()->clone()); - if (given_select.prewhere()) - select.setExpression(ASTSelectQuery::Expression::WHERE, given_select.prewhere()->clone()); - - // After overriding the group by clause, we finish the possible aggregations directly - if (processed_stage >= QueryProcessingStage::Enum::WithMergeableState && given_select.groupBy()) - select.setExpression(ASTSelectQuery::Expression::GROUP_BY, given_select.groupBy()->clone()); auto interpreter = InterpreterSelectQuery( - ast, + query_info.query, context, storage_from_base_parts_of_projection, nullptr, - SelectQueryOptions{processed_stage}.ignoreAggregation().ignoreProjections()); + SelectQueryOptions{processed_stage}.projectionQuery()); + + QueryPlan ordinary_query_plan; + interpreter.buildQueryPlan(ordinary_query_plan); + + const auto & expressions = interpreter.getAnalysisResult(); + if (processed_stage == QueryProcessingStage::Enum::FetchColumns && expressions.before_where) + { + auto where_step = std::make_unique( + ordinary_query_plan.getCurrentDataStream(), + expressions.before_where, + expressions.where_column_name, + expressions.remove_where_filter); + where_step->setStepDescription("WHERE"); + ordinary_query_plan.addStep(std::move(where_step)); + } + ordinary_pipe = QueryPipeline::getPipe(interpreter.execute().pipeline); } @@ -757,7 +762,8 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd Poco::Logger * log, size_t num_streams, ReadFromMergeTree::IndexStats & index_stats, - bool use_skip_indexes) + bool use_skip_indexes, + bool check_limits) { RangesInDataParts parts_with_ranges(parts.size()); const Settings & settings = context->getSettingsRef(); @@ -885,7 +891,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd if (!ranges.ranges.empty()) { - if (limits.max_rows || leaf_limits.max_rows) + if (check_limits && (limits.max_rows || leaf_limits.max_rows)) { /// Fail fast if estimated number of rows to read exceeds the limit auto current_rows_estimate = ranges.getRowsCount(); @@ -1150,7 +1156,8 @@ size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( log, num_streams, index_stats, - false); + true /* use_skip_indexes */, + false /* check_limits */); return index_stats.back().num_granules_after; } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index bd2a79f0aee..de5ca1f0138 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -174,6 +174,7 @@ public: /// Filter parts using primary key and secondary indexes. /// For every part, select mark ranges to read. + /// If 'check_limits = true' it will throw exception if the amount of data exceed the limits from settings. static RangesInDataParts filterPartsByPrimaryKeyAndSkipIndexes( MergeTreeData::DataPartsVector && parts, StorageMetadataPtr metadata_snapshot, @@ -184,7 +185,8 @@ public: Poco::Logger * log, size_t num_streams, ReadFromMergeTree::IndexStats & index_stats, - bool use_skip_indexes); + bool use_skip_indexes, + bool check_limits); /// Create expression for sampling. /// Also, calculate _sample_factor if needed. diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 5b77ac9ec4a..15e7ed4c1d0 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -195,16 +195,14 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt throw Exception("Cancelled moving parts.", ErrorCodes::ABORTED); auto settings = data->getSettings(); - auto part = moving_part.part; - LOG_TRACE(log, "Cloning part {}", part->name); - auto disk = moving_part.reserved_space->getDisk(); + LOG_DEBUG(log, "Cloning part {} from {} to {}", part->name, part->volume->getDisk()->getName(), disk->getName()); + const String directory_to_move = "moving"; - if (settings->allow_s3_zero_copy_replication) + if (disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication) { - /// Try to fetch part from S3 without copy and fallback to default copy - /// if it's not possible + /// Try zero-copy replication and fallback to default copy if it's not possible moving_part.part->assertOnDisk(); String path_to_clone = fs::path(data->getRelativeDataPath()) / directory_to_move / ""; String relative_path = part->relative_path; diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp index e8ea56b6531..f5ae5162676 100644 --- a/src/Storages/MergeTree/MergeTreeReadPool.cpp +++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp @@ -231,6 +231,19 @@ std::vector MergeTreeReadPool::fillPerPartInfo( auto [required_columns, required_pre_columns, should_reorder] = getReadTaskColumns(data, metadata_snapshot, part.data_part, column_names, prewhere_info, check_columns); + if (predict_block_size_bytes) + { + const auto & required_column_names = required_columns.getNames(); + const auto & required_pre_column_names = required_pre_columns.getNames(); + NameSet complete_column_names(required_column_names.begin(), required_column_names.end()); + complete_column_names.insert(required_pre_column_names.begin(), required_pre_column_names.end()); + + per_part_size_predictor.emplace_back(std::make_unique( + part.data_part, Names(complete_column_names.begin(), complete_column_names.end()), sample_block)); + } + else + per_part_size_predictor.emplace_back(nullptr); + /// will be used to distinguish between PREWHERE and WHERE columns when applying filter const auto & required_column_names = required_columns.getNames(); per_part_column_name_set.emplace_back(required_column_names.begin(), required_column_names.end()); @@ -240,14 +253,6 @@ std::vector MergeTreeReadPool::fillPerPartInfo( per_part_should_reorder.push_back(should_reorder); parts_with_idx.push_back({ part.data_part, part.part_index_in_query }); - - if (predict_block_size_bytes) - { - per_part_size_predictor.emplace_back(std::make_unique( - part.data_part, column_names, sample_block)); - } - else - per_part_size_predictor.emplace_back(nullptr); } return per_part_sum_marks; diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 81833b76735..d546b2a95af 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -94,9 +94,17 @@ try MarkRanges mark_ranges_for_task = { all_mark_ranges.back() }; all_mark_ranges.pop_back(); - auto size_predictor = (preferred_block_size_bytes == 0) - ? nullptr - : std::make_unique(data_part, ordered_names, metadata_snapshot->getSampleBlock()); + std::unique_ptr size_predictor; + if (preferred_block_size_bytes) + { + const auto & required_column_names = task_columns.columns.getNames(); + const auto & required_pre_column_names = task_columns.pre_columns.getNames(); + NameSet complete_column_names(required_column_names.begin(), required_column_names.end()); + complete_column_names.insert(required_pre_column_names.begin(), required_pre_column_names.end()); + + size_predictor = std::make_unique( + data_part, Names(complete_column_names.begin(), complete_column_names.end()), metadata_snapshot->getSampleBlock()); + } task = std::make_unique( data_part, mark_ranges_for_task, part_index_in_query, ordered_names, column_name_set, diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index ce342a69fe0..1e4b61e13d9 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -72,9 +72,17 @@ try storage, metadata_snapshot, data_part, required_columns, prewhere_info, check_columns); - auto size_predictor = (preferred_block_size_bytes == 0) - ? nullptr - : std::make_unique(data_part, ordered_names, metadata_snapshot->getSampleBlock()); + std::unique_ptr size_predictor; + if (preferred_block_size_bytes) + { + const auto & required_column_names = task_columns.columns.getNames(); + const auto & required_pre_column_names = task_columns.pre_columns.getNames(); + NameSet complete_column_names(required_column_names.begin(), required_column_names.end()); + complete_column_names.insert(required_pre_column_names.begin(), required_pre_column_names.end()); + + size_predictor = std::make_unique( + data_part, Names(complete_column_names.begin(), complete_column_names.end()), metadata_snapshot->getSampleBlock()); + } /// will be used to distinguish between PREWHERE and WHERE columns when applying filter const auto & column_names = task_columns.columns.getNames(); diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index e82b1966461..2a3c7ed00a1 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -54,7 +54,7 @@ MergeTreeSequentialSource::MergeTreeSequentialSource( MergeTreeReaderSettings reader_settings = { - /// bytes to use AIO (this is hack) + /// bytes to use direct IO (this is hack) .min_bytes_to_use_direct_io = read_with_direct_io ? 1UL : std::numeric_limits::max(), .max_read_buffer_size = DBMS_DEFAULT_BUFFER_SIZE, .save_marks_in_cache = false diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 888ca80e015..d018059c248 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -57,6 +57,7 @@ struct Settings; M(Bool, in_memory_parts_insert_sync, false, "If true insert of part with in-memory format will wait for fsync of WAL", 0) \ M(UInt64, non_replicated_deduplication_window, 0, "How many last blocks of hashes should be kept on disk (0 - disabled).", 0) \ M(UInt64, max_parts_to_merge_at_once, 100, "Max amount of parts which can be merged at once (0 - disabled). Doesn't affect OPTIMIZE FINAL query.", 0) \ + M(UInt64, merge_selecting_sleep_ms, 5000, "Sleep time for merge selecting when no part selected, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters", 0) \ \ /** Inserts settings. */ \ M(UInt64, parts_to_delay_insert, 150, "If table contains at least that many active parts in single partition, artificially slow down insert into table.", 0) \ @@ -74,7 +75,7 @@ struct Settings; M(Seconds, prefer_fetch_merged_part_time_threshold, 3600, "If time passed after replication log entry creation exceeds this threshold and sum size of parts is greater than \"prefer_fetch_merged_part_size_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.", 0) \ M(UInt64, prefer_fetch_merged_part_size_threshold, 10ULL * 1024 * 1024 * 1024, "If sum size of parts exceeds this threshold and time passed after replication log entry creation is greater than \"prefer_fetch_merged_part_time_threshold\", prefer fetching merged part from replica instead of doing merge locally. To speed up very long merges.", 0) \ M(Seconds, execute_merges_on_single_replica_time_threshold, 0, "When greater than zero only a single replica starts the merge immediately, others wait up to that amount of time to download the result instead of doing merges locally. If the chosen replica doesn't finish the merge during that amount of time, fallback to standard behavior happens.", 0) \ - M(Seconds, s3_execute_merges_on_single_replica_time_threshold, 3 * 60 * 60, "When greater than zero only a single replica starts the merge immediatelys when merged part on S3 storage and 'allow_s3_zero_copy_replication' is enabled.", 0) \ + M(Seconds, remote_fs_execute_merges_on_single_replica_time_threshold, 3 * 60 * 60, "When greater than zero only a single replica starts the merge immediatelys when merged part on shared storage and 'allow_remote_fs_zero_copy_replication' is enabled.", 0) \ M(Seconds, try_fetch_recompressed_part_timeout, 7200, "Recompression works slow in most cases, so we don't start merge with recompression until this timeout and trying to fetch recompressed part from replica which assigned this merge with recompression.", 0) \ M(Bool, always_fetch_merged_part, 0, "If true, replica never merge parts and always download merged parts from other replicas.", 0) \ M(UInt64, max_suspicious_broken_parts, 10, "Max broken parts, if more - deny automatic deletion.", 0) \ @@ -123,13 +124,14 @@ struct Settings; M(UInt64, concurrent_part_removal_threshold, 100, "Activate concurrent part removal (see 'max_part_removal_threads') only if the number of inactive data parts is at least this.", 0) \ M(String, storage_policy, "default", "Name of storage disk policy", 0) \ M(Bool, allow_nullable_key, false, "Allow Nullable types as primary keys.", 0) \ - M(Bool, allow_s3_zero_copy_replication, false, "Allow Zero-copy replication over S3", 0) \ + M(Bool, allow_remote_fs_zero_copy_replication, false, "Allow Zero-copy replication over remote fs", 0) \ M(Bool, remove_empty_parts, true, "Remove empty parts after they were pruned by TTL, mutation, or collapsing merge algorithm", 0) \ M(Bool, assign_part_uuids, false, "Generate UUIDs for parts. Before enabling check that all replicas support new format.", 0) \ M(Int64, max_partitions_to_read, -1, "Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited. This setting is the default that can be overridden by the query-level setting with the same name.", 0) \ M(UInt64, max_concurrent_queries, 0, "Max number of concurrently executed queries related to the MergeTree table (0 - disabled). Queries will still be limited by other max_concurrent_queries settings.", 0) \ M(UInt64, min_marks_to_honor_max_concurrent_queries, 0, "Minimal number of marks to honor the MergeTree-level's max_concurrent_queries (0 - disabled). Queries will still be limited by other max_concurrent_queries settings.", 0) \ M(UInt64, min_bytes_to_rebalance_partition_over_jbod, 0, "Minimal amount of bytes to enable part rebalance over JBOD array (0 - disabled).", 0) \ + M(Bool, check_sample_column_is_correct, true, "Check columns or columns by hash for sampling are unsigned integer.", 0) \ \ /** Experimental/work in progress feature. Unsafe for production. */ \ M(UInt64, part_moves_between_shards_enable, 0, "Experimental/Incomplete feature to move parts between shards. Does not take into account sharding expressions.", 0) \ diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index 10e2d77eb27..06856c73888 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -62,7 +62,7 @@ void ReplicatedMergeTreeCleanupThread::iterate() /// Both use relative_data_path which changes during rename, so we /// do it under share lock storage.clearOldWriteAheadLogs(); - storage.clearOldTemporaryDirectories(); + storage.clearOldTemporaryDirectories(storage.getSettings()->temporary_directories_lifetime.totalSeconds()); } /// This is loose condition: no problem if we actually had lost leadership at this moment diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp index 65da6080e86..13e05681fd9 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp @@ -56,9 +56,9 @@ bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplica(const Re } -bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplicaS3Shared(const ReplicatedMergeTreeLogEntryData & entry) const +bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplicaShared(const ReplicatedMergeTreeLogEntryData & entry) const { - time_t threshold = s3_execute_merges_on_single_replica_time_threshold; + time_t threshold = remote_fs_execute_merges_on_single_replica_time_threshold; return ( threshold > 0 /// feature turned on && entry.type == ReplicatedMergeTreeLogEntry::MERGE_PARTS /// it is a merge log entry @@ -100,24 +100,25 @@ std::optional ReplicatedMergeTreeMergeStrategyPicker::pickReplicaToExecu void ReplicatedMergeTreeMergeStrategyPicker::refreshState() { - auto threshold = storage.getSettings()->execute_merges_on_single_replica_time_threshold.totalSeconds(); - auto threshold_s3 = 0; - if (storage.getSettings()->allow_s3_zero_copy_replication) - threshold_s3 = storage.getSettings()->s3_execute_merges_on_single_replica_time_threshold.totalSeconds(); + const auto settings = storage.getSettings(); + auto threshold = settings->execute_merges_on_single_replica_time_threshold.totalSeconds(); + auto threshold_init = 0; + if (settings->allow_remote_fs_zero_copy_replication) + threshold_init = settings->remote_fs_execute_merges_on_single_replica_time_threshold.totalSeconds(); if (threshold == 0) /// we can reset the settings w/o lock (it's atomic) execute_merges_on_single_replica_time_threshold = threshold; - if (threshold_s3 == 0) - s3_execute_merges_on_single_replica_time_threshold = threshold_s3; - if (threshold == 0 && threshold_s3 == 0) + if (threshold_init == 0) + remote_fs_execute_merges_on_single_replica_time_threshold = threshold_init; + if (threshold == 0 && threshold_init == 0) return; auto now = time(nullptr); /// the setting was already enabled, and last state refresh was done recently if (((threshold != 0 && execute_merges_on_single_replica_time_threshold != 0) - || (threshold_s3 != 0 && s3_execute_merges_on_single_replica_time_threshold != 0)) + || (threshold_init != 0 && remote_fs_execute_merges_on_single_replica_time_threshold != 0)) && now - last_refresh_time < REFRESH_STATE_MINIMUM_INTERVAL_SECONDS) return; @@ -146,15 +147,15 @@ void ReplicatedMergeTreeMergeStrategyPicker::refreshState() LOG_WARNING(storage.log, "Can't find current replica in the active replicas list, or too few active replicas to use execute_merges_on_single_replica_time_threshold!"); /// we can reset the settings w/o lock (it's atomic) execute_merges_on_single_replica_time_threshold = 0; - s3_execute_merges_on_single_replica_time_threshold = 0; + remote_fs_execute_merges_on_single_replica_time_threshold = 0; return; } std::lock_guard lock(mutex); if (threshold != 0) /// Zeros already reset execute_merges_on_single_replica_time_threshold = threshold; - if (threshold_s3 != 0) - s3_execute_merges_on_single_replica_time_threshold = threshold_s3; + if (threshold_init != 0) + remote_fs_execute_merges_on_single_replica_time_threshold = threshold_init; last_refresh_time = now; current_replica_index = current_replica_index_tmp; active_replicas = active_replicas_tmp; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h index 8adf206676a..f6a19173f77 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h @@ -52,9 +52,9 @@ public: /// and we may need to do a fetch (or postpone) instead of merge bool shouldMergeOnSingleReplica(const ReplicatedMergeTreeLogEntryData & entry) const; - /// return true if s3_execute_merges_on_single_replica_time_threshold feature is active + /// return true if remote_fs_execute_merges_on_single_replica_time_threshold feature is active /// and we may need to do a fetch (or postpone) instead of merge - bool shouldMergeOnSingleReplicaS3Shared(const ReplicatedMergeTreeLogEntryData & entry) const; + bool shouldMergeOnSingleReplicaShared(const ReplicatedMergeTreeLogEntryData & entry) const; /// returns the replica name /// and it's not current replica should do the merge @@ -72,7 +72,7 @@ private: uint64_t getEntryHash(const ReplicatedMergeTreeLogEntryData & entry) const; std::atomic execute_merges_on_single_replica_time_threshold = 0; - std::atomic s3_execute_merges_on_single_replica_time_threshold = 0; + std::atomic remote_fs_execute_merges_on_single_replica_time_threshold = 0; std::atomic last_refresh_time = 0; std::mutex mutex; diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 9cc2787697d..15beb94404b 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -47,6 +47,7 @@ public: QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)); } + bool supportsPrewhere() const override { return true; } bool supportsIndexForIn() const override { return true; } diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index 70251a940cc..2fa96ac5cf2 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -327,6 +327,16 @@ ASTPtr StorageMaterializedPostgreSQL::getColumnDeclaration(const DataTypePtr & d return make_decimal_expression("Decimal256"); } + if (which.isDateTime64()) + { + auto ast_expression = std::make_shared(); + + ast_expression->name = "DateTime64"; + ast_expression->arguments = std::make_shared(); + ast_expression->arguments->children.emplace_back(std::make_shared(UInt32(6))); + return ast_expression; + } + return std::make_shared(data_type->getName()); } diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index fc308667db9..cf2c4d72f59 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -156,6 +156,7 @@ struct SelectQueryInfo /// If not null, it means we choose a projection to execute current query. std::optional projection; bool ignore_projections = false; + bool is_projection_query = false; }; } diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index f4d6ec5c6f7..21fa06e19f0 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -1093,7 +1093,7 @@ ClusterPtr StorageDistributed::skipUnusedShards( size_t limit = local_context->getSettingsRef().optimize_skip_unused_shards_limit; if (!limit || limit > SSIZE_MAX) { - throw Exception("optimize_skip_unused_shards_limit out of range (0, {}]", ErrorCodes::ARGUMENT_OUT_OF_BOUND, SSIZE_MAX); + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "optimize_skip_unused_shards_limit out of range (0, {}]", SSIZE_MAX); } // To interpret limit==0 as limit is reached ++limit; diff --git a/src/Storages/StorageInMemoryMetadata.h b/src/Storages/StorageInMemoryMetadata.h index 861cb5866ee..d0d60f608d7 100644 --- a/src/Storages/StorageInMemoryMetadata.h +++ b/src/Storages/StorageInMemoryMetadata.h @@ -28,7 +28,6 @@ struct StorageInMemoryMetadata ConstraintsDescription constraints; /// Table projections. Currently supported for MergeTree only. ProjectionsDescription projections; - mutable const ProjectionDescription * selected_projection{}; /// PARTITION BY expression. Currently supported for MergeTree only. KeyDescription partition_key; /// PRIMARY KEY expression. If absent, than equal to order_by_ast. diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 61fbbbc3086..b43cb6d71a0 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -28,6 +28,7 @@ #include #include +#include #define DBMS_STORAGE_LOG_DATA_FILE_EXTENSION ".bin" @@ -719,6 +720,34 @@ CheckResults StorageLog::checkData(const ASTPtr & /* query */, ContextPtr contex } +IStorage::ColumnSizeByName StorageLog::getColumnSizes() const +{ + std::shared_lock lock(rwlock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + ColumnSizeByName column_sizes; + FileChecker::Map file_sizes = file_checker.getFileSizes(); + + for (const auto & column : getInMemoryMetadata().getColumns().getAllPhysical()) + { + ISerialization::StreamCallback stream_callback = [&, this] (const ISerialization::SubstreamPath & substream_path) + { + String stream_name = ISerialization::getFileNameForStream(column, substream_path); + ColumnSize & size = column_sizes[column.name]; + auto it = files.find(stream_name); + if (it != files.end()) + size.data_compressed += file_sizes[fileName(it->second.data_file_path)]; + }; + + ISerialization::SubstreamPath substream_path; + auto serialization = column.type->getDefaultSerialization(); + serialization->enumerateStreams(stream_callback, substream_path); + } + + return column_sizes; +} + void registerStorageLog(StorageFactory & factory) { StorageFactory::StorageFeatures features{ diff --git a/src/Storages/StorageLog.h b/src/Storages/StorageLog.h index 6fea00edefd..799bad26c7c 100644 --- a/src/Storages/StorageLog.h +++ b/src/Storages/StorageLog.h @@ -45,6 +45,7 @@ public: bool storesDataOnDisk() const override { return true; } Strings getDataPaths() const override { return {DB::fullPath(disk, table_path)}; } bool supportsSubcolumns() const override { return true; } + ColumnSizeByName getColumnSizes() const override; protected: /** Attach the table with the appropriate name, along the appropriate path (with / at the end), @@ -87,7 +88,7 @@ private: DiskPtr disk; String table_path; - std::shared_timed_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Files files; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 55ccd60ea38..05d18e65068 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1068,7 +1068,7 @@ bool StorageMergeTree::scheduleDataProcessingJob(IBackgroundJobExecutor & execut /// All use relative_data_path which changes during rename /// so execute under share lock. clearOldPartsFromFilesystem(); - clearOldTemporaryDirectories(); + clearOldTemporaryDirectories(getSettings()->temporary_directories_lifetime.totalSeconds()); clearOldWriteAheadLogs(); clearOldMutations(); clearEmptyParts(); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 211a626e8d4..1f28f4f19f9 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -234,6 +234,10 @@ public: else if (which.isFloat64()) nested_column = ColumnFloat64::create(); else if (which.isDate()) nested_column = ColumnUInt16::create(); else if (which.isDateTime()) nested_column = ColumnUInt32::create(); + else if (which.isDateTime64()) + { + nested_column = ColumnDecimal::create(0, 6); + } else if (which.isDecimal32()) { const auto & type = typeid_cast *>(nested.get()); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index beb987ae605..d44b86fe9bb 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -155,7 +155,6 @@ namespace ActionLocks static const auto QUEUE_UPDATE_ERROR_SLEEP_MS = 1 * 1000; -static const auto MERGE_SELECTING_SLEEP_MS = 5 * 1000; static const auto MUTATIONS_FINALIZING_SLEEP_MS = 1 * 1000; static const auto MUTATIONS_FINALIZING_IDLE_SLEEP_MS = 5 * 1000; @@ -607,11 +606,14 @@ void StorageReplicatedMergeTree::createNewZooKeeperNodes() zookeeper->createIfNotExists(zookeeper_path + "/mutations", String()); zookeeper->createIfNotExists(replica_path + "/mutation_pointer", String()); - /// Nodes for zero-copy S3 replication - if (storage_settings.get()->allow_s3_zero_copy_replication) + /// Nodes for remote fs zero-copy replication + const auto settings = getSettings(); + if (settings->allow_remote_fs_zero_copy_replication) { zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3", String()); zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_s3/shared", String()); + zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_hdfs", String()); + zookeeper->createIfNotExists(zookeeper_path + "/zero_copy_hdfs/shared", String()); } /// Part movement. @@ -1728,23 +1730,19 @@ bool StorageReplicatedMergeTree::tryExecuteMerge(const LogEntry & entry) future_merged_part.updatePath(*this, reserved_space); future_merged_part.merge_type = entry.merge_type; - if (storage_settings_ptr->allow_s3_zero_copy_replication) + if (reserved_space->getDisk()->supportZeroCopyReplication() + && storage_settings_ptr->allow_remote_fs_zero_copy_replication + && merge_strategy_picker.shouldMergeOnSingleReplicaShared(entry)) { - if (auto disk = reserved_space->getDisk(); disk->getType() == DB::DiskType::Type::S3) - { - if (merge_strategy_picker.shouldMergeOnSingleReplicaS3Shared(entry)) - { - if (!replica_to_execute_merge_picked) - replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); + if (!replica_to_execute_merge_picked) + replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); - if (replica_to_execute_merge) - { - LOG_DEBUG(log, - "Prefer fetching part {} from replica {} due s3_execute_merges_on_single_replica_time_threshold", - entry.new_part_name, replica_to_execute_merge.value()); - return false; - } - } + if (replica_to_execute_merge) + { + LOG_DEBUG(log, + "Prefer fetching part {} from replica {} due to remote_fs_execute_merges_on_single_replica_time_threshold", + entry.new_part_name, replica_to_execute_merge.value()); + return false; } } @@ -2168,7 +2166,7 @@ bool StorageReplicatedMergeTree::executeFetchShared( { if (source_replica.empty()) { - LOG_INFO(log, "No active replica has part {} on S3.", new_part_name); + LOG_INFO(log, "No active replica has part {} on shared storage.", new_part_name); return false; } @@ -2783,6 +2781,16 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo } } + { + /// Check "is_lost" version after retrieving queue and parts. + /// If version has changed, then replica most likely has been dropped and parts set is inconsistent, + /// so throw exception and retry cloning. + Coordination::Stat is_lost_stat_new; + zookeeper->get(fs::path(source_path) / "is_lost", &is_lost_stat_new); + if (is_lost_stat_new.version != source_is_lost_stat.version) + throw Exception(ErrorCodes::REPLICA_STATUS_CHANGED, "Cannot clone {}, because it suddenly become lost", source_replica); + } + tryRemovePartsFromZooKeeperWithRetries(parts_to_remove_from_zk); auto local_active_parts = getDataParts(); @@ -3347,7 +3355,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() if (create_result != CreateMergeEntryResult::Ok && create_result != CreateMergeEntryResult::LogUpdated) { - merge_selecting_task->scheduleAfter(MERGE_SELECTING_SLEEP_MS); + merge_selecting_task->scheduleAfter(storage_settings_ptr->merge_selecting_sleep_ms); } else { @@ -7201,10 +7209,9 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) if (!part.volume) return; DiskPtr disk = part.volume->getDisk(); - if (!disk) - return; - if (disk->getType() != DB::DiskType::Type::S3) + if (!disk || !disk->supportZeroCopyReplication()) return; + String zero_copy = fmt::format("zero_copy_{}", DiskType::toString(disk->getType())); zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); if (!zookeeper) @@ -7213,7 +7220,7 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name / id / replica_name; + String zookeeper_node = fs::path(zookeeper_path) / zero_copy / "shared" / part.name / id / replica_name; LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); @@ -7242,10 +7249,9 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par if (!part.volume) return true; DiskPtr disk = part.volume->getDisk(); - if (!disk) - return true; - if (disk->getType() != DB::DiskType::Type::S3) + if (!disk || !disk->supportZeroCopyReplication()) return true; + String zero_copy = fmt::format("zero_copy_{}", DiskType::toString(disk->getType())); zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); if (!zookeeper) @@ -7254,7 +7260,7 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par String id = part.getUniqueId(); boost::replace_all(id, "/", "_"); - String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; + String zookeeper_part_node = fs::path(zookeeper_path) / zero_copy / "shared" / part.name; String zookeeper_part_uniq_node = fs::path(zookeeper_part_node) / id; String zookeeper_node = fs::path(zookeeper_part_uniq_node) / replica_name; @@ -7289,16 +7295,14 @@ bool StorageReplicatedMergeTree::tryToFetchIfShared( const DiskPtr & disk, const String & path) { - const auto data_settings = getSettings(); - if (!data_settings->allow_s3_zero_copy_replication) + const auto settings = getSettings(); + auto disk_type = disk->getType(); + if (!(disk->supportZeroCopyReplication() && settings->allow_remote_fs_zero_copy_replication)) return false; - if (disk->getType() != DB::DiskType::Type::S3) - return false; + String replica = getSharedDataReplica(part, disk_type); - String replica = getSharedDataReplica(part); - - /// We can't fetch part when none replicas have this part on S3 + /// We can't fetch part when none replicas have this part on a same type remote disk if (replica.empty()) return false; @@ -7307,7 +7311,7 @@ bool StorageReplicatedMergeTree::tryToFetchIfShared( String StorageReplicatedMergeTree::getSharedDataReplica( - const IMergeTreeDataPart & part) const + const IMergeTreeDataPart & part, DiskType::Type disk_type) const { String best_replica; @@ -7315,7 +7319,8 @@ String StorageReplicatedMergeTree::getSharedDataReplica( if (!zookeeper) return best_replica; - String zookeeper_part_node = fs::path(zookeeper_path) / "zero_copy_s3" / "shared" / part.name; + String zero_copy = fmt::format("zero_copy_{}", DiskType::toString(disk_type)); + String zookeeper_part_node = fs::path(zookeeper_path) / zero_copy / "shared" / part.name; Strings ids; zookeeper->tryGetChildren(zookeeper_part_node, ids); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index 28dd3c760a8..800f419cb76 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -225,10 +225,10 @@ public: /// Fetch part only when it stored on shared storage like S3 bool executeFetchShared(const String & source_replica, const String & new_part_name, const DiskPtr & disk, const String & path); - /// Lock part in zookeeper for use common S3 data in several nodes + /// Lock part in zookeeper for use shared data in several nodes void lockSharedData(const IMergeTreeDataPart & part) const override; - /// Unlock common S3 data part in zookeeper + /// Unlock shared data part in zookeeper /// Return true if data unlocked /// Return false if data is still used by another node bool unlockSharedData(const IMergeTreeDataPart & part) const override; @@ -236,8 +236,8 @@ public: /// Fetch part only if some replica has it on shared storage like S3 bool tryToFetchIfShared(const IMergeTreeDataPart & part, const DiskPtr & disk, const String & path) override; - /// Get best replica having this partition on S3 - String getSharedDataReplica(const IMergeTreeDataPart & part) const; + /// Get best replica having this partition on a same type remote disk + String getSharedDataReplica(const IMergeTreeDataPart & part, DiskType::Type disk_type) const; inline String getReplicaName() const { return replica_name; } diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp new file mode 100644 index 00000000000..c900c7b7e09 --- /dev/null +++ b/src/Storages/StorageSQLite.cpp @@ -0,0 +1,174 @@ +#include "StorageSQLite.h" + +#if USE_SQLITE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int SQLITE_ENGINE_ERROR; +} + +StorageSQLite::StorageSQLite( + const StorageID & table_id_, + SQLitePtr sqlite_db_, + const String & remote_table_name_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + ContextPtr context_) + : IStorage(table_id_) + , WithContext(context_->getGlobalContext()) + , remote_table_name(remote_table_name_) + , global_context(context_) + , sqlite_db(sqlite_db_) +{ + StorageInMemoryMetadata storage_metadata; + storage_metadata.setColumns(columns_); + storage_metadata.setConstraints(constraints_); + setInMemoryMetadata(storage_metadata); +} + + +Pipe StorageSQLite::read( + const Names & column_names, + const StorageMetadataPtr & metadata_snapshot, + SelectQueryInfo & query_info, + ContextPtr context_, + QueryProcessingStage::Enum, + size_t max_block_size, + unsigned int) +{ + metadata_snapshot->check(column_names, getVirtuals(), getStorageID()); + + String query = transformQueryForExternalDatabase( + query_info, + metadata_snapshot->getColumns().getOrdinary(), + IdentifierQuotingStyle::DoubleQuotes, + "", + remote_table_name, + context_); + + Block sample_block; + for (const String & column_name : column_names) + { + auto column_data = metadata_snapshot->getColumns().getPhysical(column_name); + sample_block.insert({column_data.type, column_data.name}); + } + + return Pipe(std::make_shared( + std::make_shared(sqlite_db, query, sample_block, max_block_size))); +} + + +class SQLiteBlockOutputStream : public IBlockOutputStream +{ +public: + explicit SQLiteBlockOutputStream( + const StorageSQLite & storage_, + const StorageMetadataPtr & metadata_snapshot_, + StorageSQLite::SQLitePtr sqlite_db_, + const String & remote_table_name_) + : storage{storage_} + , metadata_snapshot(metadata_snapshot_) + , sqlite_db(sqlite_db_) + , remote_table_name(remote_table_name_) + { + } + + Block getHeader() const override { return metadata_snapshot->getSampleBlock(); } + + void write(const Block & block) override + { + WriteBufferFromOwnString sqlbuf; + + sqlbuf << "INSERT INTO "; + sqlbuf << doubleQuoteString(remote_table_name); + sqlbuf << " ("; + + for (auto it = block.begin(); it != block.end(); ++it) + { + if (it != block.begin()) + sqlbuf << ", "; + sqlbuf << quoteString(it->name); + } + + sqlbuf << ") VALUES "; + + auto writer = FormatFactory::instance().getOutputStream("Values", sqlbuf, metadata_snapshot->getSampleBlock(), storage.getContext()); + writer->write(block); + + sqlbuf << ";"; + + char * err_message = nullptr; + int status = sqlite3_exec(sqlite_db.get(), sqlbuf.str().c_str(), nullptr, nullptr, &err_message); + + if (status != SQLITE_OK) + { + String err_msg(err_message); + sqlite3_free(err_message); + throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR, + "Failed to execute sqlite INSERT query. Status: {}. Message: {}", + status, err_msg); + } + } + +private: + const StorageSQLite & storage; + StorageMetadataPtr metadata_snapshot; + StorageSQLite::SQLitePtr sqlite_db; + String remote_table_name; +}; + + +BlockOutputStreamPtr StorageSQLite::write(const ASTPtr & /* query */, const StorageMetadataPtr & metadata_snapshot, ContextPtr) +{ + return std::make_shared(*this, metadata_snapshot, sqlite_db, remote_table_name); +} + + +void registerStorageSQLite(StorageFactory & factory) +{ + factory.registerStorage("SQLite", [](const StorageFactory::Arguments & args) -> StoragePtr + { + ASTs & engine_args = args.engine_args; + + if (engine_args.size() != 2) + throw Exception("SQLite database requires 2 arguments: database path, table name", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + for (auto & engine_arg : engine_args) + engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, args.getLocalContext()); + + const auto database_path = engine_args[0]->as().value.safeGet(); + const auto table_name = engine_args[1]->as().value.safeGet(); + + auto sqlite_db = openSQLiteDB(database_path, args.getContext()); + + return StorageSQLite::create(args.table_id, sqlite_db, + table_name, args.columns, args.constraints, args.getContext()); + }, + { + .source_access_type = AccessType::SQLITE, + }); +} + +} + +#endif diff --git a/src/Storages/StorageSQLite.h b/src/Storages/StorageSQLite.h new file mode 100644 index 00000000000..63b7a6fd415 --- /dev/null +++ b/src/Storages/StorageSQLite.h @@ -0,0 +1,53 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_SQLITE +#include +#include + +#include // Y_IGNORE + + +namespace DB +{ + +class StorageSQLite final : public shared_ptr_helper, public IStorage, public WithContext +{ +friend struct shared_ptr_helper; + +public: + using SQLitePtr = std::shared_ptr; + + StorageSQLite( + const StorageID & table_id_, + SQLitePtr sqlite_db_, + const String & remote_table_name_, + const ColumnsDescription & columns_, + const ConstraintsDescription & constraints_, + ContextPtr context_); + + std::string getName() const override { return "SQLite"; } + + Pipe read( + const Names & column_names, + const StorageMetadataPtr & /*metadata_snapshot*/, + SelectQueryInfo & query_info, + ContextPtr context, + QueryProcessingStage::Enum processed_stage, + size_t max_block_size, + unsigned num_streams) override; + + BlockOutputStreamPtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; + +private: + String remote_table_name; + ContextPtr global_context; + SQLitePtr sqlite_db; +}; + +} + +#endif diff --git a/src/Storages/StorageTinyLog.cpp b/src/Storages/StorageTinyLog.cpp index 689b1307f4d..342101d91cc 100644 --- a/src/Storages/StorageTinyLog.cpp +++ b/src/Storages/StorageTinyLog.cpp @@ -4,6 +4,7 @@ #include #include +#include #include @@ -523,6 +524,34 @@ CheckResults StorageTinyLog::checkData(const ASTPtr & /* query */, ContextPtr co return file_checker.check(); } +IStorage::ColumnSizeByName StorageTinyLog::getColumnSizes() const +{ + std::shared_lock lock(rwlock, std::chrono::seconds(DBMS_DEFAULT_LOCK_ACQUIRE_TIMEOUT_SEC)); + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + + ColumnSizeByName column_sizes; + FileChecker::Map file_sizes = file_checker.getFileSizes(); + + for (const auto & column : getInMemoryMetadata().getColumns().getAllPhysical()) + { + ISerialization::StreamCallback stream_callback = [&, this] (const ISerialization::SubstreamPath & substream_path) + { + String stream_name = ISerialization::getFileNameForStream(column, substream_path); + ColumnSize & size = column_sizes[column.name]; + auto it = files.find(stream_name); + if (it != files.end()) + size.data_compressed += file_sizes[fileName(it->second.data_file_path)]; + }; + + ISerialization::SubstreamPath substream_path; + auto serialization = column.type->getDefaultSerialization(); + serialization->enumerateStreams(stream_callback, substream_path); + } + + return column_sizes; +} + void StorageTinyLog::truncate( const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr, TableExclusiveLockHolder &) { diff --git a/src/Storages/StorageTinyLog.h b/src/Storages/StorageTinyLog.h index 71763a6403e..849b0731a47 100644 --- a/src/Storages/StorageTinyLog.h +++ b/src/Storages/StorageTinyLog.h @@ -45,6 +45,7 @@ public: void truncate(const ASTPtr &, const StorageMetadataPtr & metadata_snapshot, ContextPtr, TableExclusiveLockHolder &) override; + ColumnSizeByName getColumnSizes() const override; protected: StorageTinyLog( DiskPtr disk_, @@ -71,7 +72,7 @@ private: Files files; FileChecker file_checker; - std::shared_timed_mutex rwlock; + mutable std::shared_timed_mutex rwlock; Poco::Logger * log; diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 8f65147bb11..0058b58f537 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -98,7 +98,7 @@ protected: Names cols_required_for_sorting_key; Names cols_required_for_primary_key; Names cols_required_for_sampling; - MergeTreeData::ColumnSizeByName column_sizes; + IStorage::ColumnSizeByName column_sizes; { StoragePtr storage = storages.at(std::make_pair(database_name, table_name)); diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index f45acb0efd9..bed8eadc19c 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -95,6 +95,7 @@ const char * auto_contributors[] { "Anatoly Pugachev", "ana-uvarova", "AnaUvarova", + "Andreas Hunkeler", "AndreevDm", "Andrei Bodrov", "Andrei Chulkov", @@ -280,6 +281,7 @@ const char * auto_contributors[] { "Dongdong Yang", "DoomzD", "Dr. Strange Looker", + "d.v.semenov", "eaxdev", "eejoin", "egatov", @@ -290,6 +292,7 @@ const char * auto_contributors[] { "Eldar Zaitov", "Elena Baskakova", "elenaspb2019", + "elevankoff", "Elghazal Ahmed", "Elizaveta Mironyuk", "emakarov", @@ -434,6 +437,7 @@ const char * auto_contributors[] { "Ivan Starkov", "ivanzhukov", "Ivan Zhukov", + "Jack Song", "JackyWoo", "Jacob Hayes", "jakalletti", @@ -476,6 +480,7 @@ const char * auto_contributors[] { "Konstantin Lebedev", "Konstantin Malanchev", "Konstantin Podshumok", + "Konstantin Rudenskii", "Korenevskiy Denis", "Korviakov Andrey", "koshachy", @@ -488,6 +493,7 @@ const char * auto_contributors[] { "kshvakov", "kssenii", "l", + "l1tsolaiki", "lalex", "Latysheva Alexandra", "lehasm", @@ -515,6 +521,7 @@ const char * auto_contributors[] { "long2ice", "Lopatin Konstantin", "Loud_Scream", + "ltybc-coder", "luc1ph3r", "Lucid Dreams", "Luis Bosque", @@ -633,6 +640,7 @@ const char * auto_contributors[] { "nicelulu", "Nickita", "Nickolay Yastrebov", + "nickzhwang", "Nicolae Vartolomei", "Nico Mandery", "Nico Piderman", @@ -871,6 +879,7 @@ const char * auto_contributors[] { "Veselkov Konstantin", "vic", "vicdashkov", + "Victor", "Victor Tarnavsky", "Viktor Taranenko", "vinity", @@ -947,6 +956,7 @@ const char * auto_contributors[] { "Yuriy Korzhenevskiy", "Yury Karpovich", "Yury Stankevich", + "ywill3", "zamulla", "zhang2014", "zhangshengyu", @@ -957,11 +967,13 @@ const char * auto_contributors[] { "Zhichun Wu", "Zhipeng", "zhukai", + "Zijie Lu", "zlx19950903", "Zoran Pandovski", "zvonand", "zvrr", "zvvr", + "zxc111", "zzsmdfj", "Артем Стрельцов", "Владислав Тихонов", @@ -980,6 +992,7 @@ const char * auto_contributors[] { "张风啸", "徐炘", "曲正鹏", + "未来星___费", "极客青年", "谢磊", "贾顺名(Jarvis)", diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp index 8b119492340..7a8ee75803f 100644 --- a/src/Storages/System/StorageSystemStackTrace.cpp +++ b/src/Storages/System/StorageSystemStackTrace.cpp @@ -223,7 +223,7 @@ void StorageSystemStackTrace::fillData(MutableColumns & res_columns, ContextPtr, { constexpr size_t comm_buf_size = 32; /// More than enough for thread name ReadBufferFromFile comm(thread_name_path.string(), comm_buf_size); - readStringUntilEOF(thread_name, comm); + readEscapedStringUntilEOL(thread_name, comm); comm.close(); } diff --git a/src/Storages/System/StorageSystemWarnings.cpp b/src/Storages/System/StorageSystemWarnings.cpp new file mode 100644 index 00000000000..76b35e9b555 --- /dev/null +++ b/src/Storages/System/StorageSystemWarnings.cpp @@ -0,0 +1,21 @@ +#include +#include + + +namespace DB +{ + +NamesAndTypesList StorageSystemWarnings::getNamesAndTypes() +{ + return { + {"message", std::make_shared()}, + }; +} + +void StorageSystemWarnings::fillData(MutableColumns & res_columns, ContextPtr context, const SelectQueryInfo &) const +{ + for (const auto & warning : context->getWarnings()) + res_columns[0]->insert(warning); +} + +} diff --git a/src/Storages/System/StorageSystemWarnings.h b/src/Storages/System/StorageSystemWarnings.h new file mode 100644 index 00000000000..087c4016aff --- /dev/null +++ b/src/Storages/System/StorageSystemWarnings.h @@ -0,0 +1,27 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + +/** Implements system.warnings table that contains warnings about server configuration + * to be displayed in clickhouse-client. + */ +class StorageSystemWarnings final : public shared_ptr_helper, + public IStorageSystemOneBlock { +public: + std::string getName() const override { return "SystemWarnings"; } + + static NamesAndTypesList getNamesAndTypes(); + +protected: + friend struct shared_ptr_helper; + using IStorageSystemOneBlock::IStorageSystemOneBlock; + + void fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const override; +}; +} diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp index 7da65b09d6d..b3cc254a392 100644 --- a/src/Storages/System/attachSystemTables.cpp +++ b/src/Storages/System/attachSystemTables.cpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #if !defined(ARCADIA_BUILD) @@ -116,6 +117,7 @@ void attachSystemTablesLocal(IDatabase & system_database) attach(system_database, "user_directories"); attach(system_database, "privileges"); attach(system_database, "errors"); + attach(system_database, "warnings"); attach(system_database, "data_skipping_indices"); #if !defined(ARCADIA_BUILD) attach(system_database, "licenses"); diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index 0b302ee437a..333f35774e2 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -67,6 +67,11 @@ void registerStorageMaterializedPostgreSQL(StorageFactory & factory); void registerStorageExternalDistributed(StorageFactory & factory); #endif +#if USE_SQLITE +void registerStorageSQLite(StorageFactory & factory); +#endif + + void registerStorages() { auto & factory = StorageFactory::instance(); @@ -128,6 +133,10 @@ void registerStorages() #if USE_MYSQL || USE_LIBPQXX registerStorageExternalDistributed(factory); #endif + + #if USE_SQLITE + registerStorageSQLite(factory); + #endif } } diff --git a/src/Storages/ya.make b/src/Storages/ya.make index 495ec9c4fd6..d878fea7d34 100644 --- a/src/Storages/ya.make +++ b/src/Storages/ya.make @@ -142,6 +142,7 @@ SRCS( StorageMySQL.cpp StorageNull.cpp StorageReplicatedMergeTree.cpp + StorageSQLite.cpp StorageSet.cpp StorageStripeLog.cpp StorageTinyLog.cpp @@ -209,6 +210,7 @@ SRCS( System/StorageSystemTables.cpp System/StorageSystemUserDirectories.cpp System/StorageSystemUsers.cpp + System/StorageSystemWarnings.cpp System/StorageSystemZeros.cpp System/StorageSystemZooKeeper.cpp System/attachSystemTables.cpp diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 4d3524c7563..40bfa2cbb6b 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -153,11 +153,6 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr if (arg_num < args.size()) throw Exception(help_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - /// ExpressionAnalyzer will be created in InterpreterSelectQuery that will meet these `Identifier` when processing the request. - /// We need to mark them as the name of the database or table, because the default value is column. - for (auto ast : args) - setIdentifierSpecial(ast); - if (!cluster_name.empty()) { /// Use an existing cluster from the main config diff --git a/src/TableFunctions/TableFunctionSQLite.cpp b/src/TableFunctions/TableFunctionSQLite.cpp new file mode 100644 index 00000000000..e9edcb3d1d4 --- /dev/null +++ b/src/TableFunctions/TableFunctionSQLite.cpp @@ -0,0 +1,89 @@ +#include + +#if USE_SQLITE + +#include +#include + +#include +#include +#include "registerTableFunctions.h" + +#include +#include + +#include +#include + +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; + extern const int SQLITE_ENGINE_ERROR; +} + + +StoragePtr TableFunctionSQLite::executeImpl(const ASTPtr & /*ast_function*/, + ContextPtr context, const String & table_name, ColumnsDescription /*cached_columns*/) const +{ + auto columns = getActualTableStructure(context); + + auto storage = StorageSQLite::create(StorageID(getDatabaseName(), table_name), + sqlite_db, + remote_table_name, + columns, ConstraintsDescription{}, context); + + storage->startup(); + return storage; +} + + +ColumnsDescription TableFunctionSQLite::getActualTableStructure(ContextPtr /* context */) const +{ + auto columns = fetchSQLiteTableStructure(sqlite_db.get(), remote_table_name); + + if (!columns) + throw Exception(ErrorCodes::SQLITE_ENGINE_ERROR, "Failed to fetch table structure for {}", remote_table_name); + + return ColumnsDescription{*columns}; +} + + +void TableFunctionSQLite::parseArguments(const ASTPtr & ast_function, ContextPtr context) +{ + const auto & func_args = ast_function->as(); + + if (!func_args.arguments) + throw Exception("Table function 'sqlite' must have arguments.", ErrorCodes::BAD_ARGUMENTS); + + ASTs & args = func_args.arguments->children; + + if (args.size() != 2) + throw Exception("SQLite database requires 2 arguments: database path, table name", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + + database_path = args[0]->as().value.safeGet(); + remote_table_name = args[1]->as().value.safeGet(); + + sqlite_db = openSQLiteDB(database_path, context); +} + + +void registerTableFunctionSQLite(TableFunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/TableFunctions/TableFunctionSQLite.h b/src/TableFunctions/TableFunctionSQLite.h new file mode 100644 index 00000000000..4649dac016f --- /dev/null +++ b/src/TableFunctions/TableFunctionSQLite.h @@ -0,0 +1,36 @@ +#pragma once +#if !defined(ARCADIA_BUILD) +#include "config_core.h" +#endif + +#if USE_SQLITE +#include +#include + + +namespace DB +{ + +class TableFunctionSQLite : public ITableFunction +{ +public: + static constexpr auto name = "sqlite"; + std::string getName() const override { return name; } + +private: + StoragePtr executeImpl( + const ASTPtr & ast_function, ContextPtr context, + const std::string & table_name, ColumnsDescription cached_columns) const override; + + const char * getStorageTypeName() const override { return "SQLite"; } + + ColumnsDescription getActualTableStructure(ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + + String database_path, remote_table_name; + std::shared_ptr sqlite_db; +}; + +} + +#endif diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index 6cf40c4f090..2aee91997b0 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -42,6 +42,10 @@ void registerTableFunctions() registerTableFunctionPostgreSQL(factory); #endif +#if USE_SQLITE + registerTableFunctionSQLite(factory); +#endif + registerTableFunctionDictionary(factory); } diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index c49fafc5f86..6a13c82caba 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -42,6 +42,10 @@ void registerTableFunctionMySQL(TableFunctionFactory & factory); void registerTableFunctionPostgreSQL(TableFunctionFactory & factory); #endif +#if USE_SQLITE +void registerTableFunctionSQLite(TableFunctionFactory & factory); +#endif + void registerTableFunctionDictionary(TableFunctionFactory & factory); void registerTableFunctions(); diff --git a/src/TableFunctions/ya.make b/src/TableFunctions/ya.make index f50e345f2d8..e957c923423 100644 --- a/src/TableFunctions/ya.make +++ b/src/TableFunctions/ya.make @@ -22,6 +22,7 @@ SRCS( TableFunctionNull.cpp TableFunctionNumbers.cpp TableFunctionRemote.cpp + TableFunctionSQLite.cpp TableFunctionURL.cpp TableFunctionValues.cpp TableFunctionView.cpp diff --git a/tests/integration/README.md b/tests/integration/README.md index cc504f01f82..8c353658705 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -5,14 +5,14 @@ This directory contains tests that involve several ClickHouse instances, custom ### Running natively Prerequisites: -* Ubuntu 14.04 (Trusty) or higher. +* Ubuntu 20.04 (Focal) or higher. * [docker](https://www.docker.com/community-edition#/download). Minimum required API version: 1.25, check with `docker version`. You must install latest Docker from https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/#set-up-the-repository Don't use Docker from your system repository. -* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev libkrb5-dev` +* [pip](https://pypi.python.org/pypi/pip) and `libpq-dev`. To install: `sudo apt-get install python3-pip libpq-dev zlib1g-dev libcrypto++-dev libssl-dev libkrb5-dev python3-dev` * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest` * [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install: @@ -25,25 +25,29 @@ sudo -H pip install \ confluent-kafka \ dicttoxml \ docker \ - docker-compose==1.22.0 \ + docker-compose \ grpcio \ grpcio-tools \ kafka-python \ kazoo \ minio \ protobuf \ - psycopg2-binary==2.7.5 \ + psycopg2-binary \ pymongo \ + pytz \ pytest \ pytest-timeout \ redis \ tzlocal \ urllib3 \ requests-kerberos \ - dict2xml + dict2xml \ + hypothesis \ + pyhdfs \ + pika ``` -(highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio` +(highly not recommended) If you really want to use OS packages on modern debian/ubuntu instead of "pip": `sudo apt install -y docker docker-compose python3-pytest python3-dicttoxml python3-docker python3-pymysql python3-protobuf python3-pymongo python3-tzlocal python3-kazoo python3-psycopg2 kafka-python python3-pytest-timeout python3-minio` If you want to run the tests under a non-privileged user, you must add this user to `docker` group: `sudo usermod -aG docker $USER` and re-login. (You must close all your sessions (for example, restart your computer)) diff --git a/tests/integration/helpers/hdfs_api.py b/tests/integration/helpers/hdfs_api.py index 2000de8c36a..3d2d647d0ed 100644 --- a/tests/integration/helpers/hdfs_api.py +++ b/tests/integration/helpers/hdfs_api.py @@ -102,6 +102,7 @@ class HDFSApi(object): return response_data else: logging.error(f"unexpected response_data.status_code {response_data.status_code} != {expected_code}") + time.sleep(1) response_data.raise_for_status() diff --git a/tests/integration/helpers/test_tools.py b/tests/integration/helpers/test_tools.py index ef530c4836b..b5d40659629 100644 --- a/tests/integration/helpers/test_tools.py +++ b/tests/integration/helpers/test_tools.py @@ -39,6 +39,9 @@ class TSV: def __str__(self): return '\n'.join(self.lines) + def __repr__(self): + return self.__str__() + def __len__(self): return len(self.lines) diff --git a/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py new file mode 100644 index 00000000000..b3f5c68cf68 --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_select_aggregate_alias_column.py @@ -0,0 +1,29 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__, name="aggregate_alias_column") +node1 = cluster.add_instance('node1', with_zookeeper=False) +node2 = cluster.add_instance('node2', + with_zookeeper=False, image='yandex/clickhouse-server', tag='21.7.2.7', stay_alive=True, + with_installed_binary=True) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_select_aggregate_alias_column(start_cluster): + node1.query("create table tab (x UInt64, x_alias UInt64 ALIAS x) engine = Memory") + node2.query("create table tab (x UInt64, x_alias UInt64 ALIAS x) engine = Memory") + node1.query('insert into tab values (1)') + node2.query('insert into tab values (1)') + + node1.query("select sum(x_alias) from remote('node{1,2}', default, tab)") + node2.query("select sum(x_alias) from remote('node{1,2}', default, tab)") diff --git a/tests/integration/test_config_substitutions/configs/config_env.xml b/tests/integration/test_config_substitutions/configs/config_env.xml index 712855c47c0..2d63b9c688d 100644 --- a/tests/integration/test_config_substitutions/configs/config_env.xml +++ b/tests/integration/test_config_substitutions/configs/config_env.xml @@ -10,5 +10,8 @@ default default + + + diff --git a/tests/integration/test_config_substitutions/configs/config_incl.xml b/tests/integration/test_config_substitutions/configs/config_incl.xml index 383a23af1ff..43ec78ff8ef 100644 --- a/tests/integration/test_config_substitutions/configs/config_incl.xml +++ b/tests/integration/test_config_substitutions/configs/config_incl.xml @@ -1,5 +1,5 @@ - /etc/clickhouse-server/config.d/max_query_size.xml + /etc/clickhouse-server/config.d/include_from_source.xml @@ -11,5 +11,8 @@ default default + + + diff --git a/tests/integration/test_config_substitutions/configs/config_include_from_env.xml b/tests/integration/test_config_substitutions/configs/config_include_from_env.xml index 71e11235749..79b650f3d9e 100644 --- a/tests/integration/test_config_substitutions/configs/config_include_from_env.xml +++ b/tests/integration/test_config_substitutions/configs/config_include_from_env.xml @@ -11,5 +11,7 @@ default default + + diff --git a/tests/integration/test_config_substitutions/configs/config_zk.xml b/tests/integration/test_config_substitutions/configs/config_zk.xml index aa589e9f9d3..9fad5658445 100644 --- a/tests/integration/test_config_substitutions/configs/config_zk.xml +++ b/tests/integration/test_config_substitutions/configs/config_zk.xml @@ -10,5 +10,8 @@ default default + + + diff --git a/tests/integration/test_config_substitutions/configs/include_from_source.xml b/tests/integration/test_config_substitutions/configs/include_from_source.xml new file mode 100644 index 00000000000..6095180bb59 --- /dev/null +++ b/tests/integration/test_config_substitutions/configs/include_from_source.xml @@ -0,0 +1,17 @@ + + 99999 + + + + + default + + + + + + + default + + + diff --git a/tests/integration/test_config_substitutions/configs/max_query_size.xml b/tests/integration/test_config_substitutions/configs/max_query_size.xml deleted file mode 100644 index 9ec61368be9..00000000000 --- a/tests/integration/test_config_substitutions/configs/max_query_size.xml +++ /dev/null @@ -1,3 +0,0 @@ - - 99999 - diff --git a/tests/integration/test_config_substitutions/test.py b/tests/integration/test_config_substitutions/test.py index 565cd1c0e97..aec3f1d3635 100644 --- a/tests/integration/test_config_substitutions/test.py +++ b/tests/integration/test_config_substitutions/test.py @@ -8,11 +8,11 @@ node2 = cluster.add_instance('node2', user_configs=['configs/config_env.xml'], env_variables={"MAX_QUERY_SIZE": "55555"}) node3 = cluster.add_instance('node3', user_configs=['configs/config_zk.xml'], with_zookeeper=True) node4 = cluster.add_instance('node4', user_configs=['configs/config_incl.xml'], - main_configs=['configs/max_query_size.xml']) # include value 77777 + main_configs=['configs/include_from_source.xml']) # include value 77777 node5 = cluster.add_instance('node5', user_configs=['configs/config_allow_databases.xml']) node6 = cluster.add_instance('node6', user_configs=['configs/config_include_from_env.xml'], - env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/max_query_size.xml"}, - main_configs=['configs/max_query_size.xml']) + env_variables={"INCLUDE_FROM_ENV": "/etc/clickhouse-server/config.d/include_from_source.xml"}, + main_configs=['configs/include_from_source.xml']) @pytest.fixture(scope="module") @@ -20,6 +20,8 @@ def start_cluster(): try: def create_zk_roots(zk): zk.create(path="/setting/max_query_size", value=b"77777", makepath=True) + zk.create(path="/users_from_zk_1", value=b"default", makepath=True) + zk.create(path="/users_from_zk_2", value=b"default", makepath=True) cluster.add_zookeeper_startup_command(create_zk_roots) @@ -37,6 +39,18 @@ def test_config(start_cluster): assert node6.query("select value from system.settings where name = 'max_query_size'") == "99999\n" +def test_include_config(start_cluster): + # + assert node4.query("select 1") + assert node4.query("select 1", user="user_1") + assert node4.query("select 1", user="user_2") + + # memory + + encrypted + disk_s3 + 1234567812345678 + diff --git a/tests/integration/test_disk_types/configs/storage.xml b/tests/integration/test_disk_types/configs/storage.xml index 1167a4f7382..4d8050c050c 100644 --- a/tests/integration/test_disk_types/configs/storage.xml +++ b/tests/integration/test_disk_types/configs/storage.xml @@ -15,6 +15,11 @@ hdfs http://hdfs1:9000/data/ + + encrypted + disk_s3 + 1234567812345678 + diff --git a/tests/integration/test_disk_types/test.py b/tests/integration/test_disk_types/test.py index 3f1a656d98f..35e900c3c9f 100644 --- a/tests/integration/test_disk_types/test.py +++ b/tests/integration/test_disk_types/test.py @@ -6,6 +6,7 @@ disk_types = { "disk_s3": "s3", "disk_memory": "memory", "disk_hdfs": "hdfs", + "disk_encrypted": "encrypted", } diff --git a/tests/integration/test_encrypted_disk/__init__.py b/tests/integration/test_encrypted_disk/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_encrypted_disk/configs/storage.xml b/tests/integration/test_encrypted_disk/configs/storage.xml new file mode 100644 index 00000000000..b0485178b13 --- /dev/null +++ b/tests/integration/test_encrypted_disk/configs/storage.xml @@ -0,0 +1,61 @@ + + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + memory + + + local + /disk/ + + + encrypted + disk_s3 + encrypted/ + 1234567812345678 + + + encrypted + disk_local + encrypted/ + abcdefghijklmnop + + + + + +
+ disk_local_encrypted +
+ + + + +
+ disk_local +
+ + disk_local_encrypted + + + + + +
+ disk_s3 +
+ + disk_s3_encrypted + + + + + + diff --git a/tests/integration/test_encrypted_disk/test.py b/tests/integration/test_encrypted_disk/test.py new file mode 100644 index 00000000000..64085991ade --- /dev/null +++ b/tests/integration/test_encrypted_disk/test.py @@ -0,0 +1,110 @@ +import pytest +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException + + +FIRST_PART_NAME = "all_1_1_0" + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + node = cluster.add_instance("node", + main_configs=["configs/storage.xml"], + tmpfs=["/disk:size=100M"], + with_minio=True) + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.mark.parametrize("policy", ["encrypted_policy", "local_policy", "s3_policy"]) +def test_encrypted_disk(cluster, policy): + node = cluster.instances["node"] + node.query( + """ + CREATE TABLE encrypted_test ( + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS storage_policy='{}' + """.format(policy) + ) + + node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") + select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" + assert node.query(select_query) == "(0,'data'),(1,'data')" + + node.query("INSERT INTO encrypted_test VALUES (2,'data'),(3,'data')") + node.query("OPTIMIZE TABLE encrypted_test FINAL") + assert node.query(select_query) == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" + + node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY") + + +@pytest.mark.parametrize("policy,disk,encrypted_disk", [("local_policy", "disk_local", "disk_local_encrypted"), ("s3_policy", "disk_s3", "disk_s3_encrypted")]) +def test_part_move(cluster, policy, disk, encrypted_disk): + node = cluster.instances["node"] + node.query( + """ + CREATE TABLE encrypted_test ( + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS storage_policy='{}' + """.format(policy) + ) + + node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") + select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" + assert node.query(select_query) == "(0,'data'),(1,'data')" + + node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk)) + assert node.query(select_query) == "(0,'data'),(1,'data')" + + with pytest.raises(QueryRuntimeException) as exc: + node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk)) + + assert("Part '{}' is already on disk '{}'".format(FIRST_PART_NAME, encrypted_disk) in str(exc.value)) + + node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, disk)) + assert node.query(select_query) == "(0,'data'),(1,'data')" + + node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY") + + +@pytest.mark.parametrize("policy,encrypted_disk", [("local_policy", "disk_local_encrypted"), ("s3_policy", "disk_s3_encrypted")]) +def test_optimize_table(cluster, policy, encrypted_disk): + node = cluster.instances["node"] + node.query( + """ + CREATE TABLE encrypted_test ( + id Int64, + data String + ) ENGINE=MergeTree() + ORDER BY id + SETTINGS storage_policy='{}' + """.format(policy) + ) + + node.query("INSERT INTO encrypted_test VALUES (0,'data'),(1,'data')") + select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values" + assert node.query(select_query) == "(0,'data'),(1,'data')" + + node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk)) + assert node.query(select_query) == "(0,'data'),(1,'data')" + + node.query("INSERT INTO encrypted_test VALUES (2,'data'),(3,'data')") + node.query("OPTIMIZE TABLE encrypted_test FINAL") + + with pytest.raises(QueryRuntimeException) as exc: + node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk)) + + assert("Part {} is not exists or not active".format(FIRST_PART_NAME) in str(exc.value)) + + assert node.query(select_query) == "(0,'data'),(1,'data'),(2,'data'),(3,'data')" + + node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY") diff --git a/tests/integration/test_merge_tree_hdfs/test.py b/tests/integration/test_merge_tree_hdfs/test.py index 223ad2e1af2..d26692a0d93 100644 --- a/tests/integration/test_merge_tree_hdfs/test.py +++ b/tests/integration/test_merge_tree_hdfs/test.py @@ -78,7 +78,7 @@ def wait_for_delete_hdfs_objects(cluster, expected, num_tries=30): while num_tries > 0: num_hdfs_objects = len(fs.listdir('/clickhouse')) if num_hdfs_objects == expected: - break; + break num_tries -= 1 time.sleep(1) assert(len(fs.listdir('/clickhouse')) == expected) diff --git a/tests/integration/test_postgresql_replica_database_engine/test.py b/tests/integration/test_postgresql_replica_database_engine/test.py index 97fd461e640..ed26ab82bc7 100644 --- a/tests/integration/test_postgresql_replica_database_engine/test.py +++ b/tests/integration/test_postgresql_replica_database_engine/test.py @@ -236,7 +236,7 @@ def test_different_data_types(started_cluster): ( key Integer NOT NULL PRIMARY KEY, a Date[] NOT NULL, -- Date - b Timestamp[] NOT NULL, -- DateTime + b Timestamp[] NOT NULL, -- DateTime64(6) c real[][] NOT NULL, -- Float32 d double precision[][] NOT NULL, -- Float64 e decimal(5, 5)[][][] NOT NULL, -- Decimal32 @@ -253,11 +253,11 @@ def test_different_data_types(started_cluster): for i in range(10): instance.query(''' INSERT INTO postgres_database.test_data_types VALUES - ({}, -32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12', '2000-05-12', 0.2, 0.2)'''.format(i)) + ({}, -32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12.012345', '2000-05-12', 0.2, 0.2)'''.format(i)) check_tables_are_synchronized('test_data_types', 'id'); result = instance.query('SELECT * FROM test_database.test_data_types ORDER BY id LIMIT 1;') - assert(result == '0\t-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12\t2000-05-12\t0.20000\t0.20000\n') + assert(result == '0\t-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t0.20000\t0.20000\n') for i in range(10): col = random.choice(['a', 'b', 'c']) @@ -270,7 +270,7 @@ def test_different_data_types(started_cluster): "VALUES (" "0, " "['2000-05-12', '2000-05-12'], " - "['2000-05-12 12:12:12', '2000-05-12 12:12:12'], " + "['2000-05-12 12:12:12.012345', '2000-05-12 12:12:12.012345'], " "[[1.12345], [1.12345], [1.12345]], " "[[1.1234567891], [1.1234567891], [1.1234567891]], " "[[[0.11111, 0.11111]], [[0.22222, 0.22222]], [[0.33333, 0.33333]]], " @@ -284,7 +284,7 @@ def test_different_data_types(started_cluster): expected = ( "0\t" + "['2000-05-12','2000-05-12']\t" + - "['2000-05-12 12:12:12','2000-05-12 12:12:12']\t" + + "['2000-05-12 12:12:12.012345','2000-05-12 12:12:12.012345']\t" + "[[1.12345],[1.12345],[1.12345]]\t" + "[[1.1234567891],[1.1234567891],[1.1234567891]]\t" + "[[[0.11111,0.11111]],[[0.22222,0.22222]],[[0.33333,0.33333]]]\t" @@ -622,7 +622,7 @@ def test_virtual_columns(started_cluster): instance.query("INSERT INTO postgres_database.postgresql_replica_0 SELECT number, number from numbers(10)") check_tables_are_synchronized('postgresql_replica_0'); - # just check that it works, no check with `expected` becuase _version is taken as LSN, which will be different each time. + # just check that it works, no check with `expected` because _version is taken as LSN, which will be different each time. result = instance.query('SELECT key, value, _sign, _version FROM test_database.postgresql_replica_0;') print(result) diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/__init__.py b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/configs/config.d/storage_conf.xml new file mode 100644 index 00000000000..46a11a8fe16 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/configs/config.d/storage_conf.xml @@ -0,0 +1,86 @@ + + + + + hdfs + hdfs://hdfs1:9000/clickhouse1/ + + + hdfs + hdfs://hdfs1:9000/clickhouse1/ + + + hdfs + hdfs://hdfs1:9000/clickhouse2/ + + + + + +
+ hdfs1 +
+ + + + +
+ default +
+ + hdfs1 + + + 0.0 + + + +
+ hdfs2 +
+ + hdfs1 + + + + + +
+ hdfs1_again +
+ + hdfs1 + + + + + + + + 1024000 + 1 + 1 + + + + + + + node1 + 9000 + + + + + node2 + 9000 + + + + + + + test_cluster + 1 + + diff --git a/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py new file mode 100644 index 00000000000..f426c3619a4 --- /dev/null +++ b/tests/integration/test_replicated_merge_tree_hdfs_zero_copy/test.py @@ -0,0 +1,212 @@ +import logging +from string import Template +import time + +import pytest +from helpers.cluster import ClickHouseCluster + +from pyhdfs import HdfsClient + +SHARDS = 2 +FILES_OVERHEAD_PER_TABLE = 1 # format_version.txt +FILES_OVERHEAD_PER_PART_COMPACT = 7 + + +def wait_for_hdfs_objects(cluster, fp, expected, num_tries=30): + fs = HdfsClient(hosts=cluster.hdfs_ip) + while num_tries > 0: + num_hdfs_objects = len(fs.listdir(fp)) + if num_hdfs_objects == expected: + break + num_tries -= 1 + time.sleep(1) + assert(len(fs.listdir(fp)) == expected) + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance("node1", main_configs=["configs/config.d/storage_conf.xml"], + macros={'replica': 'node1'}, + with_zookeeper=True, + with_hdfs=True) + cluster.add_instance("node2", main_configs=["configs/config.d/storage_conf.xml"], + macros={'replica': 'node2'}, + with_zookeeper=True, + with_hdfs=True) + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + fs = HdfsClient(hosts=cluster.hdfs_ip) + fs.mkdirs('/clickhouse1') + fs.mkdirs('/clickhouse2') + logging.info("Created HDFS directory") + + yield cluster + finally: + cluster.shutdown() + + +def test_hdfs_zero_copy_replication_insert(cluster): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + try: + node1.query( + """ + CREATE TABLE hdfs_test ON CLUSTER test_cluster (dt DateTime, id Int64) + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/hdfs_test', '{replica}') + ORDER BY (dt, id) + SETTINGS storage_policy='hdfs_only' + """ + ) + wait_for_hdfs_objects(cluster, "/clickhouse1", SHARDS * FILES_OVERHEAD_PER_TABLE) + + node1.query("INSERT INTO hdfs_test VALUES (now() - INTERVAL 3 DAY, 10)") + node2.query("SYSTEM SYNC REPLICA hdfs_test") + assert node1.query("SELECT count() FROM hdfs_test FORMAT Values") == "(1)" + assert node2.query("SELECT count() FROM hdfs_test FORMAT Values") == "(1)" + assert node1.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)" + assert node2.query("SELECT id FROM hdfs_test ORDER BY dt FORMAT Values") == "(10)" + assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values") == "('all','hdfs1')" + assert node2.query("SELECT partition_id,disk_name FROM system.parts WHERE table='hdfs_test' FORMAT Values") == "('all','hdfs1')" + wait_for_hdfs_objects(cluster, "/clickhouse1", SHARDS * FILES_OVERHEAD_PER_TABLE + FILES_OVERHEAD_PER_PART_COMPACT) + finally: + node1.query("DROP TABLE IF EXISTS hdfs_test NO DELAY") + node2.query("DROP TABLE IF EXISTS hdfs_test NO DELAY") + + + +@pytest.mark.parametrize( + ("storage_policy", "init_objects"), + [("hybrid", 0), + ("tiered", 0), + ("tiered_copy", FILES_OVERHEAD_PER_TABLE)] +) +def test_hdfs_zero_copy_replication_single_move(cluster, storage_policy, init_objects): + node1 = cluster.instances["node1"] + try: + node1.query( + Template(""" + CREATE TABLE single_node_move_test (dt DateTime, id Int64) + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/single_node_move_test', '{replica}') + ORDER BY (dt, id) + SETTINGS storage_policy='$policy' + """).substitute(policy=storage_policy) + ) + wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects) + + node1.query("INSERT INTO single_node_move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)") + assert node1.query("SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values") == "(10),(11)" + + node1.query("ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'external'") + assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='single_node_move_test' FORMAT Values") == "('all','hdfs1')" + assert node1.query("SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values") == "(10),(11)" + wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT) + + node1.query("ALTER TABLE single_node_move_test MOVE PARTITION ID 'all' TO VOLUME 'main'") + assert node1.query("SELECT id FROM single_node_move_test ORDER BY dt FORMAT Values") == "(10),(11)" + finally: + node1.query("DROP TABLE IF EXISTS single_node_move_test NO DELAY") + + +@pytest.mark.parametrize( + ("storage_policy", "init_objects"), + [("hybrid", 0), + ("tiered", 0), + ("tiered_copy", SHARDS * FILES_OVERHEAD_PER_TABLE)] +) +def test_hdfs_zero_copy_replication_move(cluster, storage_policy, init_objects): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + try: + node1.query( + Template(""" + CREATE TABLE move_test ON CLUSTER test_cluster (dt DateTime, id Int64) + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/move_test', '{replica}') + ORDER BY (dt, id) + SETTINGS storage_policy='$policy' + """).substitute(policy=storage_policy) + ) + wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects) + + node1.query("INSERT INTO move_test VALUES (now() - INTERVAL 3 DAY, 10), (now() - INTERVAL 1 DAY, 11)") + node2.query("SYSTEM SYNC REPLICA move_test") + + assert node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" + assert node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" + + node1.query("ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'") + wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT) + + node2.query("ALTER TABLE move_test MOVE PARTITION ID 'all' TO VOLUME 'external'") + assert node1.query("SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values") == "('all','hdfs1')" + assert node2.query("SELECT partition_id,disk_name FROM system.parts WHERE table='move_test' FORMAT Values") == "('all','hdfs1')" + assert node1.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" + assert node2.query("SELECT id FROM move_test ORDER BY dt FORMAT Values") == "(10),(11)" + wait_for_hdfs_objects(cluster, "/clickhouse1", init_objects + FILES_OVERHEAD_PER_PART_COMPACT) + finally: + node1.query("DROP TABLE IF EXISTS move_test NO DELAY") + node2.query("DROP TABLE IF EXISTS move_test NO DELAY") + + +@pytest.mark.parametrize( + ("storage_policy"), ["hybrid", "tiered", "tiered_copy"] +) +def test_hdfs_zero_copy_with_ttl_move(cluster, storage_policy): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + try: + node1.query( + Template(""" + CREATE TABLE ttl_move_test ON CLUSTER test_cluster (dt DateTime, id Int64) + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/ttl_move_test', '{replica}') + ORDER BY (dt, id) + TTL dt + INTERVAL 2 DAY TO VOLUME 'external' + SETTINGS storage_policy='$policy' + """).substitute(policy=storage_policy) + ) + + node1.query("INSERT INTO ttl_move_test VALUES (now() - INTERVAL 3 DAY, 10)") + node1.query("INSERT INTO ttl_move_test VALUES (now() - INTERVAL 1 DAY, 11)") + + node1.query("OPTIMIZE TABLE ttl_move_test FINAL") + node2.query("SYSTEM SYNC REPLICA ttl_move_test") + + assert node1.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" + assert node2.query("SELECT count() FROM ttl_move_test FORMAT Values") == "(2)" + assert node1.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values") == "(10),(11)" + assert node2.query("SELECT id FROM ttl_move_test ORDER BY id FORMAT Values") == "(10),(11)" + finally: + node1.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") + node2.query("DROP TABLE IF EXISTS ttl_move_test NO DELAY") + + +def test_hdfs_zero_copy_with_ttl_delete(cluster): + node1 = cluster.instances["node1"] + node2 = cluster.instances["node2"] + try: + node1.query( + """ + CREATE TABLE ttl_delete_test ON CLUSTER test_cluster (dt DateTime, id Int64) + ENGINE=ReplicatedMergeTree('/clickhouse/tables/{cluster}/{shard}/ttl_delete_test', '{replica}') + ORDER BY (dt, id) + TTL dt + INTERVAL 2 DAY + SETTINGS storage_policy='tiered' + """ + ) + + node1.query("INSERT INTO ttl_delete_test VALUES (now() - INTERVAL 3 DAY, 10)") + node1.query("INSERT INTO ttl_delete_test VALUES (now() - INTERVAL 1 DAY, 11)") + + node1.query("OPTIMIZE TABLE ttl_delete_test FINAL") + node2.query("SYSTEM SYNC REPLICA ttl_delete_test") + + assert node1.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1)" + assert node2.query("SELECT count() FROM ttl_delete_test FORMAT Values") == "(1)" + assert node1.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values") == "(11)" + assert node2.query("SELECT id FROM ttl_delete_test ORDER BY id FORMAT Values") == "(11)" + finally: + node1.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") + node2.query("DROP TABLE IF EXISTS ttl_delete_test NO DELAY") diff --git a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml index 1f75a4efeae..d22ac8113a8 100644 --- a/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3/configs/config.d/storage_conf.xml @@ -21,7 +21,7 @@ 0 - 0 + 0 diff --git a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml index d8c7f49fc49..0cf9191c4af 100644 --- a/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml +++ b/tests/integration/test_replicated_merge_tree_s3_zero_copy/configs/config.d/storage_conf.xml @@ -21,7 +21,7 @@ 0 - 1 + 1 diff --git a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml index db639cabb63..89c97aa3360 100644 --- a/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml +++ b/tests/integration/test_s3_zero_copy_replication/configs/config.d/s3.xml @@ -66,7 +66,7 @@ 1024 1 - 1 + 1 diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 307879265df..9e10ed22f78 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -85,10 +85,10 @@ def test_postgres_conversions(started_cluster): h timestamp, i date, j decimal(5, 3), k numeric, l boolean)''') node1.query(''' INSERT INTO TABLE FUNCTION postgresql('postgres1:5432', 'clickhouse', 'test_types', 'postgres', 'mysecretpassword') VALUES - (-32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12', '2000-05-12', 22.222, 22.222, 1)''') + (-32768, -2147483648, -9223372036854775808, 1.12345, 1.1234567890, 2147483647, 9223372036854775807, '2000-05-12 12:12:12.012345', '2000-05-12', 22.222, 22.222, 1)''') result = node1.query(''' SELECT a, b, c, d, e, f, g, h, i, j, toDecimal128(k, 3), l FROM postgresql('postgres1:5432', 'clickhouse', 'test_types', 'postgres', 'mysecretpassword')''') - assert(result == '-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12\t2000-05-12\t22.222\t22.222\t1\n') + assert(result == '-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t22.222\t22.222\t1\n') cursor.execute("INSERT INTO test_types (l) VALUES (TRUE), (true), ('yes'), ('y'), ('1');") cursor.execute("INSERT INTO test_types (l) VALUES (FALSE), (false), ('no'), ('off'), ('0');") @@ -100,7 +100,7 @@ def test_postgres_conversions(started_cluster): '''CREATE TABLE IF NOT EXISTS test_array_dimensions ( a Date[] NOT NULL, -- Date - b Timestamp[] NOT NULL, -- DateTime + b Timestamp[] NOT NULL, -- DateTime64(6) c real[][] NOT NULL, -- Float32 d double precision[][] NOT NULL, -- Float64 e decimal(5, 5)[][][] NOT NULL, -- Decimal32 @@ -114,7 +114,7 @@ def test_postgres_conversions(started_cluster): result = node1.query(''' DESCRIBE TABLE postgresql('postgres1:5432', 'clickhouse', 'test_array_dimensions', 'postgres', 'mysecretpassword')''') expected = ('a\tArray(Date)\t\t\t\t\t\n' + - 'b\tArray(DateTime)\t\t\t\t\t\n' + + 'b\tArray(DateTime64(6))\t\t\t\t\t\n' + 'c\tArray(Array(Float32))\t\t\t\t\t\n' + 'd\tArray(Array(Float64))\t\t\t\t\t\n' + 'e\tArray(Array(Array(Decimal(5, 5))))\t\t\t\t\t\n' + @@ -129,7 +129,7 @@ def test_postgres_conversions(started_cluster): node1.query("INSERT INTO TABLE FUNCTION postgresql('postgres1:5432', 'clickhouse', 'test_array_dimensions', 'postgres', 'mysecretpassword') " "VALUES (" "['2000-05-12', '2000-05-12'], " - "['2000-05-12 12:12:12', '2000-05-12 12:12:12'], " + "['2000-05-12 12:12:12.012345', '2000-05-12 12:12:12.012345'], " "[[1.12345], [1.12345], [1.12345]], " "[[1.1234567891], [1.1234567891], [1.1234567891]], " "[[[0.11111, 0.11111]], [[0.22222, 0.22222]], [[0.33333, 0.33333]]], " @@ -144,7 +144,7 @@ def test_postgres_conversions(started_cluster): SELECT * FROM postgresql('postgres1:5432', 'clickhouse', 'test_array_dimensions', 'postgres', 'mysecretpassword')''') expected = ( "['2000-05-12','2000-05-12']\t" + - "['2000-05-12 12:12:12','2000-05-12 12:12:12']\t" + + "['2000-05-12 12:12:12.012345','2000-05-12 12:12:12.012345']\t" + "[[1.12345],[1.12345],[1.12345]]\t" + "[[1.1234567891],[1.1234567891],[1.1234567891]]\t" + "[[[0.11111,0.11111]],[[0.22222,0.22222]],[[0.33333,0.33333]]]\t" diff --git a/tests/performance/jit_aggregate_functions.xml b/tests/performance/jit_aggregate_functions.xml index 21683ef2004..3e99f6d9615 100644 --- a/tests/performance/jit_aggregate_functions.xml +++ b/tests/performance/jit_aggregate_functions.xml @@ -69,6 +69,9 @@ any anyLast count + groupBitOr + groupBitAnd + groupBitXor @@ -119,7 +122,7 @@ SELECT {function}(value_1), {function}(value_2), - groupBitAnd(value_3), + sum(toUInt256(value_3)), {function}(value_3) FROM {table} GROUP BY key @@ -140,7 +143,7 @@ SELECT {function}If(value_1, predicate), {function}If(value_2, predicate), - groupBitAndIf(value_3, predicate), + sumIf(toUInt256(value_3), predicate), {function}If(value_3, predicate) FROM {table} GROUP BY key @@ -163,7 +166,7 @@ SELECT {function}(value_1), {function}(value_2), - groupBitAnd(value_3), + sum(toUInt256(value_3)), {function}(value_3), {function}(value_4), {function}(value_5) @@ -188,7 +191,7 @@ SELECT {function}If(value_1, predicate), {function}If(value_2, predicate), - groupBitAndIf(value_3, predicate), + sumIf(toUInt256(value_3), predicate), {function}If(value_3, predicate), {function}If(value_4, predicate), {function}If(value_5, predicate) @@ -212,7 +215,7 @@ SELECT {function}(WatchID), {function}(CounterID), - groupBitAnd(ClientIP), + sum(toUInt256(ClientIP)), {function}(ClientIP) FROM hits_100m_single GROUP BY intHash32(UserID) % {group_scale} @@ -224,8 +227,8 @@ {function}(WatchID), {function}(CounterID), {function}(ClientIP), - {function}(GoodEvent), - {function}(CounterClass) + {function}(IPNetworkID), + {function}(SearchEngineID) FROM hits_100m_single GROUP BY intHash32(UserID) % {group_scale} FORMAT Null @@ -235,10 +238,10 @@ SELECT {function}(WatchID), {function}(CounterID), - groupBitAnd(ClientIP), + sum(toUInt256(ClientIP)), {function}(ClientIP), - {function}(GoodEvent), - {function}(CounterClass) + {function}(IPNetworkID), + {function}(SearchEngineID) FROM hits_100m_single GROUP BY intHash32(UserID) % {group_scale} FORMAT Null @@ -260,7 +263,7 @@ SELECT {function}If(WatchID, predicate), {function}If(CounterID, predicate), - groupBitAndIf(ClientIP, predicate), + sumIf(toUInt256(ClientIP), predicate), {function}If(ClientIP, predicate) FROM hits_100m_single GROUP BY intHash32(UserID) % {group_scale} @@ -273,8 +276,8 @@ {function}If(WatchID, predicate), {function}If(CounterID, predicate), {function}If(ClientIP, predicate), - {function}If(GoodEvent, predicate), - {function}If(CounterClass, predicate) + {function}If(IPNetworkID, predicate), + {function}If(SearchEngineID, predicate) FROM hits_100m_single GROUP BY intHash32(UserID) % {group_scale} FORMAT Null @@ -285,10 +288,10 @@ SELECT {function}If(WatchID, predicate), {function}If(CounterID, predicate), - groupBitAndIf(ClientIP, predicate), + sumIf(toUInt256(ClientIP), predicate), {function}If(ClientIP, predicate), - {function}If(GoodEvent, predicate), - {function}If(CounterClass, predicate) + {function}If(IPNetworkID, predicate), + {function}If(SearchEngineID, predicate) FROM hits_100m_single GROUP BY intHash32(UserID) % {group_scale} FORMAT Null diff --git a/tests/queries/0_stateless/00850_global_join_dups.reference b/tests/queries/0_stateless/00850_global_join_dups.reference index d94e4df3425..bbe467dd08e 100644 --- a/tests/queries/0_stateless/00850_global_join_dups.reference +++ b/tests/queries/0_stateless/00850_global_join_dups.reference @@ -1,5 +1,6 @@ 1 \N +\N 0 0 0 0 diff --git a/tests/queries/0_stateless/00850_global_join_dups.sql b/tests/queries/0_stateless/00850_global_join_dups.sql index 92c1d81aa8e..5843bf9c227 100644 --- a/tests/queries/0_stateless/00850_global_join_dups.sql +++ b/tests/queries/0_stateless/00850_global_join_dups.sql @@ -22,6 +22,8 @@ GLOBAL INNER JOIN -- query from fuzzer SELECT toDateTime64(toString(toString('0000-00-00 00:00:000000-00-00 00:00:00', toDateTime64(toDateTime64('655.36', -2, NULL)))), NULL) FROM t1_00850 GLOBAL INNER JOIN (SELECT toDateTime64(toDateTime64('6553.6', '', NULL), NULL), * FROM (SELECT * FROM t2_00850) INNER JOIN (SELECT toDateTime64('6553.7', 1024, NULL), * FROM t1_00850) USING (dummy)) USING (dummy); +SELECT toString('0000-00-00 00:00:000000-00-00 00:00:00', toDateTime64(toDateTime64('655.36', -2, NULL))); + DROP TABLE t_local; DROP TABLE t1_00850; DROP TABLE t2_00850; diff --git a/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql b/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql index 75e0e482566..3a55a69c726 100644 --- a/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql +++ b/tests/queries/0_stateless/00907_set_index_with_nullable_and_low_cardinality_bug.sql @@ -8,8 +8,7 @@ CREATE TABLE null_lc_set_index ( INDEX test_user_idx (user) TYPE set(0) GRANULARITY 8192 ) ENGINE=MergeTree PARTITION BY toYYYYMMDD(timestamp) - ORDER BY (timestamp, action, cityHash64(user)) - SAMPLE BY cityHash64(user); + ORDER BY (timestamp, action, cityHash64(user)); INSERT INTO null_lc_set_index VALUES (1550883010, 'subscribe', 'alice'); INSERT INTO null_lc_set_index VALUES (1550883020, 'follow', 'bob'); diff --git a/tests/queries/0_stateless/01107_join_right_table_totals.reference b/tests/queries/0_stateless/01107_join_right_table_totals.reference index f71d3b0d05f..daf503b776d 100644 --- a/tests/queries/0_stateless/01107_join_right_table_totals.reference +++ b/tests/queries/0_stateless/01107_join_right_table_totals.reference @@ -18,3 +18,31 @@ 0 0 0 0 +1 1 +1 1 + +0 0 +1 1 +1 1 + +0 0 +1 1 +1 1 + +0 0 +1 1 +1 1 + +0 0 +1 1 + +0 0 +1 foo 1 1 300 + +0 foo 1 0 300 +1 100 1970-01-01 1 100 1970-01-01 +1 100 1970-01-01 1 200 1970-01-02 +1 200 1970-01-02 1 100 1970-01-01 +1 200 1970-01-02 1 200 1970-01-02 + +0 0 1970-01-01 0 0 1970-01-01 diff --git a/tests/queries/0_stateless/01107_join_right_table_totals.sql b/tests/queries/0_stateless/01107_join_right_table_totals.sql index a4f284e5e2d..f894b6bf8bb 100644 --- a/tests/queries/0_stateless/01107_join_right_table_totals.sql +++ b/tests/queries/0_stateless/01107_join_right_table_totals.sql @@ -35,29 +35,66 @@ FULL JOIN ) rr USING (id); -SELECT id, yago +SELECT id, yago FROM ( SELECT item_id AS id FROM t GROUP BY id ) AS ll -FULL OUTER JOIN ( SELECT item_id AS id, arrayJoin([111, 222, 333, 444]), SUM(price_sold) AS yago FROM t GROUP BY id WITH TOTALS ) AS rr +FULL OUTER JOIN ( SELECT item_id AS id, arrayJoin([111, 222, 333, 444]), SUM(price_sold) AS yago FROM t GROUP BY id WITH TOTALS ) AS rr USING (id); -SELECT id, yago +SELECT id, yago FROM ( SELECT item_id AS id, arrayJoin([111, 222, 333]) FROM t GROUP BY id WITH TOTALS ) AS ll -FULL OUTER JOIN ( SELECT item_id AS id, SUM(price_sold) AS yago FROM t GROUP BY id ) AS rr +FULL OUTER JOIN ( SELECT item_id AS id, SUM(price_sold) AS yago FROM t GROUP BY id ) AS rr USING (id); -SELECT id, yago +SELECT id, yago FROM ( SELECT item_id AS id, arrayJoin(emptyArrayInt32()) FROM t GROUP BY id WITH TOTALS ) AS ll -FULL OUTER JOIN ( SELECT item_id AS id, SUM(price_sold) AS yago FROM t GROUP BY id ) AS rr +FULL OUTER JOIN ( SELECT item_id AS id, SUM(price_sold) AS yago FROM t GROUP BY id ) AS rr USING (id); -SELECT id, yago +SELECT id, yago FROM ( SELECT item_id AS id FROM t GROUP BY id ) AS ll -FULL OUTER JOIN ( SELECT item_id AS id, arrayJoin(emptyArrayInt32()), SUM(price_sold) AS yago FROM t GROUP BY id WITH TOTALS ) AS rr +FULL OUTER JOIN ( SELECT item_id AS id, arrayJoin(emptyArrayInt32()), SUM(price_sold) AS yago FROM t GROUP BY id WITH TOTALS ) AS rr USING (id); -SELECT id, yago +SELECT id, yago FROM ( SELECT item_id AS id, arrayJoin([111, 222, 333]) FROM t GROUP BY id WITH TOTALS ) AS ll -FULL OUTER JOIN ( SELECT item_id AS id, arrayJoin([111, 222, 333, 444]), SUM(price_sold) AS yago FROM t GROUP BY id WITH TOTALS ) AS rr +FULL OUTER JOIN ( SELECT item_id AS id, arrayJoin([111, 222, 333, 444]), SUM(price_sold) AS yago FROM t GROUP BY id WITH TOTALS ) AS rr USING (id); +INSERT INTO t VALUES (1, 100, '1970-01-01'), (1, 200, '1970-01-02'); + +SELECT * +FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) l +LEFT JOIN (SELECT item_id FROM t ) r +ON l.item_id = r.item_id; + +SELECT * +FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) l +RIGHT JOIN (SELECT item_id FROM t ) r +ON l.item_id = r.item_id; + +SELECT * +FROM (SELECT item_id FROM t) l +LEFT JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS ) r +ON l.item_id = r.item_id; + +SELECT * +FROM (SELECT item_id FROM t) l +RIGHT JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS ) r +ON l.item_id = r.item_id; + +SELECT * +FROM (SELECT item_id FROM t GROUP BY item_id WITH TOTALS) l +LEFT JOIN (SELECT item_id FROM t GROUP BY item_id WITH TOTALS ) r +ON l.item_id = r.item_id; + +SELECT * +FROM (SELECT item_id, 'foo' AS key, 1 AS val FROM t GROUP BY item_id WITH TOTALS) l +LEFT JOIN (SELECT item_id, sum(price_sold) AS val FROM t GROUP BY item_id WITH TOTALS ) r +ON l.item_id = r.item_id; + +SELECT * +FROM (SELECT * FROM t GROUP BY item_id, price_sold, date WITH TOTALS) l +LEFT JOIN (SELECT * FROM t GROUP BY item_id, price_sold, date WITH TOTALS ) r +ON l.item_id = r.item_id; + DROP TABLE t; diff --git a/tests/queries/0_stateless/01176_mysql_client_interactive.expect b/tests/queries/0_stateless/01176_mysql_client_interactive.expect index b2dc88a7795..2337b7d01fe 100755 --- a/tests/queries/0_stateless/01176_mysql_client_interactive.expect +++ b/tests/queries/0_stateless/01176_mysql_client_interactive.expect @@ -22,5 +22,27 @@ expect "| dummy |" expect "| 0 |" expect "1 row in set" +# exception before start +send -- "select * from table_that_does_not_exist;\r" +expect "ERROR 60 (00000): Code: 60" + +# exception after start +send -- "select throwIf(number) from numbers(2) settings max_block_size=1;\r" +expect "ERROR 395 (00000): Code: 395" + +# other formats +send -- "select * from system.one format TSV;\r" +expect "ERROR 1 (00000): Code: 1" + +send -- "select count(number), sum(number) from numbers(10);\r" +expect "+---------------+-------------+" +expect "| count(number) | sum(number) |" +expect "+---------------+-------------+" +expect "| 10 | 45 |" +expect "+---------------+-------------+" +expect "1 row in set" +expect "Read 10 rows, 80.00 B" +expect "mysql> " + send -- "quit;\r" expect eof diff --git a/tests/queries/0_stateless/01226_dist_on_dist_global_in.reference b/tests/queries/0_stateless/01226_dist_on_dist_global_in.reference index 3d8d7fb770d..e7d4ea81714 100644 --- a/tests/queries/0_stateless/01226_dist_on_dist_global_in.reference +++ b/tests/queries/0_stateless/01226_dist_on_dist_global_in.reference @@ -2,5 +2,4 @@ GLOBAL IN 0 0 0 -0 GLOBAL NOT IN diff --git a/tests/queries/0_stateless/01226_dist_on_dist_global_in.sql b/tests/queries/0_stateless/01226_dist_on_dist_global_in.sql index 588ea9c1048..ca9b28a14f4 100644 --- a/tests/queries/0_stateless/01226_dist_on_dist_global_in.sql +++ b/tests/queries/0_stateless/01226_dist_on_dist_global_in.sql @@ -1,10 +1,8 @@ SELECT 'GLOBAL IN'; select * from remote('localhost', system.one) where dummy global in (0); -select * from remote('localhost', system.one) where toUInt64(dummy) global in numbers(1); select * from remote('localhost', system.one) where dummy global in system.one; select * from remote('localhost', system.one) where dummy global in (select 0); SELECT 'GLOBAL NOT IN'; select * from remote('localhost', system.one) where dummy global not in (0); -select * from remote('localhost', system.one) where toUInt64(dummy) global not in numbers(1); select * from remote('localhost', system.one) where dummy global not in system.one; select * from remote('localhost', system.one) where dummy global not in (select 0); diff --git a/tests/queries/0_stateless/01232_untuple.reference b/tests/queries/0_stateless/01232_untuple.reference index 44f96e1decd..21fd0c4a8a5 100644 --- a/tests/queries/0_stateless/01232_untuple.reference +++ b/tests/queries/0_stateless/01232_untuple.reference @@ -2,7 +2,7 @@ hello 1 3 world 9 9 (0,1) -key v1 v2 v3 v4 v5 +key tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 1) tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 2) tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 3) tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 4) tupleElement(argMax(tuple(v1, v2, v3, v4, v5), v1), 5) 4 10 20 10 20 30 3 70 20 10 20 30 2 11 20 10 20 30 diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 035cb902bff..95f9e407f21 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -119,6 +119,7 @@ REMOTE [] GLOBAL SOURCES MONGO [] GLOBAL SOURCES MYSQL [] GLOBAL SOURCES POSTGRES [] GLOBAL SOURCES +SQLITE [] GLOBAL SOURCES ODBC [] GLOBAL SOURCES JDBC [] GLOBAL SOURCES HDFS [] GLOBAL SOURCES diff --git a/tests/queries/0_stateless/01601_custom_tld.reference b/tests/queries/0_stateless/01601_custom_tld.reference index e056505f273..04204ebf02a 100644 --- a/tests/queries/0_stateless/01601_custom_tld.reference +++ b/tests/queries/0_stateless/01601_custom_tld.reference @@ -22,3 +22,9 @@ foobar.com foobar.com foobar.com xx.blogspot.co.at +-- www +www.foo +foo +-- vector +xx.blogspot.co.at + diff --git a/tests/queries/0_stateless/01601_custom_tld.sql b/tests/queries/0_stateless/01601_custom_tld.sql index 688dd419858..ceb00d5ff19 100644 --- a/tests/queries/0_stateless/01601_custom_tld.sql +++ b/tests/queries/0_stateless/01601_custom_tld.sql @@ -29,3 +29,11 @@ select cutToFirstSignificantSubdomainCustom('http://foobar.com', 'public_suffix_ select cutToFirstSignificantSubdomainCustom('http://foobar.com/foo', 'public_suffix_list'); select cutToFirstSignificantSubdomainCustom('http://bar.foobar.com/foo', 'public_suffix_list'); select cutToFirstSignificantSubdomainCustom('http://xx.blogspot.co.at', 'public_suffix_list'); + +select '-- www'; +select cutToFirstSignificantSubdomainCustomWithWWW('http://www.foo', 'public_suffix_list'); +select cutToFirstSignificantSubdomainCustom('http://www.foo', 'public_suffix_list'); + +select '-- vector'; +select cutToFirstSignificantSubdomainCustom('http://xx.blogspot.co.at/' || toString(number), 'public_suffix_list') from numbers(1); +select cutToFirstSignificantSubdomainCustom('there-is-no-such-domain' || toString(number), 'public_suffix_list') from numbers(1); diff --git a/tests/queries/0_stateless/01615_random_one_shard_insertion.reference b/tests/queries/0_stateless/01615_random_one_shard_insertion.reference index 448a73c4789..20ed3c2d518 100644 --- a/tests/queries/0_stateless/01615_random_one_shard_insertion.reference +++ b/tests/queries/0_stateless/01615_random_one_shard_insertion.reference @@ -1,8 +1,22 @@ -0 -0 1 1 -2 +0 +1 2 3 -3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 diff --git a/tests/queries/0_stateless/01615_random_one_shard_insertion.sql b/tests/queries/0_stateless/01615_random_one_shard_insertion.sql index 7d07629feda..59412adbdbf 100644 --- a/tests/queries/0_stateless/01615_random_one_shard_insertion.sql +++ b/tests/queries/0_stateless/01615_random_one_shard_insertion.sql @@ -1,22 +1,26 @@ -drop table if exists shard; +create database if not exists shard_0; +create database if not exists shard_1; +drop table if exists shard_0.tbl; +drop table if exists shard_1.tbl; drop table if exists distr; -create table shard (id Int32) engine = MergeTree order by cityHash64(id); -create table distr as shard engine Distributed (test_cluster_two_shards_localhost, currentDatabase(), shard); - -insert into distr (id) values (0), (1); -- { serverError 55; } +create table shard_0.tbl (number UInt64) engine = MergeTree order by number; +create table shard_1.tbl (number UInt64) engine = MergeTree order by number; +create table distr (number UInt64) engine = Distributed(test_cluster_two_shards_different_databases, '', tbl); set insert_distributed_sync = 1; - -insert into distr (id) values (0), (1); -- { serverError 55; } - -set insert_distributed_sync = 0; set insert_distributed_one_random_shard = 1; +set max_block_size = 1; +set max_insert_block_size = 1; +set min_insert_block_size_rows = 1; +insert into distr select number from numbers(20); -insert into distr (id) values (0), (1); -insert into distr (id) values (2), (3); +select count() != 0 from shard_0.tbl; +select count() != 0 from shard_1.tbl; +select * from distr order by number; -select * from distr order by id; - -drop table if exists shard; -drop table if exists distr; +drop table if exists shard_0.tbl; +drop table if exists shard_1.tbl; +drop database shard_0; +drop database shard_1; +drop table distr; diff --git a/tests/queries/0_stateless/01616_untuple_access_field.reference b/tests/queries/0_stateless/01616_untuple_access_field.reference index d00491fd7e5..9874d6464ab 100644 --- a/tests/queries/0_stateless/01616_untuple_access_field.reference +++ b/tests/queries/0_stateless/01616_untuple_access_field.reference @@ -1 +1 @@ -1 +1 2 diff --git a/tests/queries/0_stateless/01616_untuple_access_field.sql b/tests/queries/0_stateless/01616_untuple_access_field.sql index 569efca5349..82cdf80c8bc 100644 --- a/tests/queries/0_stateless/01616_untuple_access_field.sql +++ b/tests/queries/0_stateless/01616_untuple_access_field.sql @@ -1 +1 @@ -select _ut_1 from (select untuple((1,2))); +select * from (select untuple((1,2))); diff --git a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh index 072e8d75f52..1bfcf863184 100755 --- a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh +++ b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh @@ -44,7 +44,7 @@ echo "clickhouse-client --query "'"select file('"'${user_files_path}/dir'), file echo "clickhouse-client --query "'"select file('"'/tmp/c.txt'), file('${user_files_path}/b.txt')"'";echo :$?' | bash 2>/dev/null # Test relative path consists of ".." whose absolute path is out of the user_files directory. -echo "clickhouse-client --query "'"select file('"'${user_files_path}/../../../../tmp/c.txt'), file('b.txt')"'";echo :$?' | bash 2>/dev/null +echo "clickhouse-client --query "'"select file('"'${user_files_path}/../../../../../../../../../../../../../../../../../../../tmp/c.txt'), file('b.txt')"'";echo :$?' | bash 2>/dev/null echo "clickhouse-client --query "'"select file('"'../../../../a.txt'), file('${user_files_path}/b.txt')"'";echo :$?' | bash 2>/dev/null diff --git a/tests/queries/0_stateless/01659_h3_buffer_overflow.sql b/tests/queries/0_stateless/01659_h3_buffer_overflow.sql index b752059da48..f2d77641ec9 100644 --- a/tests/queries/0_stateless/01659_h3_buffer_overflow.sql +++ b/tests/queries/0_stateless/01659_h3_buffer_overflow.sql @@ -7,3 +7,4 @@ SELECT h3kRing(0xFFFFFFFFFFFFFF, 1000) FORMAT Null; SELECT h3GetBaseCell(0xFFFFFFFFFFFFFF) FORMAT Null; SELECT h3GetResolution(0xFFFFFFFFFFFFFF) FORMAT Null; SELECT h3kRing(0xFFFFFFFFFFFFFF, 10) FORMAT Null; +SELECT h3ToGeo(0xFFFFFFFFFFFFFF) FORMAT Null; diff --git a/tests/queries/0_stateless/01710_normal_projection_fix1.reference b/tests/queries/0_stateless/01710_normal_projection_fix1.reference new file mode 100644 index 00000000000..cd121fd3feb --- /dev/null +++ b/tests/queries/0_stateless/01710_normal_projection_fix1.reference @@ -0,0 +1,2 @@ +1 +1 1 diff --git a/tests/queries/0_stateless/01710_normal_projection_fix1.sql b/tests/queries/0_stateless/01710_normal_projection_fix1.sql new file mode 100644 index 00000000000..b4d7c6e8734 --- /dev/null +++ b/tests/queries/0_stateless/01710_normal_projection_fix1.sql @@ -0,0 +1,17 @@ +drop table if exists t; + +create table t (i int, j int) engine MergeTree order by i; + +insert into t values (1, 2); + +alter table t add projection x (select * order by j); + +insert into t values (1, 4); + +set allow_experimental_projection_optimization = 1, force_optimize_projection = 1; + +select i from t prewhere j = 4; + +SELECT j = 2, i FROM t PREWHERE j = 2; + +drop table t; diff --git a/tests/queries/0_stateless/01710_projection_in_index.reference b/tests/queries/0_stateless/01710_projection_in_index.reference new file mode 100644 index 00000000000..73c1df53be4 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_in_index.reference @@ -0,0 +1,2 @@ +1 1 1 +2 2 2 diff --git a/tests/queries/0_stateless/01710_projection_in_index.sql b/tests/queries/0_stateless/01710_projection_in_index.sql new file mode 100644 index 00000000000..2669d69dc9f --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_in_index.sql @@ -0,0 +1,11 @@ +drop table if exists t; + +create table t (i int, j int, k int, projection p (select * order by j)) engine MergeTree order by i settings index_granularity = 1; + +insert into t select number, number, number from numbers(10); + +set allow_experimental_projection_optimization = 1, max_rows_to_read = 3; + +select * from t where i < 5 and j in (1, 2); + +drop table t; diff --git a/tests/queries/0_stateless/01735_join_get_low_card_fix.reference b/tests/queries/0_stateless/01735_join_get_low_card_fix.reference index 0b20aead00e..a9e2f17562a 100644 --- a/tests/queries/0_stateless/01735_join_get_low_card_fix.reference +++ b/tests/queries/0_stateless/01735_join_get_low_card_fix.reference @@ -1 +1,6 @@ -yyy +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01735_join_get_low_card_fix.sql b/tests/queries/0_stateless/01735_join_get_low_card_fix.sql index bdc979bc11e..e2002112360 100644 --- a/tests/queries/0_stateless/01735_join_get_low_card_fix.sql +++ b/tests/queries/0_stateless/01735_join_get_low_card_fix.sql @@ -1,9 +1,14 @@ -drop table if exists join_tbl; +DROP TABLE IF EXISTS join_tbl; -create table join_tbl (`id` String, `name` String) engine Join(any, left, id); +CREATE TABLE join_tbl (`id` String, `name` String, lcname LowCardinality(String)) ENGINE = Join(any, left, id); -insert into join_tbl values ('xxx', 'yyy'); +INSERT INTO join_tbl VALUES ('xxx', 'yyy', 'yyy'); -select joinGet('join_tbl', 'name', toLowCardinality('xxx')); +SELECT joinGet('join_tbl', 'name', 'xxx') == 'yyy'; +SELECT joinGet('join_tbl', 'name', toLowCardinality('xxx')) == 'yyy'; +SELECT joinGet('join_tbl', 'name', toLowCardinality(materialize('xxx'))) == 'yyy'; +SELECT joinGet('join_tbl', 'lcname', 'xxx') == 'yyy'; +SELECT joinGet('join_tbl', 'lcname', toLowCardinality('xxx')) == 'yyy'; +SELECT joinGet('join_tbl', 'lcname', toLowCardinality(materialize('xxx'))) == 'yyy'; -drop table if exists join_tbl; +DROP TABLE IF EXISTS join_tbl; diff --git a/tests/queries/0_stateless/01736_null_as_default.reference b/tests/queries/0_stateless/01736_null_as_default.reference index baf83eb21d7..d201a9636ef 100644 --- a/tests/queries/0_stateless/01736_null_as_default.reference +++ b/tests/queries/0_stateless/01736_null_as_default.reference @@ -1,2 +1,5 @@ A \N +A +\N +\N diff --git a/tests/queries/0_stateless/01736_null_as_default.sql b/tests/queries/0_stateless/01736_null_as_default.sql index a00011b06d4..c897d035a50 100644 --- a/tests/queries/0_stateless/01736_null_as_default.sql +++ b/tests/queries/0_stateless/01736_null_as_default.sql @@ -2,4 +2,6 @@ drop table if exists test_enum; create table test_enum (c Nullable(Enum16('A' = 1, 'B' = 2))) engine Log; insert into test_enum values (1), (NULL); select * from test_enum; +select toString(c) from test_enum; +select toString('aaaa', NULL); drop table test_enum; diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference index 3bc986f4d2b..65b7bf54f7f 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference @@ -22,6 +22,7 @@ others different types -- prohibited different types -- conversion 0 +0 optimize_skip_unused_shards_limit 0 0 diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql index dbe76f146b0..ea7d526c039 100644 --- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql +++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql @@ -93,8 +93,6 @@ select 'errors'; -- optimize_skip_unused_shards does not support non-constants select * from dist_01756 where dummy in (select * from system.one); -- { serverError 507 } select * from dist_01756 where dummy in (toUInt8(0)); -- { serverError 507 } --- intHash64 does not accept string -select * from dist_01756 where dummy in ('0', '2'); -- { serverError 43 } -- NOT IN does not supported select * from dist_01756 where dummy not in (0, 2); -- { serverError 507 } @@ -126,6 +124,8 @@ select 'different types -- conversion'; create table dist_01756_column as system.one engine=Distributed(test_cluster_two_shards, system, one, dummy); select * from dist_01756_column where dummy in (0, '255'); select * from dist_01756_column where dummy in (0, '255foo'); -- { serverError 53 } +-- intHash64 does not accept string, but implicit conversion should be done +select * from dist_01756 where dummy in ('0', '2'); -- optimize_skip_unused_shards_limit select 'optimize_skip_unused_shards_limit'; diff --git a/tests/queries/0_stateless/01763_support_map_lowcardinality_type.reference b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.reference new file mode 100644 index 00000000000..8fdcdf3d8d5 --- /dev/null +++ b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.reference @@ -0,0 +1,2 @@ +b +{'1':1} 1 0 diff --git a/tests/queries/0_stateless/01763_support_map_lowcardinality_type.sql b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.sql new file mode 100644 index 00000000000..ccade153ca1 --- /dev/null +++ b/tests/queries/0_stateless/01763_support_map_lowcardinality_type.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS map_lc; +SET allow_experimental_map_type = 1; +CREATE TABLE map_lc +( + `kv` Map(LowCardinality(String), LowCardinality(String)) +) +ENGINE = Memory; + +INSERT INTO map_lc select map('a', 'b'); +SELECT kv['a'] FROM map_lc; +DROP TABLE map_lc; +SELECT map(toFixedString('1',1),1) AS m, m[toFixedString('1',1)],m[toFixedString('1',2)]; diff --git a/tests/queries/0_stateless/01849_geoToS2.reference b/tests/queries/0_stateless/01849_geoToS2.reference new file mode 100644 index 00000000000..08d76978791 --- /dev/null +++ b/tests/queries/0_stateless/01849_geoToS2.reference @@ -0,0 +1,42 @@ +Checking s2 index generation. +(-19.82614013111778,-41.19291183249827) 145638248314527735 +(-41.56412828256075,-16.777072680829264) 525948609053544771 +(40.76827373895363,2.853562616147134) 1723620528513494869 +(8.774109215249668,-3.271373776817451) 1913723177026859677 +(54.7243525263686,-19.21060843697615) 2414200527355011557 +(49.942295220850404,-18.63385558246411) 2446780491370050277 +(125.93850250627281,25.519361668875952) 3814912406305146887 +(51.39037388040172,29.368252361755857) 4590287096029015617 +(-165.90797307310456,54.20517787077579) 6317132534461540395 +(140.4288338857567,28.399754752831992) 6944470717484584123 +(170.14574840189854,7.606448822713084) 7280210779810726069 +(-170.42364912433663,-10.102188288980733) 8094352344009072653 +(-168.25897915006252,-38.27117023780382) 8295275405228382549 +(-46.71824230901231,5.161978621886426) 13251733624047612511 +(-64.36499761086276,-13.206225582160274) 10654167528317614093 +(-61.76193800786795,-24.969589107565216) 10670400906708524493 +(-79.24545956192031,-22.940848730236024) 10868726821406045765 +(74.00610377406458,-68.32123992734591) 12793606480989360605 +(10.610774069458158,-64.18410328814072) 13202270384266773975 +(-89.81096210929424,-57.01398354986957) 13606307743304496003 +(-19.82614,-41.19291) (-19.82614,-41.19291) ok +(-41.56413,-16.77707) (-41.56413,-16.77707) ok +(40.76827,2.85356) (40.76827,2.85356) ok +(8.77411,-3.27137) (8.77411,-3.27137) ok +(54.72435,-19.21061) (54.72435,-19.21061) ok +(49.94229,-18.63386) (49.94229,-18.63386) ok +(125.9385,25.51936) (125.9385,25.51936) ok +(51.39037,29.36825) (51.39037,29.36825) ok +(-165.90797,54.20518) (-165.90797,54.20518) ok +(140.42883,28.39976) (140.42883,28.39976) ok +(170.14575,7.60645) (170.14575,7.60645) ok +(-170.42365,-10.10219) (-170.42365,-10.10219) ok +(-168.25898,-38.27117) (-168.25898,-38.27117) ok +(5.16198,-46.71824) (5.16198,-46.71824) ok +(-64.365,-13.20623) (-64.365,-13.20623) ok +(-61.76194,-24.96959) (-61.76194,-24.96959) ok +(-79.24546,-22.94085) (-79.24546,-22.94085) ok +(74.0061,-68.32124) (74.0061,-68.32124) ok +(10.61077,-64.1841) (10.61077,-64.1841) ok +(-89.81096,-57.01398) (-89.81096,-57.01398) ok +4864204703484167331 diff --git a/tests/queries/0_stateless/01849_geoToS2.sql b/tests/queries/0_stateless/01849_geoToS2.sql new file mode 100644 index 00000000000..eb50fa81b8a --- /dev/null +++ b/tests/queries/0_stateless/01849_geoToS2.sql @@ -0,0 +1,50 @@ +DROP TABLE IF EXISTS s2_indexes; + +CREATE TABLE s2_indexes (s2_index UInt64, longitude Float64, latitude Float64) ENGINE = Memory; + +-- Random geo coordinates were generated using S2Testing::RandomPoint() method from s2 API. + +INSERT INTO s2_indexes VALUES (3814912406305146967, 125.938503, 25.519362); +INSERT INTO s2_indexes VALUES (10654167528317613967, -64.364998, -13.206226); +INSERT INTO s2_indexes VALUES (1913723177026859705, 8.774109, -3.271374); +INSERT INTO s2_indexes VALUES (13606307743304496111, -89.810962, -57.013984); +INSERT INTO s2_indexes VALUES (8094352344009072761,-170.423649, -10.102188); +INSERT INTO s2_indexes VALUES (2414200527355011659, 54.724353, -19.210608); +INSERT INTO s2_indexes VALUES (4590287096029015693, 51.390374, 29.368252); +INSERT INTO s2_indexes VALUES (10173921221664598133, 5.161979, -46.718242); +INSERT INTO s2_indexes VALUES (525948609053546189, -41.564128, -16.777073); +INSERT INTO s2_indexes VALUES (2446780491369950853, 49.94229, -18.633856); +INSERT INTO s2_indexes VALUES (1723620528513492581, 40.768274, 2.853563); +INSERT INTO s2_indexes VALUES (8295275405228383207, -168.258979, -38.271170); +INSERT INTO s2_indexes VALUES (7280210779810727639, 170.145748, 7.606449); +INSERT INTO s2_indexes VALUES (10670400906708524495, -61.761938, -24.969589); +INSERT INTO s2_indexes VALUES (10868726821406046149, -79.245460, -22.940849); +INSERT INTO s2_indexes VALUES (13202270384266773545, 10.610774, -64.184103); +INSERT INTO s2_indexes VALUES (145638248314527629, -19.826140, -41.192912); +INSERT INTO s2_indexes VALUES (12793606480989360601, 74.006104, -68.321240); +INSERT INTO s2_indexes VALUES (6317132534461540391, -165.907973, 54.205178); +INSERT INTO s2_indexes VALUES (6944470717485986643, 140.428834, 28.399755); + +SELECT 'Checking s2 index generation.'; + +SELECT s2ToGeo(s2_index), geoToS2(longitude, latitude) FROM s2_indexes ORDER BY s2_index; + +SELECT first, second, result FROM ( + SELECT + s2ToGeo(geoToS2(longitude, latitude)) AS output_geo, + tuple(roundBankers(longitude, 5), roundBankers(latitude, 5)) AS first, + tuple(roundBankers(output_geo.1, 5), roundBankers(output_geo.2, 5)) AS second, + if(first = second, 'ok', 'fail') AS result + FROM s2_indexes + ORDER BY s2_index + ); + +SELECT s2ToGeo(toUInt64(-1)); -- { serverError 36 } +SELECT s2ToGeo(nan); -- { serverError 43 } +SELECT geoToS2(toFloat64(toUInt64(-1)), toFloat64(toUInt64(-1))); +SELECT geoToS2(nan, nan); -- { serverError 43 } +SELECT geoToS2(-inf, 1.1754943508222875e-38); -- { serverError 43 } + + + +DROP TABLE IF EXISTS s2_indexes; diff --git a/tests/queries/0_stateless/01851_s2_to_geo.reference b/tests/queries/0_stateless/01851_s2_to_geo.reference new file mode 100644 index 00000000000..75b182ebd1f --- /dev/null +++ b/tests/queries/0_stateless/01851_s2_to_geo.reference @@ -0,0 +1,2 @@ +(55.779227241803866,37.63098046233757) +(55.76324102676383,37.660183005258276) diff --git a/tests/queries/0_stateless/01851_s2_to_geo.sql b/tests/queries/0_stateless/01851_s2_to_geo.sql new file mode 100644 index 00000000000..76e4b2a5346 --- /dev/null +++ b/tests/queries/0_stateless/01851_s2_to_geo.sql @@ -0,0 +1,2 @@ +select s2ToGeo(4573520603753570041); +select s2ToGeo(4573517609713934091); diff --git a/tests/queries/0_stateless/01852_s2_get_neighbors.reference b/tests/queries/0_stateless/01852_s2_get_neighbors.reference new file mode 100644 index 00000000000..3182a1c5e00 --- /dev/null +++ b/tests/queries/0_stateless/01852_s2_get_neighbors.reference @@ -0,0 +1 @@ +[5074766987100422144,5074766712222515200,5074767536856236032,5074767261978329088] diff --git a/tests/queries/0_stateless/01852_s2_get_neighbours.reference b/tests/queries/0_stateless/01852_s2_get_neighbours.reference new file mode 100644 index 00000000000..3182a1c5e00 --- /dev/null +++ b/tests/queries/0_stateless/01852_s2_get_neighbours.reference @@ -0,0 +1 @@ +[5074766987100422144,5074766712222515200,5074767536856236032,5074767261978329088] diff --git a/tests/queries/0_stateless/01852_s2_get_neighbours.sql b/tests/queries/0_stateless/01852_s2_get_neighbours.sql new file mode 100644 index 00000000000..8163f827697 --- /dev/null +++ b/tests/queries/0_stateless/01852_s2_get_neighbours.sql @@ -0,0 +1 @@ +select s2GetNeighbors(5074766849661468672); diff --git a/tests/queries/0_stateless/01853_s2_cells_intersect.reference b/tests/queries/0_stateless/01853_s2_cells_intersect.reference new file mode 100644 index 00000000000..b261da18d51 --- /dev/null +++ b/tests/queries/0_stateless/01853_s2_cells_intersect.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/01853_s2_cells_intersect.sql b/tests/queries/0_stateless/01853_s2_cells_intersect.sql new file mode 100644 index 00000000000..2a033a67d58 --- /dev/null +++ b/tests/queries/0_stateless/01853_s2_cells_intersect.sql @@ -0,0 +1,5 @@ +select s2CellsIntersect(9926595209846587392, 9926594385212866560); +select s2CellsIntersect(9926595209846587392, 9937259648002293760); + + +SELECT s2CellsIntersect(9926595209846587392, 9223372036854775806); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01854_s2_cap_contains.reference b/tests/queries/0_stateless/01854_s2_cap_contains.reference new file mode 100644 index 00000000000..16db301bb51 --- /dev/null +++ b/tests/queries/0_stateless/01854_s2_cap_contains.reference @@ -0,0 +1,3 @@ +1 +0 +1 diff --git a/tests/queries/0_stateless/01854_s2_cap_contains.sql b/tests/queries/0_stateless/01854_s2_cap_contains.sql new file mode 100644 index 00000000000..1a8d2548352 --- /dev/null +++ b/tests/queries/0_stateless/01854_s2_cap_contains.sql @@ -0,0 +1,11 @@ +select s2CapContains(1157339245694594829, 1.0, 1157347770437378819); +select s2CapContains(1157339245694594829, 1.0, 1152921504606846977); +select s2CapContains(1157339245694594829, 3.14, 1157339245694594829); + +select s2CapContains(nan, 3.14, 1157339245694594829); -- { serverError 43 } +select s2CapContains(1157339245694594829, nan, 1157339245694594829); -- { serverError 43 } +select s2CapContains(1157339245694594829, 3.14, nan); -- { serverError 43 } + + +select s2CapContains(toUInt64(-1), -1.0, toUInt64(-1)); -- { serverError 36 } +select s2CapContains(toUInt64(-1), 9999.9999, toUInt64(-1)); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01854_s2_cap_union.reference b/tests/queries/0_stateless/01854_s2_cap_union.reference new file mode 100644 index 00000000000..8be71d7ba28 --- /dev/null +++ b/tests/queries/0_stateless/01854_s2_cap_union.reference @@ -0,0 +1,3 @@ +(4534655147792050737,60.2088283994957) +(1157339245694594829,-57.29577951308232) +(1157339245694594829,180) diff --git a/tests/queries/0_stateless/01854_s2_cap_union.sql b/tests/queries/0_stateless/01854_s2_cap_union.sql new file mode 100644 index 00000000000..921a00ac663 --- /dev/null +++ b/tests/queries/0_stateless/01854_s2_cap_union.sql @@ -0,0 +1,9 @@ +select s2CapUnion(3814912406305146967, 1.0, 1157347770437378819, 1.0); +select s2CapUnion(1157339245694594829, -1.0, 1152921504606846977, -1.0); +select s2CapUnion(1157339245694594829, toFloat64(toUInt64(-1)), 1157339245694594829, toFloat64(toUInt64(-1))); + + +select s2CapUnion(nan, 3.14, 1157339245694594829, 3.14); -- { serverError 43 } +select s2CapUnion(1157339245694594829, nan, 1157339245694594829, 3.14); -- { serverError 43 } +select s2CapUnion(1157339245694594829, 3.14, nan, 3.14); -- { serverError 43 } +select s2CapUnion(1157339245694594829, 3.14, 1157339245694594829, nan); -- { serverError 43 } diff --git a/tests/queries/0_stateless/01889_sqlite_read_write.reference b/tests/queries/0_stateless/01889_sqlite_read_write.reference new file mode 100644 index 00000000000..e979b5816c5 --- /dev/null +++ b/tests/queries/0_stateless/01889_sqlite_read_write.reference @@ -0,0 +1,42 @@ +create database engine +show database tables: +table1 +table2 +table3 +table4 +table5 +show creare table: +CREATE TABLE SQLite.table1\n(\n `col1` Nullable(String),\n `col2` Nullable(Int16)\n)\nENGINE = SQLite +CREATE TABLE SQLite.table2\n(\n `col1` Nullable(Int32),\n `col2` Nullable(String)\n)\nENGINE = SQLite +describe table: +col1 Nullable(String) +col2 Nullable(Int16) +col1 Nullable(Int32) +col2 Nullable(String) +select *: +line1 1 +line2 2 +line3 3 +1 text1 +2 text2 +3 text3 +test types +CREATE TABLE SQLite.table4\n(\n `a` Nullable(Int32),\n `b` Nullable(Int32),\n `c` Nullable(Int8),\n `d` Nullable(Int16),\n `e` Nullable(Int32),\n `bigint` Nullable(String),\n `int2` Nullable(String),\n `int8` Nullable(String)\n)\nENGINE = SQLite +CREATE TABLE SQLite.table5\n(\n `a` Nullable(String),\n `b` Nullable(String),\n `c` Nullable(Float64),\n `d` Nullable(Float64),\n `e` Nullable(Float64),\n `f` Nullable(Float32)\n)\nENGINE = SQLite +create table engine with table3 +CREATE TABLE default.sqlite_table3\n(\n `col1` String,\n `col2` Int32\n)\nENGINE = SQLite + 1 +not a null 2 + 3 + 4 +line6 6 + 7 +test table function +line1 1 +line2 2 +line3 3 +line4 4 +test path in clickhouse-local +line1 1 +line2 2 +line3 3 diff --git a/tests/queries/0_stateless/01889_sqlite_read_write.sh b/tests/queries/0_stateless/01889_sqlite_read_write.sh new file mode 100755 index 00000000000..73b106e9eb4 --- /dev/null +++ b/tests/queries/0_stateless/01889_sqlite_read_write.sh @@ -0,0 +1,89 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# See 01658_read_file_to_string_column.sh +user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') + +mkdir -p ${user_files_path}/ +chmod 777 ${user_files_path} +DB_PATH=${user_files_path}/db1 + + +sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table1' +sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table2' +sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table3' +sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table4' +sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table5' + +sqlite3 ${DB_PATH} 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 ${DB_PATH} 'CREATE TABLE table2 (col1 int, col2 text);' + +chmod ugo+w ${DB_PATH} + +sqlite3 ${DB_PATH} "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" +sqlite3 ${DB_PATH} "INSERT INTO table2 VALUES (1, 'text1'), (2, 'text2'), (3, 'text3')" + +sqlite3 ${DB_PATH} 'CREATE TABLE table3 (col1 text, col2 int);' +sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 1)' +sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('not a null', 2)" +sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 3)' +sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('', 4)" + +sqlite3 ${DB_PATH} 'CREATE TABLE table4 (a int, b integer, c tinyint, d smallint, e mediumint, bigint, int2, int8)' +sqlite3 ${DB_PATH} 'CREATE TABLE table5 (a character(20), b varchar(10), c real, d double, e double precision, f float)' + + +${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' + +${CLICKHOUSE_CLIENT} --query="select 'create database engine'"; +${CLICKHOUSE_CLIENT} --query="CREATE DATABASE sqlite_database ENGINE = SQLite('${DB_PATH}')" + +${CLICKHOUSE_CLIENT} --query="select 'show database tables:'"; +${CLICKHOUSE_CLIENT} --query='SHOW TABLES FROM sqlite_database;' + +${CLICKHOUSE_CLIENT} --query="select 'show creare table:'"; +${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table1;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table2;' | sed -r 's/(.*SQLite)(.*)/\1/' + +${CLICKHOUSE_CLIENT} --query="select 'describe table:'"; +${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table1;' +${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table2;' + +${CLICKHOUSE_CLIENT} --query="select 'select *:'"; +${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table1 ORDER BY col2' +${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table2 ORDER BY col1;' + +${CLICKHOUSE_CLIENT} --query="select 'test types'"; +${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table4;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table5;' | sed -r 's/(.*SQLite)(.*)/\1/' + +${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' + + +${CLICKHOUSE_CLIENT} --query="select 'create table engine with table3'"; +${CLICKHOUSE_CLIENT} --query='DROP TABLE IF EXISTS sqlite_table3' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE sqlite_table3 (col1 String, col2 Int32) ENGINE = SQLite('${DB_PATH}', 'table3')" + +${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_table3;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="INSERT INTO sqlite_table3 VALUES ('line6', 6);" +${CLICKHOUSE_CLIENT} --query="INSERT INTO sqlite_table3 VALUES (NULL, 7);" + +${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_table3 ORDER BY col2' + + +${CLICKHOUSE_CLIENT} --query="select 'test table function'"; +${CLICKHOUSE_CLIENT} --query="INSERT INTO TABLE FUNCTION sqlite('${DB_PATH}', 'table1') SELECT 'line4', 4" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM sqlite('${DB_PATH}', 'table1') ORDER BY col2" + + +sqlite3 $CUR_DIR/db2 'DROP TABLE IF EXISTS table1' +sqlite3 $CUR_DIR/db2 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 $CUR_DIR/db2 "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" + +${CLICKHOUSE_CLIENT} --query="select 'test path in clickhouse-local'"; +${CLICKHOUSE_LOCAL} --query="SELECT * FROM sqlite('$CUR_DIR/db2', 'table1') ORDER BY col2" + +rm -r ${DB_PATH} diff --git a/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.reference b/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.reference new file mode 100644 index 00000000000..b70a1cb7c75 --- /dev/null +++ b/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.reference @@ -0,0 +1,3 @@ +8 +4 +4 diff --git a/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.sql b/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.sql new file mode 100644 index 00000000000..7aa1b0112a6 --- /dev/null +++ b/tests/queries/0_stateless/01903_correct_block_size_prediction_with_default.sql @@ -0,0 +1,11 @@ +CREATE TABLE test_extract(str String, arr Array(Array(String)) ALIAS extractAllGroupsHorizontal(str, '\\W(\\w+)=("[^"]*?"|[^",}]*)')) ENGINE=MergeTree() PARTITION BY tuple() ORDER BY tuple(); + +INSERT INTO test_extract (str) WITH range(8) as range_arr, arrayMap(x-> concat(toString(x),'Id'), range_arr) as key, arrayMap(x -> rand() % 8, range_arr) as val, arrayStringConcat(arrayMap((x,y) -> concat(x,'=',toString(y)), key, val),',') as str SELECT str FROM numbers(500000); + +ALTER TABLE test_extract ADD COLUMN `15Id` Nullable(UInt16) DEFAULT toUInt16OrNull(arrayFirst((v, k) -> (k = '4Id'), arr[2], arr[1])); + +SELECT uniq(15Id) FROM test_extract SETTINGS max_threads=1, max_memory_usage=100000000; + +SELECT uniq(15Id) FROM test_extract PREWHERE 15Id < 4 SETTINGS max_threads=1, max_memory_usage=100000000; + +SELECT uniq(15Id) FROM test_extract WHERE 15Id < 4 SETTINGS max_threads=1, max_memory_usage=100000000; diff --git a/tests/queries/0_stateless/01906_h3_to_geo.reference b/tests/queries/0_stateless/01906_h3_to_geo.reference new file mode 100644 index 00000000000..93e8600576c --- /dev/null +++ b/tests/queries/0_stateless/01906_h3_to_geo.reference @@ -0,0 +1,32 @@ +(-173.6412167681162,-14.130272474941535) +(59.48137613600854,58.020407687755686) +(172.68095885060296,-83.6576608516349) +(-94.46556851304558,-69.1999982492279) +(-8.188263637093279,-55.856179102736284) +(77.25594891852249,47.39278564360122) +(135.11348004704536,36.60778126579667) +(39.28534828967223,49.07710003066973) +(124.71163478198051,-27.481172161567258) +(-147.4887686066785,76.73237945824442) +(86.63291906118863,-25.52526285188784) +(23.27751790712118,13.126101362212724) +(-70.40163237204142,-63.12562536833242) +(15.642428355535966,40.285813505163574) +(-76.53411447979884,54.5560449693637) +(8.19906334981474,67.69370966550179) +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok +ok diff --git a/tests/queries/0_stateless/01906_h3_to_geo.sql b/tests/queries/0_stateless/01906_h3_to_geo.sql new file mode 100644 index 00000000000..aa6ecca1754 --- /dev/null +++ b/tests/queries/0_stateless/01906_h3_to_geo.sql @@ -0,0 +1,61 @@ +DROP TABLE IF EXISTS h3_indexes; + +CREATE TABLE h3_indexes (h3_index UInt64) ENGINE = Memory; + +-- Random geo coordinates were generated using the H3 tool: https://github.com/ClickHouse-Extras/h3/blob/master/src/apps/testapps/mkRandGeo.c at various resolutions from 0 to 15. +-- Corresponding H3 index values were in turn generated with those geo coordinates using `geoToH3(lon, lat, res)` ClickHouse function for the following test. + +INSERT INTO h3_indexes VALUES (579205133326352383); +INSERT INTO h3_indexes VALUES (581263419093549055); +INSERT INTO h3_indexes VALUES (589753847883235327); +INSERT INTO h3_indexes VALUES (594082350283882495); +INSERT INTO h3_indexes VALUES (598372386957426687); +INSERT INTO h3_indexes VALUES (599542359671177215); +INSERT INTO h3_indexes VALUES (604296355086598143); +INSERT INTO h3_indexes VALUES (608785214872748031); +INSERT INTO h3_indexes VALUES (615732192485572607); +INSERT INTO h3_indexes VALUES (617056794467368959); +INSERT INTO h3_indexes VALUES (624586477873168383); +INSERT INTO h3_indexes VALUES (627882919484481535); +INSERT INTO h3_indexes VALUES (634600058503392255); +INSERT INTO h3_indexes VALUES (635544851677385791); +INSERT INTO h3_indexes VALUES (639763125756281263); +INSERT INTO h3_indexes VALUES (644178757620501158); + + +SELECT h3ToGeo(h3_index) FROM h3_indexes ORDER BY h3_index; + +DROP TABLE h3_indexes; + +DROP TABLE IF EXISTS h3_geo; + +-- compare if the results of h3ToGeo and geoToH3 are the same + +CREATE TABLE h3_geo(lat Float64, lon Float64, res UInt8) ENGINE = Memory; + +INSERT INTO h3_geo VALUES (-173.6412167681162, -14.130272474941535, 0); +INSERT INTO h3_geo VALUES (59.48137613600854, 58.020407687755686, 1); +INSERT INTO h3_geo VALUES (172.68095885060296, -83.6576608516349, 2); +INSERT INTO h3_geo VALUES (-94.46556851304558, -69.1999982492279, 3); +INSERT INTO h3_geo VALUES (-8.188263637093279, -55.856179102736284, 4); +INSERT INTO h3_geo VALUES (77.25594891852249, 47.39278564360122, 5); +INSERT INTO h3_geo VALUES (135.11348004704536, 36.60778126579667, 6); +INSERT INTO h3_geo VALUES (39.28534828967223, 49.07710003066973, 7); +INSERT INTO h3_geo VALUES (124.71163478198051, -27.481172161567258, 8); +INSERT INTO h3_geo VALUES (-147.4887686066785, 76.73237945824442, 9); +INSERT INTO h3_geo VALUES (86.63291906118863, -25.52526285188784, 10); +INSERT INTO h3_geo VALUES (23.27751790712118, 13.126101362212724, 11); +INSERT INTO h3_geo VALUES (-70.40163237204142, -63.12562536833242, 12); +INSERT INTO h3_geo VALUES (15.642428355535966, 40.285813505163574, 13); +INSERT INTO h3_geo VALUES (-76.53411447979884, 54.5560449693637, 14); +INSERT INTO h3_geo VALUES (8.19906334981474, 67.69370966550179, 15); + +SELECT result FROM ( + SELECT + (lat, lon) AS input_geo, + h3ToGeo(geoToH3(lat, lon, res)) AS output_geo, + if(input_geo = output_geo, 'ok', 'fail') AS result + FROM h3_geo +); + +DROP TABLE h3_geo; diff --git a/tests/queries/0_stateless/01926_bin_unbin.reference b/tests/queries/0_stateless/01926_bin_unbin.reference index f84a858e449..731d0223bb9 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.reference +++ b/tests/queries/0_stateless/01926_bin_unbin.reference @@ -33,3 +33,7 @@ 1 1 1 +1 +1 +2D000000000000000A +001011010000000000000000000000000000000000000000000000000000000000001010 diff --git a/tests/queries/0_stateless/01926_bin_unbin.sql b/tests/queries/0_stateless/01926_bin_unbin.sql index 555770d09c6..e112f8bd8a4 100644 --- a/tests/queries/0_stateless/01926_bin_unbin.sql +++ b/tests/queries/0_stateless/01926_bin_unbin.sql @@ -37,3 +37,9 @@ select bin(unbin('0')) == '00000000'; select hex('') == bin(''); select unhex('') == unbin(''); select unhex('0') == unbin('0'); + +-- hex and bin support AggregateFunction +select hex(sumState(number)) == hex(toString(sumState(number))) from numbers(10); +select hex(avgState(number)) == hex(toString(avgState(number))) from numbers(99); +select hex(avgState(number)) from numbers(10); +select bin(avgState(number)) from numbers(10); diff --git a/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.reference b/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.reference new file mode 100644 index 00000000000..b856b079327 --- /dev/null +++ b/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.reference @@ -0,0 +1,132 @@ +-- { echoOn } + +-- Int8, Int8 +select _shard_num, * from remote('127.{1..4}', view(select toInt8(id) id from data), toInt8(id)) where id in (0, 1, 0x7f) order by _shard_num, id; +1 0 +1 0 +1 0 +1 0 +2 1 +4 127 +-- Int8, UInt8 +select _shard_num, * from remote('127.{1..4}', view(select toInt8(id) id from data), toUInt8(id)) where id in (0, 1, 0x7f) order by _shard_num, id; +1 0 +1 0 +1 0 +1 0 +2 1 +4 127 +-- UInt8, UInt8 +select _shard_num, * from remote('127.{1..4}', view(select toUInt8(id) id from data), toUInt8(id)) where id in (0, 1, 0x7f, 0x80, 0xff) order by _shard_num, id; +1 0 +1 0 +1 0 +1 0 +1 128 +2 1 +4 127 +4 255 +4 255 +4 255 +4 255 +4 255 +4 255 +4 255 +-- UInt8, Int8 +select _shard_num, * from remote('127.{1..4}', view(select toUInt8(id) id from data), toInt8(id)) where id in (0, 1, 0x7f, 0x80, 0xff) order by _shard_num, id; +1 0 +1 0 +1 0 +1 0 +2 1 +4 127 +-- Int16, Int16 +select _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toInt16(id)) where id in (0, 1, 0x7fff) order by _shard_num, id; +1 0 +1 0 +1 0 +2 1 +4 32767 +-- Int16, UInt16 +select _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toUInt16(id)) where id in (0, 1, 0x7fff) order by _shard_num, id; +1 0 +1 0 +1 0 +2 1 +4 32767 +-- UInt16, UInt16 +select _shard_num, * from remote('127.{1..4}', view(select toUInt16(id) id from data), toUInt16(id)) where id in (0, 1, 0x7fff, 0x8000, 0xffff) order by _shard_num, id; +1 0 +1 0 +1 0 +1 32768 +2 1 +4 32767 +4 65535 +4 65535 +4 65535 +4 65535 +4 65535 +-- UInt16, Int16 +select _shard_num, * from remote('127.{1..4}', view(select toUInt16(id) id from data), toInt16(id)) where id in (0, 1, 0x7fff, 0x8000, 0xffff) order by _shard_num, id; +1 0 +1 0 +1 0 +2 1 +4 32767 +-- Int32, Int32 +select _shard_num, * from remote('127.{1..4}', view(select toInt32(id) id from data), toInt32(id)) where id in (0, 1, 0x7fffffff) order by _shard_num, id; +1 0 +1 0 +2 1 +4 2147483647 +-- Int32, UInt32 +select _shard_num, * from remote('127.{1..4}', view(select toInt32(id) id from data), toUInt32(id)) where id in (0, 1, 0x7fffffff) order by _shard_num, id; +1 0 +1 0 +2 1 +4 2147483647 +-- UInt32, UInt32 +select _shard_num, * from remote('127.{1..4}', view(select toUInt32(id) id from data), toUInt32(id)) where id in (0, 1, 0x7fffffff, 0x80000000, 0xffffffff) order by _shard_num, id; +1 0 +1 0 +1 2147483648 +2 1 +4 2147483647 +4 4294967295 +4 4294967295 +4 4294967295 +-- UInt32, Int32 +select _shard_num, * from remote('127.{1..4}', view(select toUInt32(id) id from data), toInt32(id)) where id in (0, 1, 0x7fffffff, 0x80000000, 0xffffffff) order by _shard_num, id; +1 0 +1 0 +2 1 +4 2147483647 +-- Int64, Int64 +select _shard_num, * from remote('127.{1..4}', view(select toInt64(id) id from data), toInt64(id)) where id in (0, 1, 0x7fffffffffffffff) order by _shard_num, id; +1 0 +2 1 +4 9223372036854775807 +-- Int64, UInt64 +select _shard_num, * from remote('127.{1..4}', view(select toInt64(id) id from data), toUInt64(id)) where id in (0, 1, 0x7fffffffffffffff) order by _shard_num, id; +1 0 +2 1 +4 9223372036854775807 +-- UInt64, UInt64 +select _shard_num, * from remote('127.{1..4}', view(select toUInt64(id) id from data), toUInt64(id)) where id in (0, 1, 0x7fffffffffffffff, 0x8000000000000000, 0xffffffffffffffff) order by _shard_num, id; +1 0 +1 9223372036854775808 +2 1 +4 9223372036854775807 +4 18446744073709551615 +-- UInt64, Int64 +select _shard_num, * from remote('127.{1..4}', view(select toUInt64(id) id from data), toInt64(id)) where id in (0, 1, 0x7fffffffffffffff, 0x8000000000000000, 0xffffffffffffffff) order by _shard_num, id; +1 0 +2 1 +4 9223372036854775807 +-- modulo(Int8) +select distinct _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toInt8(id)%255) where id in (-1) order by _shard_num, id; +4 -1 +-- modulo(UInt8) +select distinct _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toUInt8(id)%255) where id in (-1) order by _shard_num, id; +1 -1 diff --git a/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.sql b/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.sql new file mode 100644 index 00000000000..7e53c0c2db7 --- /dev/null +++ b/tests/queries/0_stateless/01930_optimize_skip_unused_shards_rewrite_in.sql @@ -0,0 +1,63 @@ +set optimize_skip_unused_shards=1; +set force_optimize_skip_unused_shards=2; + +create temporary table data (id UInt64) engine=Memory() as with [ + 0, + 1, + 0x7f, 0x80, 0xff, + 0x7fff, 0x8000, 0xffff, + 0x7fffffff, 0x80000000, 0xffffffff, + 0x7fffffffffffffff, 0x8000000000000000, 0xffffffffffffffff +] as values select arrayJoin(values) id; + +-- { echoOn } + +-- Int8, Int8 +select _shard_num, * from remote('127.{1..4}', view(select toInt8(id) id from data), toInt8(id)) where id in (0, 1, 0x7f) order by _shard_num, id; +-- Int8, UInt8 +select _shard_num, * from remote('127.{1..4}', view(select toInt8(id) id from data), toUInt8(id)) where id in (0, 1, 0x7f) order by _shard_num, id; +-- UInt8, UInt8 +select _shard_num, * from remote('127.{1..4}', view(select toUInt8(id) id from data), toUInt8(id)) where id in (0, 1, 0x7f, 0x80, 0xff) order by _shard_num, id; +-- UInt8, Int8 +select _shard_num, * from remote('127.{1..4}', view(select toUInt8(id) id from data), toInt8(id)) where id in (0, 1, 0x7f, 0x80, 0xff) order by _shard_num, id; + +-- Int16, Int16 +select _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toInt16(id)) where id in (0, 1, 0x7fff) order by _shard_num, id; +-- Int16, UInt16 +select _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toUInt16(id)) where id in (0, 1, 0x7fff) order by _shard_num, id; +-- UInt16, UInt16 +select _shard_num, * from remote('127.{1..4}', view(select toUInt16(id) id from data), toUInt16(id)) where id in (0, 1, 0x7fff, 0x8000, 0xffff) order by _shard_num, id; +-- UInt16, Int16 +select _shard_num, * from remote('127.{1..4}', view(select toUInt16(id) id from data), toInt16(id)) where id in (0, 1, 0x7fff, 0x8000, 0xffff) order by _shard_num, id; + +-- Int32, Int32 +select _shard_num, * from remote('127.{1..4}', view(select toInt32(id) id from data), toInt32(id)) where id in (0, 1, 0x7fffffff) order by _shard_num, id; +-- Int32, UInt32 +select _shard_num, * from remote('127.{1..4}', view(select toInt32(id) id from data), toUInt32(id)) where id in (0, 1, 0x7fffffff) order by _shard_num, id; +-- UInt32, UInt32 +select _shard_num, * from remote('127.{1..4}', view(select toUInt32(id) id from data), toUInt32(id)) where id in (0, 1, 0x7fffffff, 0x80000000, 0xffffffff) order by _shard_num, id; +-- UInt32, Int32 +select _shard_num, * from remote('127.{1..4}', view(select toUInt32(id) id from data), toInt32(id)) where id in (0, 1, 0x7fffffff, 0x80000000, 0xffffffff) order by _shard_num, id; + +-- Int64, Int64 +select _shard_num, * from remote('127.{1..4}', view(select toInt64(id) id from data), toInt64(id)) where id in (0, 1, 0x7fffffffffffffff) order by _shard_num, id; +-- Int64, UInt64 +select _shard_num, * from remote('127.{1..4}', view(select toInt64(id) id from data), toUInt64(id)) where id in (0, 1, 0x7fffffffffffffff) order by _shard_num, id; +-- UInt64, UInt64 +select _shard_num, * from remote('127.{1..4}', view(select toUInt64(id) id from data), toUInt64(id)) where id in (0, 1, 0x7fffffffffffffff, 0x8000000000000000, 0xffffffffffffffff) order by _shard_num, id; +-- UInt64, Int64 +select _shard_num, * from remote('127.{1..4}', view(select toUInt64(id) id from data), toInt64(id)) where id in (0, 1, 0x7fffffffffffffff, 0x8000000000000000, 0xffffffffffffffff) order by _shard_num, id; + +-- modulo(Int8) +select distinct _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toInt8(id)%255) where id in (-1) order by _shard_num, id; +-- modulo(UInt8) +select distinct _shard_num, * from remote('127.{1..4}', view(select toInt16(id) id from data), toUInt8(id)%255) where id in (-1) order by _shard_num, id; + +-- { echoOff } + +-- those two had been reported initially by amosbird: +-- (the problem is that murmurHash3_32() returns different value to toInt64(1) and toUInt64(1)) +---- error for local node +select * from remote('127.{1..4}', view(select number id from numbers(0)), bitAnd(murmurHash3_32(id), 2147483647)) where id in (2, 3); +---- error for remote node +select * from remote('127.{1..8}', view(select number id from numbers(0)), bitAnd(murmurHash3_32(id), 2147483647)) where id in (2, 3); diff --git a/tests/queries/0_stateless/01932_global_in_function.reference b/tests/queries/0_stateless/01932_global_in_function.reference new file mode 100644 index 00000000000..44e0be8e356 --- /dev/null +++ b/tests/queries/0_stateless/01932_global_in_function.reference @@ -0,0 +1,4 @@ +0 +0 +0 +0 diff --git a/tests/queries/0_stateless/01932_global_in_function.sql b/tests/queries/0_stateless/01932_global_in_function.sql new file mode 100644 index 00000000000..467bf6c3495 --- /dev/null +++ b/tests/queries/0_stateless/01932_global_in_function.sql @@ -0,0 +1,2 @@ +select number from cluster(test_cluster_two_shards_localhost, numbers(1)) where number global in tuple(0, 1, 2, 3); +select number from cluster(test_cluster_two_shards_localhost, numbers(1)) where number global in array(0, 1, 2, 3); diff --git a/tests/queries/0_stateless/01932_remote_sharding_key_column.reference b/tests/queries/0_stateless/01932_remote_sharding_key_column.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01932_remote_sharding_key_column.sql b/tests/queries/0_stateless/01932_remote_sharding_key_column.sql new file mode 100644 index 00000000000..ded2f187821 --- /dev/null +++ b/tests/queries/0_stateless/01932_remote_sharding_key_column.sql @@ -0,0 +1,15 @@ +-- regression test for the following query: +-- +-- select * from remote('127.1', system.one, dummy) +-- +-- that produce the following error before: +-- +-- Unknown column: dummy, there are only columns . +-- +-- NOTE: that wrapping column into any function works before. +select * from remote('127.1', system.one, dummy) format Null; +select * from remote('127.1', system.one, identity(dummy)) format Null; +select * from remote('127.1', view(select * from system.one), identity(dummy)) format Null; +select * from remote('127.{1,2}', view(select * from system.one), identity(dummy)) format Null; +select * from remote('127.1', view(select * from system.one), dummy) format Null; +select * from remote('127.{1,2}', view(select * from system.one), dummy) format Null; diff --git a/tests/queries/0_stateless/01940_custom_tld_sharding_key.reference b/tests/queries/0_stateless/01940_custom_tld_sharding_key.reference new file mode 100644 index 00000000000..0989a305613 --- /dev/null +++ b/tests/queries/0_stateless/01940_custom_tld_sharding_key.reference @@ -0,0 +1 @@ +foo.com diff --git a/tests/queries/0_stateless/01940_custom_tld_sharding_key.sql b/tests/queries/0_stateless/01940_custom_tld_sharding_key.sql new file mode 100644 index 00000000000..5d38cfb18dc --- /dev/null +++ b/tests/queries/0_stateless/01940_custom_tld_sharding_key.sql @@ -0,0 +1,2 @@ +select * from remote('127.{1,2}', view(select 'foo.com' key), cityHash64(key)) where key = cutToFirstSignificantSubdomainCustom('foo.com', 'public_suffix_list') settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1; +select * from remote('127.{1,2}', view(select 'foo.com' key), cityHash64(key)) where key = cutToFirstSignificantSubdomainCustom('bar.com', 'public_suffix_list') settings optimize_skip_unused_shards=1, force_optimize_skip_unused_shards=1; diff --git a/tests/queries/0_stateless/01940_point_in_polygon_ubsan.reference b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.reference new file mode 100644 index 00000000000..573541ac970 --- /dev/null +++ b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/01940_point_in_polygon_ubsan.sql b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.sql new file mode 100644 index 00000000000..d011725691f --- /dev/null +++ b/tests/queries/0_stateless/01940_point_in_polygon_ubsan.sql @@ -0,0 +1,2 @@ +SET validate_polygons = 0; +SELECT pointInPolygon((-inf, 1023), [(10.000100135803223, 10000000000.), (inf, 0.9998999834060669), (1.1920928955078125e-7, 100.0000991821289), (1.000100016593933, 100.0000991821289)]); diff --git a/tests/queries/0_stateless/01940_totimezone_operator_monotonicity.reference b/tests/queries/0_stateless/01940_totimezone_operator_monotonicity.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/01940_totimezone_operator_monotonicity.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/01940_totimezone_operator_monotonicity.sql b/tests/queries/0_stateless/01940_totimezone_operator_monotonicity.sql new file mode 100644 index 00000000000..b8065947ead --- /dev/null +++ b/tests/queries/0_stateless/01940_totimezone_operator_monotonicity.sql @@ -0,0 +1,6 @@ +DROP TABLE IF EXISTS totimezone_op_mono; +CREATE TABLE totimezone_op_mono(i int, tz String, create_time DateTime) ENGINE MergeTree PARTITION BY toDate(create_time) ORDER BY i; +INSERT INTO totimezone_op_mono VALUES (1, 'UTC', toDateTime('2020-09-01 00:00:00', 'UTC')), (2, 'UTC', toDateTime('2020-09-02 00:00:00', 'UTC')); +SET max_rows_to_read = 1; +SELECT count() FROM totimezone_op_mono WHERE toTimeZone(create_time, 'UTC') = '2020-09-01 00:00:00'; +DROP TABLE IF EXISTS totimezone_op_mono; diff --git a/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.reference b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.reference new file mode 100644 index 00000000000..c7e9cb788cb --- /dev/null +++ b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.reference @@ -0,0 +1,10 @@ +dictGet +Value +Value +Value +Value +dictHas +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.sql b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.sql new file mode 100644 index 00000000000..a44107d6882 --- /dev/null +++ b/tests/queries/0_stateless/01941_dict_get_has_complex_single_key.sql @@ -0,0 +1,26 @@ +DROP TABLE IF EXISTS test_dictionary_source; +CREATE TABLE test_dictionary_source (key String, value String) ENGINE=TinyLog; + +INSERT INTO test_dictionary_source VALUES ('Key', 'Value'); + +DROP DICTIONARY IF EXISTS test_dictionary; +CREATE DICTIONARY test_dictionary(key String, value String) +PRIMARY KEY key +LAYOUT(COMPLEX_KEY_HASHED()) +SOURCE(CLICKHOUSE(TABLE 'test_dictionary_source')) +LIFETIME(0); + +SELECT 'dictGet'; +SELECT dictGet('test_dictionary', 'value', tuple('Key')); +SELECT dictGet('test_dictionary', 'value', tuple(materialize('Key'))); +SELECT dictGet('test_dictionary', 'value', 'Key'); +SELECT dictGet('test_dictionary', 'value', materialize('Key')); + +SELECT 'dictHas'; +SELECT dictHas('test_dictionary', tuple('Key')); +SELECT dictHas('test_dictionary', tuple(materialize('Key'))); +SELECT dictHas('test_dictionary', 'Key'); +SELECT dictHas('test_dictionary', materialize('Key')); + +DROP DICTIONARY test_dictionary; +DROP TABLE test_dictionary_source; diff --git a/tests/queries/0_stateless/01942_create_table_with_sample.reference b/tests/queries/0_stateless/01942_create_table_with_sample.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01942_create_table_with_sample.sql b/tests/queries/0_stateless/01942_create_table_with_sample.sql new file mode 100644 index 00000000000..6320edd7a31 --- /dev/null +++ b/tests/queries/0_stateless/01942_create_table_with_sample.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS sample_incorrect +(`x` UUID) +ENGINE = MergeTree +ORDER BY tuple(x) +SAMPLE BY x; -- { serverError 59 } + +DROP TABLE IF EXISTS sample_correct; +CREATE TABLE IF NOT EXISTS sample_correct +(`x` String) +ENGINE = MergeTree +ORDER BY tuple(sipHash64(x)) +SAMPLE BY sipHash64(x); + +DROP TABLE sample_correct; diff --git a/tests/queries/0_stateless/01942_untuple_transformers_msan.reference b/tests/queries/0_stateless/01942_untuple_transformers_msan.reference new file mode 100644 index 00000000000..82dea36febd --- /dev/null +++ b/tests/queries/0_stateless/01942_untuple_transformers_msan.reference @@ -0,0 +1 @@ +100.0000991821289 \N \N 1 1024 \N diff --git a/tests/queries/0_stateless/01942_untuple_transformers_msan.sql b/tests/queries/0_stateless/01942_untuple_transformers_msan.sql new file mode 100644 index 00000000000..c1be25d34ac --- /dev/null +++ b/tests/queries/0_stateless/01942_untuple_transformers_msan.sql @@ -0,0 +1 @@ +SELECT untuple(tuple(100.0000991821289)), NULL, untuple((toDateTime(9223372036854775806, -1, NULL, NULL, toDateTime(NULL, NULL)), * EXCEPT b)), NULL FROM (SELECT 1 AS a, 1024, NULL AS b); diff --git a/tests/queries/0_stateless/01943_log_column_sizes.reference b/tests/queries/0_stateless/01943_log_column_sizes.reference new file mode 100644 index 00000000000..91ae12e38ce --- /dev/null +++ b/tests/queries/0_stateless/01943_log_column_sizes.reference @@ -0,0 +1,6 @@ +27 +33 +105 +27 +33 +105 diff --git a/tests/queries/0_stateless/01943_log_column_sizes.sql b/tests/queries/0_stateless/01943_log_column_sizes.sql new file mode 100644 index 00000000000..c6cd48c33d9 --- /dev/null +++ b/tests/queries/0_stateless/01943_log_column_sizes.sql @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS test_log; +DROP TABLE IF EXISTS test_tiny_log; + +CREATE TABLE test_log (x UInt8, s String, a Array(Nullable(String))) ENGINE = Log; +CREATE TABLE test_tiny_log (x UInt8, s String, a Array(Nullable(String))) ENGINE = TinyLog; + +INSERT INTO test_log VALUES (64, 'Value1', ['Value2', 'Value3', NULL]); +INSERT INTO test_tiny_log VALUES (64, 'Value1', ['Value2', 'Value3', NULL]); + +SELECT data_compressed_bytes FROM system.columns WHERE table = 'test_log' AND database = currentDatabase(); +SELECT data_compressed_bytes FROM system.columns WHERE table = 'test_tiny_log' AND database = currentDatabase(); + +DROP TABLE test_log; +DROP TABLE test_tiny_log; \ No newline at end of file diff --git a/tests/queries/0_stateless/01943_non_deterministic_order_key.reference b/tests/queries/0_stateless/01943_non_deterministic_order_key.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01943_non_deterministic_order_key.sql b/tests/queries/0_stateless/01943_non_deterministic_order_key.sql new file mode 100644 index 00000000000..200a88ec677 --- /dev/null +++ b/tests/queries/0_stateless/01943_non_deterministic_order_key.sql @@ -0,0 +1,3 @@ +CREATE TABLE a (number UInt64) ENGINE = MergeTree ORDER BY if(now() > toDateTime('2020-06-01 13:31:40'), toInt64(number), -number); -- { serverError 36 } +CREATE TABLE b (number UInt64) ENGINE = MergeTree ORDER BY now() > toDateTime(number); -- { serverError 36 } +CREATE TABLE c (number UInt64) ENGINE = MergeTree ORDER BY now(); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01944_range_max_elements.reference b/tests/queries/0_stateless/01944_range_max_elements.reference new file mode 100644 index 00000000000..7763ac4ce96 --- /dev/null +++ b/tests/queries/0_stateless/01944_range_max_elements.reference @@ -0,0 +1,33 @@ +[] +[0] +[0,1] +[] +[0] +[0,1] +[] +[0] +[0,1] +[] +[] +[0] +[0,1] +[] +[0] +[0,1] +[] +[0] +[0,1] +[] +[0] +[] +[0] +[0,1] +[] +[0] +[0,1] +[] +[0] +[0,1] +[] +[0] +[0,1] diff --git a/tests/queries/0_stateless/01944_range_max_elements.sql b/tests/queries/0_stateless/01944_range_max_elements.sql new file mode 100644 index 00000000000..c18f61e3190 --- /dev/null +++ b/tests/queries/0_stateless/01944_range_max_elements.sql @@ -0,0 +1,7 @@ +SET function_range_max_elements_in_block = 10; +SELECT range(number % 3) FROM numbers(10); +SELECT range(number % 3) FROM numbers(11); +SELECT range(number % 3) FROM numbers(12); -- { serverError 69 } + +SET function_range_max_elements_in_block = 12; +SELECT range(number % 3) FROM numbers(12); diff --git a/tests/queries/0_stateless/01945_system_warnings.expect b/tests/queries/0_stateless/01945_system_warnings.expect new file mode 100755 index 00000000000..56d219e1040 --- /dev/null +++ b/tests/queries/0_stateless/01945_system_warnings.expect @@ -0,0 +1,40 @@ +#!/usr/bin/expect -f + +# This is a test for system.warnings. Testing in interactive mode is necessary, +# as we want to see certain warnings from client + +log_user 0 +set timeout 60 +match_max 100000 + +# A default timeout action is to do nothing, change it to fail +expect_after { + timeout { + exit 1 + } +} + +set basedir [file dirname $argv0] +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" +expect ":) " + +#find out BUILD TYPE +send -- "SELECT value FROM system.build_options WHERE name='BUILD_TYPE'\r" +expect { + "Debug" { + # Check debug message in system.warnings + send -- "SELECT message FROM system.warnings WHERE message='Server was built in debug mode. It will work slowly.'\r" + expect "Server was built in debug mode. It will work slowly." + expect ":) " + } + "RelWithDebInfo" { + # Check empty to find out existence + send -- "SELECT message FROM system.warnings WHERE 0=1\r" + expect "Ok." + expect ":) " + } +} + +# Finish test +send -- "\4" +expect eof diff --git a/tests/queries/0_stateless/01945_system_warnings.reference b/tests/queries/0_stateless/01945_system_warnings.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01946_test.reference b/tests/queries/0_stateless/01946_test.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01946_tskv.reference b/tests/queries/0_stateless/01946_tskv.reference new file mode 100644 index 00000000000..5a3b19fa88f --- /dev/null +++ b/tests/queries/0_stateless/01946_tskv.reference @@ -0,0 +1 @@ +can contain = symbol diff --git a/tests/queries/0_stateless/01946_tskv.sh b/tests/queries/0_stateless/01946_tskv.sh new file mode 100755 index 00000000000..ecc18d205d2 --- /dev/null +++ b/tests/queries/0_stateless/01946_tskv.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS tskv"; +$CLICKHOUSE_CLIENT --query="CREATE TABLE tskv (text String) ENGINE = Memory"; + +# shellcheck disable=SC2028 +echo -n 'tskv text=can contain \= symbol +' | $CLICKHOUSE_CLIENT --query="INSERT INTO tskv FORMAT TSKV"; + +$CLICKHOUSE_CLIENT --query="SELECT * FROM tskv"; +$CLICKHOUSE_CLIENT --query="DROP TABLE tskv"; diff --git a/tests/queries/0_stateless/arcadia_skip_list.txt b/tests/queries/0_stateless/arcadia_skip_list.txt index 838a2da9aff..d7581cc4e07 100644 --- a/tests/queries/0_stateless/arcadia_skip_list.txt +++ b/tests/queries/0_stateless/arcadia_skip_list.txt @@ -215,6 +215,7 @@ 01747_join_view_filter_dictionary 01748_dictionary_table_dot 01755_client_highlight_multi_line_comment_regression +01756_optimize_skip_unused_shards_rewrite_in 00950_dict_get 01683_flat_dictionary 01681_cache_dictionary_simple_key @@ -251,6 +252,16 @@ 01924_argmax_bitmap_state 01914_exchange_dictionaries 01923_different_expression_name_alias +01930_optimize_skip_unused_shards_rewrite_in 01932_null_valid_identifier 00918_json_functions 01889_sql_json_functions +01849_geoToS2 +01851_s2_to_geo +01852_s2_get_neighbours +01853_s2_cells_intersect +01854_s2_cap_contains +01854_s2_cap_union +01428_h3_range_check +01442_h3kring_range_check +01906_h3_to_geo diff --git a/tests/queries/1_stateful/00165_jit_aggregate_functions.reference b/tests/queries/1_stateful/00165_jit_aggregate_functions.reference index 2d94ad190ca..451a676754c 100644 --- a/tests/queries/1_stateful/00165_jit_aggregate_functions.reference +++ b/tests/queries/1_stateful/00165_jit_aggregate_functions.reference @@ -1,128 +1,128 @@ Aggregation using JIT compilation Simple functions -1704509 4611700827100483880 9223360787015464643 10441337359398154812 19954243669348.844 9648741.579254271 523264 -732797 4611701940806302259 9223355550934604746 977192643464016658 2054229034942.3723 51998323.94457991 475698 -598875 4611701407242345792 9223362250391155632 9312163881623734456 27615161624211.875 12261797.824844675 337212 -792887 4611699550286611812 9223290551912005343 6930300520201292824 27479710385933.586 53095331.60360441 252197 -3807842 4611710821592843606 9223326163906184987 16710274896338005145 85240848090850.69 22373416.533275086 196036 -25703952 4611709443519524003 9223353913449113943 9946868158853570839 67568783303242.086 3154349.826950714 147211 -716829 4611852156092872082 9223361623076951140 15381015774917924786 170693446547158.72 201431892.4773785 90109 -59183 4611730685242027332 9223354909338698162 8078812522502896568 94622946187035.42 1425270865.0901496 85379 -33010362 4611704682869732882 9223268545373999677 2064452191838585926 26532987929602.555 3695122.4062526934 77807 -800784 4611752907938305166 9223340418389788041 18082918611792817587 233352070043266.62 36535786.81446395 77492 -20810645 4611712185532639162 9223218900001937412 4996531385439292694 68246505203164.63 6316535.831023813 73213 -25843850 4611690025407720929 9223346023778617822 12755881190906812868 185015319325648.16 9962165.34831339 68945 -23447120 4611796031755620254 9223329309291309758 17231649548755339966 255019232629204.38 7937191.271698021 67570 -14739804 4611692230555590277 9223313509005166531 2458378896777063244 38308020331864.36 14590240.469105456 64174 -32077710 4611884228437061959 9223352444952988904 12965822147651192908 214467085941034.7 7257521.096258734 60456 -22446879 4611846229717089436 9223124373140579096 13530160492087688838 231724477077663.4 4737362.521046629 58389 -170282 4611833225706935900 9223371583739401906 8076893424988479310 141657635880324.8 1613795518.1065989 57017 -11482817 4611708000353743073 9223337838355779113 14841435427430843458 283531099960470.8 9938452.835998287 52345 -63469 4611695097019173921 9223353530156141191 6296784708578574520 120762239817777.88 579655378.4603049 52142 -29103473 4611744585914335132 9223333530281362537 5908285283932344933 123712996438970.34 867841.595541967 47758 +1704509 4611700827100483880 9223360787015464643 10441337359398154812 19954243669348.844 9648741.579254271 523264 9223372036854775807 4611686018427387904 4544239379628300646 +732797 4611701940806302259 9223355550934604746 977192643464016658 2054229034942.3723 51998323.94457991 475698 9223372036854775807 4611686018427387904 4091184823334377716 +598875 4611701407242345792 9223362250391155632 9312163881623734456 27615161624211.875 12261797.824844675 337212 9223372036854775807 4611686018427387904 3725992504798702670 +792887 4611699550286611812 9223290551912005343 6930300520201292824 27479710385933.586 53095331.60360441 252197 9223372036854775807 4611686018427387904 6536441508464694614 +3807842 4611710821592843606 9223326163906184987 16710274896338005145 85240848090850.69 22373416.533275086 196036 9223372036854775807 4611686018427387904 1797862753609257231 +25703952 4611709443519524003 9223353913449113943 9946868158853570839 67568783303242.086 3154349.826950714 147211 9223372036854775807 4611686018427387904 8737124378202300429 +716829 4611852156092872082 9223361623076951140 15381015774917924786 170693446547158.72 201431892.4773785 90109 9223372036854775807 4611686018427387904 8209915323001116338 +59183 4611730685242027332 9223354909338698162 8078812522502896568 94622946187035.42 1425270865.0901496 85379 9223372036854775807 4611686018427387904 8909082036598843562 +33010362 4611704682869732882 9223268545373999677 2064452191838585926 26532987929602.555 3695122.4062526934 77807 9223372036854775807 4611686018427387904 5411365383789552292 +800784 4611752907938305166 9223340418389788041 18082918611792817587 233352070043266.62 36535786.81446395 77492 9223372036854775807 4611686018427387904 2059255810151375435 +20810645 4611712185532639162 9223218900001937412 4996531385439292694 68246505203164.63 6316535.831023813 73213 9223372036854775807 4611686018427387904 8852740550386113674 +25843850 4611690025407720929 9223346023778617822 12755881190906812868 185015319325648.16 9962165.34831339 68945 9223372036854775807 4611686018427387904 7849665866595760148 +23447120 4611796031755620254 9223329309291309758 17231649548755339966 255019232629204.38 7937191.271698021 67570 9223372036854775807 4611686018427387904 3435410911925610424 +14739804 4611692230555590277 9223313509005166531 2458378896777063244 38308020331864.36 14590240.469105456 64174 9223372036854775807 4611686018427387904 511910855240035342 +32077710 4611884228437061959 9223352444952988904 12965822147651192908 214467085941034.7 7257521.096258734 60456 9223372036854775807 4611686018427387904 2256071920672551964 +22446879 4611846229717089436 9223124373140579096 13530160492087688838 231724477077663.4 4737362.521046629 58389 9223372036854775807 4611686018427387904 6236276364886386410 +170282 4611833225706935900 9223371583739401906 8076893424988479310 141657635880324.8 1613795518.1065989 57017 9223372036854775807 4611686018427387904 4755775861151848768 +11482817 4611708000353743073 9223337838355779113 14841435427430843458 283531099960470.8 9938452.835998287 52345 9223372036854775807 4611686018427387904 5371586112642152558 +63469 4611695097019173921 9223353530156141191 6296784708578574520 120762239817777.88 579655378.4603049 52142 9223372036854775807 4611686018427387904 4150567963952988110 +29103473 4611744585914335132 9223333530281362537 5908285283932344933 123712996438970.34 867841.595541967 47758 9223372036854775807 4611686018427387904 3238284030821087319 Simple functions with non compilable function -1704509 4611700827100483880 9223360787015464643 10441337359398154812 4611686018427387904 19954243669348.844 9648741.579254271 523264 -732797 4611701940806302259 9223355550934604746 977192643464016658 4611686018427387904 2054229034942.3723 51998323.94457991 475698 -598875 4611701407242345792 9223362250391155632 9312163881623734456 4611686018427387904 27615161624211.875 12261797.824844675 337212 -792887 4611699550286611812 9223290551912005343 6930300520201292824 4611686018427387904 27479710385933.586 53095331.60360441 252197 -3807842 4611710821592843606 9223326163906184987 16710274896338005145 4611686018427387904 85240848090850.69 22373416.533275086 196036 -25703952 4611709443519524003 9223353913449113943 9946868158853570839 4611686018427387904 67568783303242.086 3154349.826950714 147211 -716829 4611852156092872082 9223361623076951140 15381015774917924786 4611686018427387904 170693446547158.72 201431892.4773785 90109 -59183 4611730685242027332 9223354909338698162 8078812522502896568 4611686018427387904 94622946187035.42 1425270865.0901496 85379 -33010362 4611704682869732882 9223268545373999677 2064452191838585926 4611686018427387904 26532987929602.555 3695122.4062526934 77807 -800784 4611752907938305166 9223340418389788041 18082918611792817587 4611686018427387904 233352070043266.62 36535786.81446395 77492 -20810645 4611712185532639162 9223218900001937412 4996531385439292694 4611686018427387904 68246505203164.63 6316535.831023813 73213 -25843850 4611690025407720929 9223346023778617822 12755881190906812868 4611686018427387904 185015319325648.16 9962165.34831339 68945 -23447120 4611796031755620254 9223329309291309758 17231649548755339966 4611686018427387904 255019232629204.38 7937191.271698021 67570 -14739804 4611692230555590277 9223313509005166531 2458378896777063244 4611686018427387904 38308020331864.36 14590240.469105456 64174 -32077710 4611884228437061959 9223352444952988904 12965822147651192908 4611686018427387904 214467085941034.7 7257521.096258734 60456 -22446879 4611846229717089436 9223124373140579096 13530160492087688838 4611686018427387904 231724477077663.4 4737362.521046629 58389 -170282 4611833225706935900 9223371583739401906 8076893424988479310 4611686018427387904 141657635880324.8 1613795518.1065989 57017 -11482817 4611708000353743073 9223337838355779113 14841435427430843458 4611686018427387904 283531099960470.8 9938452.835998287 52345 -63469 4611695097019173921 9223353530156141191 6296784708578574520 4611686018427387904 120762239817777.88 579655378.4603049 52142 -29103473 4611744585914335132 9223333530281362537 5908285283932344933 4611686018427387904 123712996438970.34 867841.595541967 47758 +1704509 4611700827100483880 9223360787015464643 10441337359398154812 3620921835565807284859452 19954243669348.844 9648741.579254271 523264 9223372036854775807 4611686018427387904 4544239379628300646 +732797 4611701940806302259 9223355550934604746 977192643464016658 3289442827160604417733394 2054229034942.3723 51998323.94457991 475698 9223372036854775807 4611686018427387904 4091184823334377716 +598875 4611701407242345792 9223362250391155632 9312163881623734456 2330921446573746856380600 27615161624211.875 12261797.824844675 337212 9223372036854775807 4611686018427387904 3725992504798702670 +792887 4611699550286611812 9223290551912005343 6930300520201292824 1745179600137886041476120 27479710385933.586 53095331.60360441 252197 9223372036854775807 4611686018427387904 6536441508464694614 +3807842 4611710821592843606 9223326163906184987 16710274896338005145 1356295121550317411019929 85240848090850.69 22373416.533275086 196036 9223372036854775807 4611686018427387904 1797862753609257231 +25703952 4611709443519524003 9223353913449113943 9946868158853570839 1018731388338768841564439 67568783303242.086 3154349.826950714 147211 9223372036854775807 4611686018427387904 8737124378202300429 +716829 4611852156092872082 9223361623076951140 15381015774917924786 623810478612337115371442 170693446547158.72 201431892.4773785 90109 9223372036854775807 4611686018427387904 8209915323001116338 +59183 4611730685242027332 9223354909338698162 8078812522502896568 589916507545680254024632 94622946187035.42 1425270865.0901496 85379 9223372036854775807 4611686018427387904 8909082036598843562 +33010362 4611704682869732882 9223268545373999677 2064452191838585926 538517864195994778911814 26532987929602.555 3695122.4062526934 77807 9223372036854775807 4611686018427387904 5411365383789552292 +800784 4611752907938305166 9223340418389788041 18082918611792817587 535545510122473785781683 233352070043266.62 36535786.81446395 77492 9223372036854775807 4611686018427387904 2059255810151375435 +20810645 4611712185532639162 9223218900001937412 4996531385439292694 506405014842860050255126 68246505203164.63 6316535.831023813 73213 9223372036854775807 4611686018427387904 8852740550386113674 +25843850 4611690025407720929 9223346023778617822 12755881190906812868 476547495537329753708996 185015319325648.16 9962165.34831339 68945 9223372036854775807 4611686018427387904 7849665866595760148 +23447120 4611796031755620254 9223329309291309758 17231649548755339966 467236365548464278670014 255019232629204.38 7937191.271698021 67570 9223372036854775807 4611686018427387904 3435410911925610424 +14739804 4611692230555590277 9223313509005166531 2458378896777063244 444126268697527941770060 38308020331864.36 14590240.469105456 64174 9223372036854775807 4611686018427387904 511910855240035342 +32077710 4611884228437061959 9223352444952988904 12965822147651192908 417407443977973675608140 214467085941034.7 7257521.096258734 60456 9223372036854775807 4611686018427387904 2256071920672551964 +22446879 4611846229717089436 9223124373140579096 13530160492087688838 403462269796593691082374 231724477077663.4 4737362.521046629 58389 9223372036854775807 4611686018427387904 6236276364886386410 +170282 4611833225706935900 9223371583739401906 8076893424988479310 394417911933408911581006 141657635880324.8 1613795518.1065989 57017 9223372036854775807 4611686018427387904 4755775861151848768 +11482817 4611708000353743073 9223337838355779113 14841435427430843458 361995300393829962204226 283531099960470.8 9938452.835998287 52345 9223372036854775807 4611686018427387904 5371586112642152558 +63469 4611695097019173921 9223353530156141191 6296784708578574520 360843057610541117735096 120762239817777.88 579655378.4603049 52142 9223372036854775807 4611686018427387904 4150567963952988110 +29103473 4611744585914335132 9223333530281362537 5908285283932344933 330534668598011678200421 123712996438970.34 867841.595541967 47758 9223372036854775807 4611686018427387904 3238284030821087319 Simple functions if combinator -1704509 4611700827100483880 9223310246721229500 16398241567152875142 62618822667209.71 2224726.7626273884 261874 -732797 4611721382223060002 9223355550934604746 16281585268876620522 68472164943295.68 5898616.931652982 237784 -598875 4611701407242345792 9223362250391155632 3577699408183553052 21300140553347.42 53771550.26565126 167966 -792887 4611699550286611812 9223164887726235740 7088177025760385824 56461952267903.89 92835869.96920013 125539 -3807842 4611710821592843606 9223283397553859544 5756765290752687660 58835559208469.4 39794091.419183925 97845 -25703952 4611784761593342388 9223241341744449690 4782279928971192568 65182094768443.91 9276773.708181158 73368 -716829 4611852156092872082 9223361623076951140 8613712481895484190 191445613359755.62 291083243.75407773 44993 -59183 4611730685242027332 9223354909338698162 18369075291092794110 429013599530392 5925109959.715378 42817 -33010362 4611704682869732882 9223092117352620518 9991152681891671022 257099731913529.5 12412830.045471078 38861 -800784 4611752907938305166 9223309994342931384 5251877538869750510 135472890315726.03 53535427.52018088 38767 -20810645 4611712185532639162 9223218900001937412 11803718472901310700 323593455407553 10496765.20741332 36477 -25843850 4611744529689964352 9223346023778617822 127137885677350808 3700925266420.715 18966925.191309396 34353 -23447120 4611796031755620254 9223329309291309758 1841522159325376278 54534534450526.42 6271211.193812284 33768 -14739804 4611762063154116632 9223007205463222212 16302703534054321116 506987919332451.8 6885575.861759452 32156 -32077710 4612033458080771112 9223352444952988904 421072759851674408 13955745719596.793 12220152.393889504 30172 -22446879 4611846229717089436 9223124373140579096 6577134317587565298 224866980668999.47 2482202.163802278 29249 -170282 4611833225706935900 9223371583739401906 15764226366913732386 551447384017691 2515144222.953728 28587 -11482817 4611990575414646848 9223302669582414438 9828522700609834800 378121905921203.2 34845264.2080656 25993 -63469 4612175339998036670 9222961628400798084 17239621485933250238 663164390134376.5 7825349797.6059 25996 -29103473 4611744585914335132 9223035551850347954 12590190375872647672 525927999326314.7 26049107.15514301 23939 +1704509 4611700827100483880 9223310246721229500 16398241567152875142 62618822667209.71 2224726.7626273884 261874 9223372036854775806 4611686018427387904 4518874482384062894 +732797 4611721382223060002 9223355550934604746 16281585268876620522 68472164943295.68 5898616.931652982 237784 9223372036854775806 4611686018427387904 3641900047478154650 +598875 4611701407242345792 9223362250391155632 3577699408183553052 21300140553347.42 53771550.26565126 167966 9223372036854775806 4611686018427387904 1688477495230210408 +792887 4611699550286611812 9223164887726235740 7088177025760385824 56461952267903.89 92835869.96920013 125539 9223372036854775806 4611686018427387904 4850868151095058072 +3807842 4611710821592843606 9223283397553859544 5756765290752687660 58835559208469.4 39794091.419183925 97845 9223372036854775806 4611686018427387904 6845214684357194564 +25703952 4611784761593342388 9223241341744449690 4782279928971192568 65182094768443.91 9276773.708181158 73368 9223372036854775806 4611686018427387904 1384302533387727316 +716829 4611852156092872082 9223361623076951140 8613712481895484190 191445613359755.62 291083243.75407773 44993 9223372036854775806 4611686018427387904 6344483471397203854 +59183 4611730685242027332 9223354909338698162 18369075291092794110 429013599530392 5925109959.715378 42817 9223372036854775806 4611686018427387904 5909305558020042898 +33010362 4611704682869732882 9223092117352620518 9991152681891671022 257099731913529.5 12412830.045471078 38861 9223372036854775806 4611686018427387904 4672855013852508626 +800784 4611752907938305166 9223309994342931384 5251877538869750510 135472890315726.03 53535427.52018088 38767 9223372036854775806 4611686018427387904 7801864489649220514 +20810645 4611712185532639162 9223218900001937412 11803718472901310700 323593455407553 10496765.20741332 36477 9223372036854775806 4611686018427387904 5941995311893397960 +25843850 4611744529689964352 9223346023778617822 127137885677350808 3700925266420.715 18966925.191309396 34353 9223372036854775806 4611686018427387904 6700111718676827412 +23447120 4611796031755620254 9223329309291309758 1841522159325376278 54534534450526.42 6271211.193812284 33768 9223372036854775806 4611686018427387904 2325654077031843898 +14739804 4611762063154116632 9223007205463222212 16302703534054321116 506987919332451.8 6885575.861759452 32156 9223372036854775806 4611686018427387904 2114922310535979832 +32077710 4612033458080771112 9223352444952988904 421072759851674408 13955745719596.793 12220152.393889504 30172 9223372036854775806 4611686018427387904 4399934528735249092 +22446879 4611846229717089436 9223124373140579096 6577134317587565298 224866980668999.47 2482202.163802278 29249 9223372036854775806 4611686018427387904 8763910740678180498 +170282 4611833225706935900 9223371583739401906 15764226366913732386 551447384017691 2515144222.953728 28587 9223372036854775806 4611686018427387904 8217388408377809010 +11482817 4611990575414646848 9223302669582414438 9828522700609834800 378121905921203.2 34845264.2080656 25993 9223372036854775806 4611686018427387904 4689180182672571856 +63469 4612175339998036670 9222961628400798084 17239621485933250238 663164390134376.5 7825349797.6059 25996 9223372036854775806 4611686018427387904 2067736879306995526 +29103473 4611744585914335132 9223035551850347954 12590190375872647672 525927999326314.7 26049107.15514301 23939 9223372036854775806 4611686018427387904 8318055464870862444 Aggregation without JIT compilation Simple functions -1704509 4611700827100483880 9223360787015464643 10441337359398154812 19954243669348.844 9648741.579254271 523264 -732797 4611701940806302259 9223355550934604746 977192643464016658 2054229034942.3723 51998323.94457991 475698 -598875 4611701407242345792 9223362250391155632 9312163881623734456 27615161624211.875 12261797.824844675 337212 -792887 4611699550286611812 9223290551912005343 6930300520201292824 27479710385933.586 53095331.60360441 252197 -3807842 4611710821592843606 9223326163906184987 16710274896338005145 85240848090850.69 22373416.533275086 196036 -25703952 4611709443519524003 9223353913449113943 9946868158853570839 67568783303242.086 3154349.826950714 147211 -716829 4611852156092872082 9223361623076951140 15381015774917924786 170693446547158.72 201431892.4773785 90109 -59183 4611730685242027332 9223354909338698162 8078812522502896568 94622946187035.42 1425270865.0901496 85379 -33010362 4611704682869732882 9223268545373999677 2064452191838585926 26532987929602.555 3695122.4062526934 77807 -800784 4611752907938305166 9223340418389788041 18082918611792817587 233352070043266.62 36535786.81446395 77492 -20810645 4611712185532639162 9223218900001937412 4996531385439292694 68246505203164.63 6316535.831023813 73213 -25843850 4611690025407720929 9223346023778617822 12755881190906812868 185015319325648.16 9962165.34831339 68945 -23447120 4611796031755620254 9223329309291309758 17231649548755339966 255019232629204.38 7937191.271698021 67570 -14739804 4611692230555590277 9223313509005166531 2458378896777063244 38308020331864.36 14590240.469105456 64174 -32077710 4611884228437061959 9223352444952988904 12965822147651192908 214467085941034.7 7257521.096258734 60456 -22446879 4611846229717089436 9223124373140579096 13530160492087688838 231724477077663.4 4737362.521046629 58389 -170282 4611833225706935900 9223371583739401906 8076893424988479310 141657635880324.8 1613795518.1065989 57017 -11482817 4611708000353743073 9223337838355779113 14841435427430843458 283531099960470.8 9938452.835998287 52345 -63469 4611695097019173921 9223353530156141191 6296784708578574520 120762239817777.88 579655378.4603049 52142 -29103473 4611744585914335132 9223333530281362537 5908285283932344933 123712996438970.34 867841.595541967 47758 +1704509 4611700827100483880 9223360787015464643 10441337359398154812 19954243669348.844 9648741.579254271 523264 9223372036854775807 4611686018427387904 4544239379628300646 +732797 4611701940806302259 9223355550934604746 977192643464016658 2054229034942.3723 51998323.94457991 475698 9223372036854775807 4611686018427387904 4091184823334377716 +598875 4611701407242345792 9223362250391155632 9312163881623734456 27615161624211.875 12261797.824844675 337212 9223372036854775807 4611686018427387904 3725992504798702670 +792887 4611699550286611812 9223290551912005343 6930300520201292824 27479710385933.586 53095331.60360441 252197 9223372036854775807 4611686018427387904 6536441508464694614 +3807842 4611710821592843606 9223326163906184987 16710274896338005145 85240848090850.69 22373416.533275086 196036 9223372036854775807 4611686018427387904 1797862753609257231 +25703952 4611709443519524003 9223353913449113943 9946868158853570839 67568783303242.086 3154349.826950714 147211 9223372036854775807 4611686018427387904 8737124378202300429 +716829 4611852156092872082 9223361623076951140 15381015774917924786 170693446547158.72 201431892.4773785 90109 9223372036854775807 4611686018427387904 8209915323001116338 +59183 4611730685242027332 9223354909338698162 8078812522502896568 94622946187035.42 1425270865.0901496 85379 9223372036854775807 4611686018427387904 8909082036598843562 +33010362 4611704682869732882 9223268545373999677 2064452191838585926 26532987929602.555 3695122.4062526934 77807 9223372036854775807 4611686018427387904 5411365383789552292 +800784 4611752907938305166 9223340418389788041 18082918611792817587 233352070043266.62 36535786.81446395 77492 9223372036854775807 4611686018427387904 2059255810151375435 +20810645 4611712185532639162 9223218900001937412 4996531385439292694 68246505203164.63 6316535.831023813 73213 9223372036854775807 4611686018427387904 8852740550386113674 +25843850 4611690025407720929 9223346023778617822 12755881190906812868 185015319325648.16 9962165.34831339 68945 9223372036854775807 4611686018427387904 7849665866595760148 +23447120 4611796031755620254 9223329309291309758 17231649548755339966 255019232629204.38 7937191.271698021 67570 9223372036854775807 4611686018427387904 3435410911925610424 +14739804 4611692230555590277 9223313509005166531 2458378896777063244 38308020331864.36 14590240.469105456 64174 9223372036854775807 4611686018427387904 511910855240035342 +32077710 4611884228437061959 9223352444952988904 12965822147651192908 214467085941034.7 7257521.096258734 60456 9223372036854775807 4611686018427387904 2256071920672551964 +22446879 4611846229717089436 9223124373140579096 13530160492087688838 231724477077663.4 4737362.521046629 58389 9223372036854775807 4611686018427387904 6236276364886386410 +170282 4611833225706935900 9223371583739401906 8076893424988479310 141657635880324.8 1613795518.1065989 57017 9223372036854775807 4611686018427387904 4755775861151848768 +11482817 4611708000353743073 9223337838355779113 14841435427430843458 283531099960470.8 9938452.835998287 52345 9223372036854775807 4611686018427387904 5371586112642152558 +63469 4611695097019173921 9223353530156141191 6296784708578574520 120762239817777.88 579655378.4603049 52142 9223372036854775807 4611686018427387904 4150567963952988110 +29103473 4611744585914335132 9223333530281362537 5908285283932344933 123712996438970.34 867841.595541967 47758 9223372036854775807 4611686018427387904 3238284030821087319 Simple functions with non compilable function -1704509 4611700827100483880 9223360787015464643 10441337359398154812 4611686018427387904 19954243669348.844 9648741.579254271 523264 -732797 4611701940806302259 9223355550934604746 977192643464016658 4611686018427387904 2054229034942.3723 51998323.94457991 475698 -598875 4611701407242345792 9223362250391155632 9312163881623734456 4611686018427387904 27615161624211.875 12261797.824844675 337212 -792887 4611699550286611812 9223290551912005343 6930300520201292824 4611686018427387904 27479710385933.586 53095331.60360441 252197 -3807842 4611710821592843606 9223326163906184987 16710274896338005145 4611686018427387904 85240848090850.69 22373416.533275086 196036 -25703952 4611709443519524003 9223353913449113943 9946868158853570839 4611686018427387904 67568783303242.086 3154349.826950714 147211 -716829 4611852156092872082 9223361623076951140 15381015774917924786 4611686018427387904 170693446547158.72 201431892.4773785 90109 -59183 4611730685242027332 9223354909338698162 8078812522502896568 4611686018427387904 94622946187035.42 1425270865.0901496 85379 -33010362 4611704682869732882 9223268545373999677 2064452191838585926 4611686018427387904 26532987929602.555 3695122.4062526934 77807 -800784 4611752907938305166 9223340418389788041 18082918611792817587 4611686018427387904 233352070043266.62 36535786.81446395 77492 -20810645 4611712185532639162 9223218900001937412 4996531385439292694 4611686018427387904 68246505203164.63 6316535.831023813 73213 -25843850 4611690025407720929 9223346023778617822 12755881190906812868 4611686018427387904 185015319325648.16 9962165.34831339 68945 -23447120 4611796031755620254 9223329309291309758 17231649548755339966 4611686018427387904 255019232629204.38 7937191.271698021 67570 -14739804 4611692230555590277 9223313509005166531 2458378896777063244 4611686018427387904 38308020331864.36 14590240.469105456 64174 -32077710 4611884228437061959 9223352444952988904 12965822147651192908 4611686018427387904 214467085941034.7 7257521.096258734 60456 -22446879 4611846229717089436 9223124373140579096 13530160492087688838 4611686018427387904 231724477077663.4 4737362.521046629 58389 -170282 4611833225706935900 9223371583739401906 8076893424988479310 4611686018427387904 141657635880324.8 1613795518.1065989 57017 -11482817 4611708000353743073 9223337838355779113 14841435427430843458 4611686018427387904 283531099960470.8 9938452.835998287 52345 -63469 4611695097019173921 9223353530156141191 6296784708578574520 4611686018427387904 120762239817777.88 579655378.4603049 52142 -29103473 4611744585914335132 9223333530281362537 5908285283932344933 4611686018427387904 123712996438970.34 867841.595541967 47758 +1704509 4611700827100483880 9223360787015464643 10441337359398154812 3620921835565807284859452 19954243669348.844 9648741.579254271 523264 9223372036854775807 4611686018427387904 4544239379628300646 +732797 4611701940806302259 9223355550934604746 977192643464016658 3289442827160604417733394 2054229034942.3723 51998323.94457991 475698 9223372036854775807 4611686018427387904 4091184823334377716 +598875 4611701407242345792 9223362250391155632 9312163881623734456 2330921446573746856380600 27615161624211.875 12261797.824844675 337212 9223372036854775807 4611686018427387904 3725992504798702670 +792887 4611699550286611812 9223290551912005343 6930300520201292824 1745179600137886041476120 27479710385933.586 53095331.60360441 252197 9223372036854775807 4611686018427387904 6536441508464694614 +3807842 4611710821592843606 9223326163906184987 16710274896338005145 1356295121550317411019929 85240848090850.69 22373416.533275086 196036 9223372036854775807 4611686018427387904 1797862753609257231 +25703952 4611709443519524003 9223353913449113943 9946868158853570839 1018731388338768841564439 67568783303242.086 3154349.826950714 147211 9223372036854775807 4611686018427387904 8737124378202300429 +716829 4611852156092872082 9223361623076951140 15381015774917924786 623810478612337115371442 170693446547158.72 201431892.4773785 90109 9223372036854775807 4611686018427387904 8209915323001116338 +59183 4611730685242027332 9223354909338698162 8078812522502896568 589916507545680254024632 94622946187035.42 1425270865.0901496 85379 9223372036854775807 4611686018427387904 8909082036598843562 +33010362 4611704682869732882 9223268545373999677 2064452191838585926 538517864195994778911814 26532987929602.555 3695122.4062526934 77807 9223372036854775807 4611686018427387904 5411365383789552292 +800784 4611752907938305166 9223340418389788041 18082918611792817587 535545510122473785781683 233352070043266.62 36535786.81446395 77492 9223372036854775807 4611686018427387904 2059255810151375435 +20810645 4611712185532639162 9223218900001937412 4996531385439292694 506405014842860050255126 68246505203164.63 6316535.831023813 73213 9223372036854775807 4611686018427387904 8852740550386113674 +25843850 4611690025407720929 9223346023778617822 12755881190906812868 476547495537329753708996 185015319325648.16 9962165.34831339 68945 9223372036854775807 4611686018427387904 7849665866595760148 +23447120 4611796031755620254 9223329309291309758 17231649548755339966 467236365548464278670014 255019232629204.38 7937191.271698021 67570 9223372036854775807 4611686018427387904 3435410911925610424 +14739804 4611692230555590277 9223313509005166531 2458378896777063244 444126268697527941770060 38308020331864.36 14590240.469105456 64174 9223372036854775807 4611686018427387904 511910855240035342 +32077710 4611884228437061959 9223352444952988904 12965822147651192908 417407443977973675608140 214467085941034.7 7257521.096258734 60456 9223372036854775807 4611686018427387904 2256071920672551964 +22446879 4611846229717089436 9223124373140579096 13530160492087688838 403462269796593691082374 231724477077663.4 4737362.521046629 58389 9223372036854775807 4611686018427387904 6236276364886386410 +170282 4611833225706935900 9223371583739401906 8076893424988479310 394417911933408911581006 141657635880324.8 1613795518.1065989 57017 9223372036854775807 4611686018427387904 4755775861151848768 +11482817 4611708000353743073 9223337838355779113 14841435427430843458 361995300393829962204226 283531099960470.8 9938452.835998287 52345 9223372036854775807 4611686018427387904 5371586112642152558 +63469 4611695097019173921 9223353530156141191 6296784708578574520 360843057610541117735096 120762239817777.88 579655378.4603049 52142 9223372036854775807 4611686018427387904 4150567963952988110 +29103473 4611744585914335132 9223333530281362537 5908285283932344933 330534668598011678200421 123712996438970.34 867841.595541967 47758 9223372036854775807 4611686018427387904 3238284030821087319 Simple functions if combinator -1704509 4611700827100483880 9223310246721229500 16398241567152875142 2224726.7626273884 261874 -732797 4611721382223060002 9223355550934604746 16281585268876620522 5898616.931652982 237784 -598875 4611701407242345792 9223362250391155632 3577699408183553052 53771550.26565126 167966 -792887 4611699550286611812 9223164887726235740 7088177025760385824 92835869.96920013 125539 -3807842 4611710821592843606 9223283397553859544 5756765290752687660 39794091.419183925 97845 -25703952 4611784761593342388 9223241341744449690 4782279928971192568 9276773.708181158 73368 -716829 4611852156092872082 9223361623076951140 8613712481895484190 291083243.75407773 44993 -59183 4611730685242027332 9223354909338698162 18369075291092794110 5925109959.715378 42817 -33010362 4611704682869732882 9223092117352620518 9991152681891671022 12412830.045471078 38861 -800784 4611752907938305166 9223309994342931384 5251877538869750510 53535427.52018088 38767 -20810645 4611712185532639162 9223218900001937412 11803718472901310700 10496765.20741332 36477 -25843850 4611744529689964352 9223346023778617822 127137885677350808 18966925.191309396 34353 -23447120 4611796031755620254 9223329309291309758 1841522159325376278 6271211.193812284 33768 -14739804 4611762063154116632 9223007205463222212 16302703534054321116 6885575.861759452 32156 -32077710 4612033458080771112 9223352444952988904 421072759851674408 12220152.393889504 30172 -22446879 4611846229717089436 9223124373140579096 6577134317587565298 2482202.163802278 29249 -170282 4611833225706935900 9223371583739401906 15764226366913732386 2515144222.953728 28587 -11482817 4611990575414646848 9223302669582414438 9828522700609834800 34845264.2080656 25993 -63469 4612175339998036670 9222961628400798084 17239621485933250238 7825349797.6059 25996 -29103473 4611744585914335132 9223035551850347954 12590190375872647672 26049107.15514301 23939 +1704509 4611700827100483880 9223310246721229500 16398241567152875142 62618822667209.71 2224726.7626273884 261874 9223372036854775806 4611686018427387904 4518874482384062894 +732797 4611721382223060002 9223355550934604746 16281585268876620522 68472164943295.68 5898616.931652982 237784 9223372036854775806 4611686018427387904 3641900047478154650 +598875 4611701407242345792 9223362250391155632 3577699408183553052 21300140553347.42 53771550.26565126 167966 9223372036854775806 4611686018427387904 1688477495230210408 +792887 4611699550286611812 9223164887726235740 7088177025760385824 56461952267903.89 92835869.96920013 125539 9223372036854775806 4611686018427387904 4850868151095058072 +3807842 4611710821592843606 9223283397553859544 5756765290752687660 58835559208469.4 39794091.419183925 97845 9223372036854775806 4611686018427387904 6845214684357194564 +25703952 4611784761593342388 9223241341744449690 4782279928971192568 65182094768443.91 9276773.708181158 73368 9223372036854775806 4611686018427387904 1384302533387727316 +716829 4611852156092872082 9223361623076951140 8613712481895484190 191445613359755.62 291083243.75407773 44993 9223372036854775806 4611686018427387904 6344483471397203854 +59183 4611730685242027332 9223354909338698162 18369075291092794110 429013599530392 5925109959.715378 42817 9223372036854775806 4611686018427387904 5909305558020042898 +33010362 4611704682869732882 9223092117352620518 9991152681891671022 257099731913529.5 12412830.045471078 38861 9223372036854775806 4611686018427387904 4672855013852508626 +800784 4611752907938305166 9223309994342931384 5251877538869750510 135472890315726.03 53535427.52018088 38767 9223372036854775806 4611686018427387904 7801864489649220514 +20810645 4611712185532639162 9223218900001937412 11803718472901310700 323593455407553 10496765.20741332 36477 9223372036854775806 4611686018427387904 5941995311893397960 +25843850 4611744529689964352 9223346023778617822 127137885677350808 3700925266420.715 18966925.191309396 34353 9223372036854775806 4611686018427387904 6700111718676827412 +23447120 4611796031755620254 9223329309291309758 1841522159325376278 54534534450526.42 6271211.193812284 33768 9223372036854775806 4611686018427387904 2325654077031843898 +14739804 4611762063154116632 9223007205463222212 16302703534054321116 506987919332451.8 6885575.861759452 32156 9223372036854775806 4611686018427387904 2114922310535979832 +32077710 4612033458080771112 9223352444952988904 421072759851674408 13955745719596.793 12220152.393889504 30172 9223372036854775806 4611686018427387904 4399934528735249092 +22446879 4611846229717089436 9223124373140579096 6577134317587565298 224866980668999.47 2482202.163802278 29249 9223372036854775806 4611686018427387904 8763910740678180498 +170282 4611833225706935900 9223371583739401906 15764226366913732386 551447384017691 2515144222.953728 28587 9223372036854775806 4611686018427387904 8217388408377809010 +11482817 4611990575414646848 9223302669582414438 9828522700609834800 378121905921203.2 34845264.2080656 25993 9223372036854775806 4611686018427387904 4689180182672571856 +63469 4612175339998036670 9222961628400798084 17239621485933250238 663164390134376.5 7825349797.6059 25996 9223372036854775806 4611686018427387904 2067736879306995526 +29103473 4611744585914335132 9223035551850347954 12590190375872647672 525927999326314.7 26049107.15514301 23939 9223372036854775806 4611686018427387904 8318055464870862444 diff --git a/tests/queries/1_stateful/00165_jit_aggregate_functions.sql b/tests/queries/1_stateful/00165_jit_aggregate_functions.sql index 90917209d1b..6c13c6e4d42 100644 --- a/tests/queries/1_stateful/00165_jit_aggregate_functions.sql +++ b/tests/queries/1_stateful/00165_jit_aggregate_functions.sql @@ -5,18 +5,52 @@ SELECT 'Aggregation using JIT compilation'; SELECT 'Simple functions'; -SELECT CounterID, min(WatchID), max(WatchID), sum(WatchID), avg(WatchID), avgWeighted(WatchID, CounterID), count(WatchID) FROM test.hits +SELECT + CounterID, + min(WatchID), + max(WatchID), + sum(WatchID), + avg(WatchID), + avgWeighted(WatchID, CounterID), + count(WatchID), + groupBitOr(WatchID), + groupBitAnd(WatchID), + groupBitXor(WatchID) +FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; SELECT 'Simple functions with non compilable function'; -SELECT CounterID, min(WatchID), max(WatchID), sum(WatchID), groupBitAnd(WatchID), avg(WatchID), avgWeighted(WatchID, CounterID), count(WatchID) FROM test.hits +SELECT + CounterID, + min(WatchID), + max(WatchID), + sum(WatchID), + sum(toUInt128(WatchID)), + avg(WatchID), + avgWeighted(WatchID, CounterID), + count(WatchID), + groupBitOr(WatchID), + groupBitAnd(WatchID), + groupBitXor(WatchID) +FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; SELECT 'Simple functions if combinator'; WITH (WatchID % 2 == 0) AS predicate -SELECT CounterID, minIf(WatchID,predicate), maxIf(WatchID, predicate), sumIf(WatchID, predicate), avgIf(WatchID, predicate), avgWeightedIf(WatchID, CounterID, predicate), countIf(WatchID, predicate) FROM test.hits +SELECT + CounterID, + minIf(WatchID,predicate), + maxIf(WatchID, predicate), + sumIf(WatchID, predicate), + avgIf(WatchID, predicate), + avgWeightedIf(WatchID, CounterID, predicate), + countIf(WatchID, predicate), + groupBitOrIf(WatchID, predicate), + groupBitAndIf(WatchID, predicate), + groupBitXorIf(WatchID, predicate) +FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; SET compile_aggregate_expressions = 0; @@ -25,15 +59,49 @@ SELECT 'Aggregation without JIT compilation'; SELECT 'Simple functions'; -SELECT CounterID, min(WatchID), max(WatchID), sum(WatchID), avg(WatchID), avgWeighted(WatchID, CounterID), count(WatchID) FROM test.hits +SELECT + CounterID, + min(WatchID), + max(WatchID), + sum(WatchID), + avg(WatchID), + avgWeighted(WatchID, CounterID), + count(WatchID), + groupBitOr(WatchID), + groupBitAnd(WatchID), + groupBitXor(WatchID) +FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; SELECT 'Simple functions with non compilable function'; -SELECT CounterID, min(WatchID), max(WatchID), sum(WatchID), groupBitAnd(WatchID), avg(WatchID), avgWeighted(WatchID, CounterID), count(WatchID) FROM test.hits +SELECT + CounterID, + min(WatchID), + max(WatchID), + sum(WatchID), + sum(toUInt128(WatchID)), + avg(WatchID), + avgWeighted(WatchID, CounterID), + count(WatchID), + groupBitOr(WatchID), + groupBitAnd(WatchID), + groupBitXor(WatchID) +FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; SELECT 'Simple functions if combinator'; WITH (WatchID % 2 == 0) AS predicate -SELECT CounterID, minIf(WatchID,predicate), maxIf(WatchID, predicate), sumIf(WatchID, predicate), avgWeightedIf(WatchID, CounterID, predicate), countIf(WatchID, predicate) FROM test.hits +SELECT + CounterID, + minIf(WatchID,predicate), + maxIf(WatchID, predicate), + sumIf(WatchID, predicate), + avgIf(WatchID, predicate), + avgWeightedIf(WatchID, CounterID, predicate), + countIf(WatchID, predicate), + groupBitOrIf(WatchID, predicate), + groupBitAndIf(WatchID, predicate), + groupBitXorIf(WatchID, predicate) +FROM test.hits GROUP BY CounterID ORDER BY count() DESC LIMIT 20; diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index b2f00dcfb87..fd800d3bc33 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -484,6 +484,7 @@ "01702_system_query_log", // It's ok to execute in parallel with oter tests but not several instances of the same test. "01748_dictionary_table_dot", // creates database "00950_dict_get", + "01615_random_one_shard_insertion", "01683_flat_dictionary", "01681_cache_dictionary_simple_key", "01682_cache_dictionary_complex_key", @@ -504,6 +505,7 @@ "01824_prefer_global_in_and_join", "01870_modulo_partition_key", "01870_buffer_flush", // creates database + "01889_sqlite_read_write", "01889_postgresql_protocol_null_fields", "01889_check_row_policy_defined_using_user_function", "01921_concurrent_ttl_and_normal_merges_zookeeper_long", // heavy test, better to run sequentially diff --git a/tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml b/tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml index ceaa497c561..e45c4519c73 100644 --- a/tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml +++ b/tests/testflows/kerberos/configs/clickhouse1/config.d/kerberos.xml @@ -1,5 +1,6 @@ + EXAMPLE.COM - \ No newline at end of file + diff --git a/tests/testflows/kerberos/configs/clickhouse3/config.d/kerberos.xml b/tests/testflows/kerberos/configs/clickhouse3/config.d/kerberos.xml new file mode 100644 index 00000000000..e45c4519c73 --- /dev/null +++ b/tests/testflows/kerberos/configs/clickhouse3/config.d/kerberos.xml @@ -0,0 +1,6 @@ + + + + EXAMPLE.COM + + diff --git a/tests/testflows/kerberos/configs/kerberos/etc/krb5.conf b/tests/testflows/kerberos/configs/kerberos/etc/krb5.conf index b963fc25daa..602ca76abbe 100644 --- a/tests/testflows/kerberos/configs/kerberos/etc/krb5.conf +++ b/tests/testflows/kerberos/configs/kerberos/etc/krb5.conf @@ -3,17 +3,14 @@ [libdefaults] default_realm = EXAMPLE.COM - ticket_lifetime = 24000 - dns_lookup_realm = false - dns_lookup_kdc = false - dns_fallback = false - rdns = false + ticket_lifetime = 36000 + dns_lookup_kdc = false [realms] - EXAMPLE.COM = { - kdc = kerberos - admin_server = kerberos - } + EXAMPLE.COM = { + kdc = kerberos_env_kerberos_1.krbnet + admin_server = kerberos_env_kerberos_1.krbnet + } OTHER.COM = { kdc = kerberos admin_server = kerberos @@ -22,6 +19,10 @@ [domain_realm] docker-compose_default = EXAMPLE.COM .docker-compose_default = EXAMPLE.COM + krbnet = EXAMPLE.COM + .krbnet = EXAMPLE.COM + kerberos_env_default = EXAMPLE.COM + .kerberos_env_default = EXAMPLE.COM [appdefaults] validate = false diff --git a/tests/testflows/kerberos/kerberos_env/docker-compose.yml b/tests/testflows/kerberos/kerberos_env/docker-compose.yml index d1a74662a83..e89d18a5299 100644 --- a/tests/testflows/kerberos/kerberos_env/docker-compose.yml +++ b/tests/testflows/kerberos/kerberos_env/docker-compose.yml @@ -73,3 +73,8 @@ services: condition: service_healthy kerberos: condition: service_healthy + +networks: + default: + name: krbnet + driver: bridge diff --git a/tests/testflows/kerberos/kerberos_env/kerberos-service.yml b/tests/testflows/kerberos/kerberos_env/kerberos-service.yml index 3f21e93e0b6..b34751258da 100644 --- a/tests/testflows/kerberos/kerberos_env/kerberos-service.yml +++ b/tests/testflows/kerberos/kerberos_env/kerberos-service.yml @@ -3,7 +3,6 @@ version: '2.3' services: kerberos: image: zvonand/docker-krb5-server:1.0.0 - restart: always expose: - "88" - "464" @@ -17,7 +16,7 @@ services: environment: KRB5_PASS: pwd KRB5_REALM: EXAMPLE.COM - KRB5_KDC: localhost + KRB5_KDC: 0.0.0.0 volumes: - "${CLICKHOUSE_TESTS_DIR}/configs/kerberos/etc/krb5kdc/kdc.conf:/etc/krb5kdc/kdc.conf" - "${CLICKHOUSE_TESTS_DIR}/_instances/kerberos/krb5kdc/log/kdc.log:/usr/local/var/krb5kdc/kdc.log" diff --git a/tests/testflows/kerberos/regression.py b/tests/testflows/kerberos/regression.py index ca174aaff08..0e8b0a55c2e 100755 --- a/tests/testflows/kerberos/regression.py +++ b/tests/testflows/kerberos/regression.py @@ -10,6 +10,7 @@ from helpers.argparser import argparser from kerberos.requirements.requirements import * xfails = { + "config/principal and realm specified/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/26197")], } @@ -43,5 +44,6 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): Feature(run=load("kerberos.tests.parallel", "parallel"), flags=TE) + if main(): regression() diff --git a/tests/testflows/kerberos/requirements/requirements.md b/tests/testflows/kerberos/requirements/requirements.md index 2121dd343b8..8f2b3b7e11e 100644 --- a/tests/testflows/kerberos/requirements/requirements.md +++ b/tests/testflows/kerberos/requirements/requirements.md @@ -9,38 +9,41 @@ * 4 [Requirements](#requirements) * 4.1 [Generic](#generic) * 4.1.1 [RQ.SRS-016.Kerberos](#rqsrs-016kerberos) - * 4.2 [Configuration](#configuration) - * 4.2.1 [RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods](#rqsrs-016kerberosconfigurationmultipleauthmethods) - * 4.2.2 [RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled](#rqsrs-016kerberosconfigurationkerberosnotenabled) - * 4.2.3 [RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections](#rqsrs-016kerberosconfigurationmultiplekerberossections) - * 4.2.4 [RQ.SRS-016.Kerberos.Configuration.WrongUserRealm](#rqsrs-016kerberosconfigurationwronguserrealm) - * 4.2.5 [RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified](#rqsrs-016kerberosconfigurationprincipalandrealmspecified) - * 4.2.6 [RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections](#rqsrs-016kerberosconfigurationmultipleprincipalsections) - * 4.2.7 [RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections](#rqsrs-016kerberosconfigurationmultiplerealmsections) - * 4.3 [Valid User](#valid-user) - * 4.3.1 [RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser](#rqsrs-016kerberosvaliduserxmlconfigureduser) - * 4.3.2 [RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser](#rqsrs-016kerberosvaliduserrbacconfigureduser) - * 4.3.3 [RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured](#rqsrs-016kerberosvaliduserkerberosnotconfigured) - * 4.4 [Invalid User](#invalid-user) - * 4.4.1 [RQ.SRS-016.Kerberos.InvalidUser](#rqsrs-016kerberosinvaliduser) - * 4.4.2 [RQ.SRS-016.Kerberos.InvalidUser.UserDeleted](#rqsrs-016kerberosinvaliduseruserdeleted) - * 4.5 [Kerberos Not Available](#kerberos-not-available) - * 4.5.1 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket](#rqsrs-016kerberoskerberosnotavailableinvalidserverticket) - * 4.5.2 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket](#rqsrs-016kerberoskerberosnotavailableinvalidclientticket) - * 4.5.3 [RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets](#rqsrs-016kerberoskerberosnotavailablevalidtickets) - * 4.6 [Kerberos Restarted](#kerberos-restarted) - * 4.6.1 [RQ.SRS-016.Kerberos.KerberosServerRestarted](#rqsrs-016kerberoskerberosserverrestarted) - * 4.7 [Performance](#performance) - * 4.7.1 [RQ.SRS-016.Kerberos.Performance](#rqsrs-016kerberosperformance) - * 4.8 [Parallel Requests processing](#parallel-requests-processing) - * 4.8.1 [RQ.SRS-016.Kerberos.Parallel](#rqsrs-016kerberosparallel) - * 4.8.2 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos](#rqsrs-016kerberosparallelvalidrequestskerberosandnonkerberos) - * 4.8.3 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials](#rqsrs-016kerberosparallelvalidrequestssamecredentials) - * 4.8.4 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials](#rqsrs-016kerberosparallelvalidrequestsdifferentcredentials) - * 4.8.5 [RQ.SRS-016.Kerberos.Parallel.ValidInvalid](#rqsrs-016kerberosparallelvalidinvalid) - * 4.8.6 [RQ.SRS-016.Kerberos.Parallel.Deletion](#rqsrs-016kerberosparalleldeletion) + * 4.2 [Ping](#ping) + * 4.2.1 [RQ.SRS-016.Kerberos.Ping](#rqsrs-016kerberosping) + * 4.3 [Configuration](#configuration) + * 4.3.1 [RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods](#rqsrs-016kerberosconfigurationmultipleauthmethods) + * 4.3.2 [RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled](#rqsrs-016kerberosconfigurationkerberosnotenabled) + * 4.3.3 [RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections](#rqsrs-016kerberosconfigurationmultiplekerberossections) + * 4.3.4 [RQ.SRS-016.Kerberos.Configuration.WrongUserRealm](#rqsrs-016kerberosconfigurationwronguserrealm) + * 4.3.5 [RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified](#rqsrs-016kerberosconfigurationprincipalandrealmspecified) + * 4.3.6 [RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections](#rqsrs-016kerberosconfigurationmultipleprincipalsections) + * 4.3.7 [RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections](#rqsrs-016kerberosconfigurationmultiplerealmsections) + * 4.4 [Valid User](#valid-user) + * 4.4.1 [RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser](#rqsrs-016kerberosvaliduserxmlconfigureduser) + * 4.4.2 [RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser](#rqsrs-016kerberosvaliduserrbacconfigureduser) + * 4.4.3 [RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured](#rqsrs-016kerberosvaliduserkerberosnotconfigured) + * 4.5 [Invalid User](#invalid-user) + * 4.5.1 [RQ.SRS-016.Kerberos.InvalidUser](#rqsrs-016kerberosinvaliduser) + * 4.5.2 [RQ.SRS-016.Kerberos.InvalidUser.UserDeleted](#rqsrs-016kerberosinvaliduseruserdeleted) + * 4.6 [Kerberos Not Available](#kerberos-not-available) + * 4.6.1 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket](#rqsrs-016kerberoskerberosnotavailableinvalidserverticket) + * 4.6.2 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket](#rqsrs-016kerberoskerberosnotavailableinvalidclientticket) + * 4.6.3 [RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets](#rqsrs-016kerberoskerberosnotavailablevalidtickets) + * 4.7 [Kerberos Restarted](#kerberos-restarted) + * 4.7.1 [RQ.SRS-016.Kerberos.KerberosServerRestarted](#rqsrs-016kerberoskerberosserverrestarted) + * 4.8 [Performance](#performance) + * 4.8.1 [RQ.SRS-016.Kerberos.Performance](#rqsrs-016kerberosperformance) + * 4.9 [Parallel Requests processing](#parallel-requests-processing) + * 4.9.1 [RQ.SRS-016.Kerberos.Parallel](#rqsrs-016kerberosparallel) + * 4.9.2 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos](#rqsrs-016kerberosparallelvalidrequestskerberosandnonkerberos) + * 4.9.3 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials](#rqsrs-016kerberosparallelvalidrequestssamecredentials) + * 4.9.4 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials](#rqsrs-016kerberosparallelvalidrequestsdifferentcredentials) + * 4.9.5 [RQ.SRS-016.Kerberos.Parallel.ValidInvalid](#rqsrs-016kerberosparallelvalidinvalid) + * 4.9.6 [RQ.SRS-016.Kerberos.Parallel.Deletion](#rqsrs-016kerberosparalleldeletion) * 5 [References](#references) + ## Revision History This document is stored in an electronic form using [Git] source control management software @@ -85,6 +88,13 @@ version: 1.0 [ClickHouse] SHALL support user authentication using [Kerberos] server. +### Ping + +#### RQ.SRS-016.Kerberos.Ping +version: 1.0 + +Docker containers SHALL be able to ping each other. + ### Configuration #### RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods @@ -278,4 +288,3 @@ version: 1.0 [Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/kerberos/requirements/requirements.md [Git]: https://git-scm.com/ [Kerberos terminology]: https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm - diff --git a/tests/testflows/kerberos/requirements/requirements.py b/tests/testflows/kerberos/requirements/requirements.py index 5c49e7d127f..418c51ca8b3 100644 --- a/tests/testflows/kerberos/requirements/requirements.py +++ b/tests/testflows/kerberos/requirements/requirements.py @@ -1,6 +1,6 @@ # These requirements were auto generated # from software requirements specification (SRS) -# document by TestFlows v1.6.201216.1172002. +# document by TestFlows v1.6.210312.1172513. # Do not edit by hand but re-generate instead # using 'tfs requirements generate' command. from testflows.core import Specification @@ -23,6 +23,21 @@ RQ_SRS_016_Kerberos = Requirement( level=3, num='4.1.1') +RQ_SRS_016_Kerberos_Ping = Requirement( + name='RQ.SRS-016.Kerberos.Ping', + version='1.0', + priority=None, + group=None, + type=None, + uid=None, + description=( + 'Docker containers SHALL be able to ping each other.\n' + '\n' + ), + link=None, + level=3, + num='4.2.1') + RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods = Requirement( name='RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods', version='1.0', @@ -36,7 +51,7 @@ RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods = Requirement( ), link=None, level=3, - num='4.2.1') + num='4.3.1') RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled = Requirement( name='RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled', @@ -74,7 +89,7 @@ RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled = Requirement( ), link=None, level=3, - num='4.2.2') + num='4.3.2') RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections = Requirement( name='RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections', @@ -89,7 +104,7 @@ RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections = Requirement( ), link=None, level=3, - num='4.2.3') + num='4.3.3') RQ_SRS_016_Kerberos_Configuration_WrongUserRealm = Requirement( name='RQ.SRS-016.Kerberos.Configuration.WrongUserRealm', @@ -104,7 +119,7 @@ RQ_SRS_016_Kerberos_Configuration_WrongUserRealm = Requirement( ), link=None, level=3, - num='4.2.4') + num='4.3.4') RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified = Requirement( name='RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified', @@ -119,7 +134,7 @@ RQ_SRS_016_Kerberos_Configuration_PrincipalAndRealmSpecified = Requirement( ), link=None, level=3, - num='4.2.5') + num='4.3.5') RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections = Requirement( name='RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections', @@ -134,7 +149,7 @@ RQ_SRS_016_Kerberos_Configuration_MultiplePrincipalSections = Requirement( ), link=None, level=3, - num='4.2.6') + num='4.3.6') RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections = Requirement( name='RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections', @@ -149,7 +164,7 @@ RQ_SRS_016_Kerberos_Configuration_MultipleRealmSections = Requirement( ), link=None, level=3, - num='4.2.7') + num='4.3.7') RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser = Requirement( name='RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser', @@ -179,7 +194,7 @@ RQ_SRS_016_Kerberos_ValidUser_XMLConfiguredUser = Requirement( ), link=None, level=3, - num='4.3.1') + num='4.4.1') RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser = Requirement( name='RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser', @@ -204,7 +219,7 @@ RQ_SRS_016_Kerberos_ValidUser_RBACConfiguredUser = Requirement( ), link=None, level=3, - num='4.3.2') + num='4.4.2') RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured = Requirement( name='RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured', @@ -219,7 +234,7 @@ RQ_SRS_016_Kerberos_ValidUser_KerberosNotConfigured = Requirement( ), link=None, level=3, - num='4.3.3') + num='4.4.3') RQ_SRS_016_Kerberos_InvalidUser = Requirement( name='RQ.SRS-016.Kerberos.InvalidUser', @@ -234,7 +249,7 @@ RQ_SRS_016_Kerberos_InvalidUser = Requirement( ), link=None, level=3, - num='4.4.1') + num='4.5.1') RQ_SRS_016_Kerberos_InvalidUser_UserDeleted = Requirement( name='RQ.SRS-016.Kerberos.InvalidUser.UserDeleted', @@ -249,7 +264,7 @@ RQ_SRS_016_Kerberos_InvalidUser_UserDeleted = Requirement( ), link=None, level=3, - num='4.4.2') + num='4.5.2') RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket = Requirement( name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket', @@ -264,7 +279,7 @@ RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidServerTicket = Requirement( ), link=None, level=3, - num='4.5.1') + num='4.6.1') RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket = Requirement( name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket', @@ -279,7 +294,7 @@ RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket = Requirement( ), link=None, level=3, - num='4.5.2') + num='4.6.2') RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets = Requirement( name='RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets', @@ -294,7 +309,7 @@ RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets = Requirement( ), link=None, level=3, - num='4.5.3') + num='4.6.3') RQ_SRS_016_Kerberos_KerberosServerRestarted = Requirement( name='RQ.SRS-016.Kerberos.KerberosServerRestarted', @@ -309,7 +324,7 @@ RQ_SRS_016_Kerberos_KerberosServerRestarted = Requirement( ), link=None, level=3, - num='4.6.1') + num='4.7.1') RQ_SRS_016_Kerberos_Performance = Requirement( name='RQ.SRS-016.Kerberos.Performance', @@ -324,7 +339,7 @@ RQ_SRS_016_Kerberos_Performance = Requirement( ), link=None, level=3, - num='4.7.1') + num='4.8.1') RQ_SRS_016_Kerberos_Parallel = Requirement( name='RQ.SRS-016.Kerberos.Parallel', @@ -339,7 +354,7 @@ RQ_SRS_016_Kerberos_Parallel = Requirement( ), link=None, level=3, - num='4.8.1') + num='4.9.1') RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos = Requirement( name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos', @@ -354,7 +369,7 @@ RQ_SRS_016_Kerberos_Parallel_ValidRequests_KerberosAndNonKerberos = Requirement( ), link=None, level=3, - num='4.8.2') + num='4.9.2') RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials = Requirement( name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials', @@ -369,7 +384,7 @@ RQ_SRS_016_Kerberos_Parallel_ValidRequests_SameCredentials = Requirement( ), link=None, level=3, - num='4.8.3') + num='4.9.3') RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials = Requirement( name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials', @@ -384,7 +399,7 @@ RQ_SRS_016_Kerberos_Parallel_ValidRequests_DifferentCredentials = Requirement( ), link=None, level=3, - num='4.8.4') + num='4.9.4') RQ_SRS_016_Kerberos_Parallel_ValidInvalid = Requirement( name='RQ.SRS-016.Kerberos.Parallel.ValidInvalid', @@ -399,7 +414,7 @@ RQ_SRS_016_Kerberos_Parallel_ValidInvalid = Requirement( ), link=None, level=3, - num='4.8.5') + num='4.9.5') RQ_SRS_016_Kerberos_Parallel_Deletion = Requirement( name='RQ.SRS-016.Kerberos.Parallel.Deletion', @@ -414,17 +429,17 @@ RQ_SRS_016_Kerberos_Parallel_Deletion = Requirement( ), link=None, level=3, - num='4.8.6') + num='4.9.6') QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( name='QA-SRS016 ClickHouse Kerberos Authentication', description=None, - author='Andrey Zvonov', - date='December 14, 2020', - status='-', - approved_by='-', - approved_date='-', - approved_version='-', + author=None, + date=None, + status=None, + approved_by=None, + approved_date=None, + approved_version=None, version=None, group=None, type=None, @@ -439,40 +454,43 @@ QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( Heading(name='Requirements', level=1, num='4'), Heading(name='Generic', level=2, num='4.1'), Heading(name='RQ.SRS-016.Kerberos', level=3, num='4.1.1'), - Heading(name='Configuration', level=2, num='4.2'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods', level=3, num='4.2.1'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled', level=3, num='4.2.2'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections', level=3, num='4.2.3'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.WrongUserRealm', level=3, num='4.2.4'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified', level=3, num='4.2.5'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections', level=3, num='4.2.6'), - Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections', level=3, num='4.2.7'), - Heading(name='Valid User', level=2, num='4.3'), - Heading(name='RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser', level=3, num='4.3.1'), - Heading(name='RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser', level=3, num='4.3.2'), - Heading(name='RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured', level=3, num='4.3.3'), - Heading(name='Invalid User', level=2, num='4.4'), - Heading(name='RQ.SRS-016.Kerberos.InvalidUser', level=3, num='4.4.1'), - Heading(name='RQ.SRS-016.Kerberos.InvalidUser.UserDeleted', level=3, num='4.4.2'), - Heading(name='Kerberos Not Available', level=2, num='4.5'), - Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket', level=3, num='4.5.1'), - Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket', level=3, num='4.5.2'), - Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets', level=3, num='4.5.3'), - Heading(name='Kerberos Restarted', level=2, num='4.6'), - Heading(name='RQ.SRS-016.Kerberos.KerberosServerRestarted', level=3, num='4.6.1'), - Heading(name='Performance', level=2, num='4.7'), - Heading(name='RQ.SRS-016.Kerberos.Performance', level=3, num='4.7.1'), - Heading(name='Parallel Requests processing', level=2, num='4.8'), - Heading(name='RQ.SRS-016.Kerberos.Parallel', level=3, num='4.8.1'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos', level=3, num='4.8.2'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials', level=3, num='4.8.3'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials', level=3, num='4.8.4'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidInvalid', level=3, num='4.8.5'), - Heading(name='RQ.SRS-016.Kerberos.Parallel.Deletion', level=3, num='4.8.6'), + Heading(name='Ping', level=2, num='4.2'), + Heading(name='RQ.SRS-016.Kerberos.Ping', level=3, num='4.2.1'), + Heading(name='Configuration', level=2, num='4.3'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods', level=3, num='4.3.1'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled', level=3, num='4.3.2'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections', level=3, num='4.3.3'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.WrongUserRealm', level=3, num='4.3.4'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified', level=3, num='4.3.5'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections', level=3, num='4.3.6'), + Heading(name='RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections', level=3, num='4.3.7'), + Heading(name='Valid User', level=2, num='4.4'), + Heading(name='RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser', level=3, num='4.4.1'), + Heading(name='RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser', level=3, num='4.4.2'), + Heading(name='RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured', level=3, num='4.4.3'), + Heading(name='Invalid User', level=2, num='4.5'), + Heading(name='RQ.SRS-016.Kerberos.InvalidUser', level=3, num='4.5.1'), + Heading(name='RQ.SRS-016.Kerberos.InvalidUser.UserDeleted', level=3, num='4.5.2'), + Heading(name='Kerberos Not Available', level=2, num='4.6'), + Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket', level=3, num='4.6.1'), + Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket', level=3, num='4.6.2'), + Heading(name='RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets', level=3, num='4.6.3'), + Heading(name='Kerberos Restarted', level=2, num='4.7'), + Heading(name='RQ.SRS-016.Kerberos.KerberosServerRestarted', level=3, num='4.7.1'), + Heading(name='Performance', level=2, num='4.8'), + Heading(name='RQ.SRS-016.Kerberos.Performance', level=3, num='4.8.1'), + Heading(name='Parallel Requests processing', level=2, num='4.9'), + Heading(name='RQ.SRS-016.Kerberos.Parallel', level=3, num='4.9.1'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos', level=3, num='4.9.2'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials', level=3, num='4.9.3'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials', level=3, num='4.9.4'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.ValidInvalid', level=3, num='4.9.5'), + Heading(name='RQ.SRS-016.Kerberos.Parallel.Deletion', level=3, num='4.9.6'), Heading(name='References', level=1, num='5'), ), requirements=( RQ_SRS_016_Kerberos, + RQ_SRS_016_Kerberos_Ping, RQ_SRS_016_Kerberos_Configuration_MultipleAuthMethods, RQ_SRS_016_Kerberos_Configuration_KerberosNotEnabled, RQ_SRS_016_Kerberos_Configuration_MultipleKerberosSections, @@ -501,25 +519,6 @@ QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( # QA-SRS016 ClickHouse Kerberos Authentication # Software Requirements Specification -(c) 2020 Altinity LTD. All Rights Reserved. - -**Document status:** Confidential - -**Author:** Andrey Zvonov - -**Date:** December 14, 2020 - -## Approval - -**Status:** - - -**Version:** - - -**Approved by:** - - -**Date:** - - - ## Table of Contents * 1 [Revision History](#revision-history) @@ -528,47 +527,50 @@ QA_SRS016_ClickHouse_Kerberos_Authentication = Specification( * 4 [Requirements](#requirements) * 4.1 [Generic](#generic) * 4.1.1 [RQ.SRS-016.Kerberos](#rqsrs-016kerberos) - * 4.2 [Configuration](#configuration) - * 4.2.1 [RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods](#rqsrs-016kerberosconfigurationmultipleauthmethods) - * 4.2.2 [RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled](#rqsrs-016kerberosconfigurationkerberosnotenabled) - * 4.2.3 [RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections](#rqsrs-016kerberosconfigurationmultiplekerberossections) - * 4.2.4 [RQ.SRS-016.Kerberos.Configuration.WrongUserRealm](#rqsrs-016kerberosconfigurationwronguserrealm) - * 4.2.5 [RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified](#rqsrs-016kerberosconfigurationprincipalandrealmspecified) - * 4.2.6 [RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections](#rqsrs-016kerberosconfigurationmultipleprincipalsections) - * 4.2.7 [RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections](#rqsrs-016kerberosconfigurationmultiplerealmsections) - * 4.3 [Valid User](#valid-user) - * 4.3.1 [RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser](#rqsrs-016kerberosvaliduserxmlconfigureduser) - * 4.3.2 [RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser](#rqsrs-016kerberosvaliduserrbacconfigureduser) - * 4.3.3 [RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured](#rqsrs-016kerberosvaliduserkerberosnotconfigured) - * 4.4 [Invalid User](#invalid-user) - * 4.4.1 [RQ.SRS-016.Kerberos.InvalidUser](#rqsrs-016kerberosinvaliduser) - * 4.4.2 [RQ.SRS-016.Kerberos.InvalidUser.UserDeleted](#rqsrs-016kerberosinvaliduseruserdeleted) - * 4.5 [Kerberos Not Available](#kerberos-not-available) - * 4.5.1 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket](#rqsrs-016kerberoskerberosnotavailableinvalidserverticket) - * 4.5.2 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket](#rqsrs-016kerberoskerberosnotavailableinvalidclientticket) - * 4.5.3 [RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets](#rqsrs-016kerberoskerberosnotavailablevalidtickets) - * 4.6 [Kerberos Restarted](#kerberos-restarted) - * 4.6.1 [RQ.SRS-016.Kerberos.KerberosServerRestarted](#rqsrs-016kerberoskerberosserverrestarted) - * 4.7 [Performance](#performance) - * 4.7.1 [RQ.SRS-016.Kerberos.Performance](#rqsrs-016kerberosperformance) - * 4.8 [Parallel Requests processing](#parallel-requests-processing) - * 4.8.1 [RQ.SRS-016.Kerberos.Parallel](#rqsrs-016kerberosparallel) - * 4.8.2 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos](#rqsrs-016kerberosparallelvalidrequestskerberosandnonkerberos) - * 4.8.3 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials](#rqsrs-016kerberosparallelvalidrequestssamecredentials) - * 4.8.4 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials](#rqsrs-016kerberosparallelvalidrequestsdifferentcredentials) - * 4.8.5 [RQ.SRS-016.Kerberos.Parallel.ValidInvalid](#rqsrs-016kerberosparallelvalidinvalid) - * 4.8.6 [RQ.SRS-016.Kerberos.Parallel.Deletion](#rqsrs-016kerberosparalleldeletion) + * 4.2 [Ping](#ping) + * 4.2.1 [RQ.SRS-016.Kerberos.Ping](#rqsrs-016kerberosping) + * 4.3 [Configuration](#configuration) + * 4.3.1 [RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods](#rqsrs-016kerberosconfigurationmultipleauthmethods) + * 4.3.2 [RQ.SRS-016.Kerberos.Configuration.KerberosNotEnabled](#rqsrs-016kerberosconfigurationkerberosnotenabled) + * 4.3.3 [RQ.SRS-016.Kerberos.Configuration.MultipleKerberosSections](#rqsrs-016kerberosconfigurationmultiplekerberossections) + * 4.3.4 [RQ.SRS-016.Kerberos.Configuration.WrongUserRealm](#rqsrs-016kerberosconfigurationwronguserrealm) + * 4.3.5 [RQ.SRS-016.Kerberos.Configuration.PrincipalAndRealmSpecified](#rqsrs-016kerberosconfigurationprincipalandrealmspecified) + * 4.3.6 [RQ.SRS-016.Kerberos.Configuration.MultiplePrincipalSections](#rqsrs-016kerberosconfigurationmultipleprincipalsections) + * 4.3.7 [RQ.SRS-016.Kerberos.Configuration.MultipleRealmSections](#rqsrs-016kerberosconfigurationmultiplerealmsections) + * 4.4 [Valid User](#valid-user) + * 4.4.1 [RQ.SRS-016.Kerberos.ValidUser.XMLConfiguredUser](#rqsrs-016kerberosvaliduserxmlconfigureduser) + * 4.4.2 [RQ.SRS-016.Kerberos.ValidUser.RBACConfiguredUser](#rqsrs-016kerberosvaliduserrbacconfigureduser) + * 4.4.3 [RQ.SRS-016.Kerberos.ValidUser.KerberosNotConfigured](#rqsrs-016kerberosvaliduserkerberosnotconfigured) + * 4.5 [Invalid User](#invalid-user) + * 4.5.1 [RQ.SRS-016.Kerberos.InvalidUser](#rqsrs-016kerberosinvaliduser) + * 4.5.2 [RQ.SRS-016.Kerberos.InvalidUser.UserDeleted](#rqsrs-016kerberosinvaliduseruserdeleted) + * 4.6 [Kerberos Not Available](#kerberos-not-available) + * 4.6.1 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidServerTicket](#rqsrs-016kerberoskerberosnotavailableinvalidserverticket) + * 4.6.2 [RQ.SRS-016.Kerberos.KerberosNotAvailable.InvalidClientTicket](#rqsrs-016kerberoskerberosnotavailableinvalidclientticket) + * 4.6.3 [RQ.SRS-016.Kerberos.KerberosNotAvailable.ValidTickets](#rqsrs-016kerberoskerberosnotavailablevalidtickets) + * 4.7 [Kerberos Restarted](#kerberos-restarted) + * 4.7.1 [RQ.SRS-016.Kerberos.KerberosServerRestarted](#rqsrs-016kerberoskerberosserverrestarted) + * 4.8 [Performance](#performance) + * 4.8.1 [RQ.SRS-016.Kerberos.Performance](#rqsrs-016kerberosperformance) + * 4.9 [Parallel Requests processing](#parallel-requests-processing) + * 4.9.1 [RQ.SRS-016.Kerberos.Parallel](#rqsrs-016kerberosparallel) + * 4.9.2 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.KerberosAndNonKerberos](#rqsrs-016kerberosparallelvalidrequestskerberosandnonkerberos) + * 4.9.3 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.SameCredentials](#rqsrs-016kerberosparallelvalidrequestssamecredentials) + * 4.9.4 [RQ.SRS-016.Kerberos.Parallel.ValidRequests.DifferentCredentials](#rqsrs-016kerberosparallelvalidrequestsdifferentcredentials) + * 4.9.5 [RQ.SRS-016.Kerberos.Parallel.ValidInvalid](#rqsrs-016kerberosparallelvalidinvalid) + * 4.9.6 [RQ.SRS-016.Kerberos.Parallel.Deletion](#rqsrs-016kerberosparalleldeletion) * 5 [References](#references) + ## Revision History This document is stored in an electronic form using [Git] source control management software -hosted in a [GitLab Repository]. +hosted in a [GitHub Repository]. All the updates are tracked using the [Git]'s [Revision History]. ## Introduction -This document specifies the behavior for authenticating existing users using [Kerberos] authentication protocol. +This document specifies the behavior for authenticating existing users via [Kerberos] authentication protocol. Existing [ClickHouse] users, that are properly configured, have an ability to authenticate using [Kerberos]. Kerberos authentication is only supported for HTTP requests, and users configured to authenticate via Kerberos cannot be authenticated by any other means of authentication. In order to use Kerberos authentication, Kerberos needs to be properly configured in the environment: Kerberos server must be present and user's and server's credentials must be set up. Configuring the Kerberos environment is outside the scope of this document. @@ -604,6 +606,13 @@ version: 1.0 [ClickHouse] SHALL support user authentication using [Kerberos] server. +### Ping + +#### RQ.SRS-016.Kerberos.Ping +version: 1.0 + +Docker containers SHALL be able to ping each other. + ### Configuration #### RQ.SRS-016.Kerberos.Configuration.MultipleAuthMethods @@ -784,17 +793,17 @@ version: 1.0 ## References * **ClickHouse:** https://clickhouse.tech -* **Gitlab Repository:** https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/blob/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md -* **Revision History:** https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/commits/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md +* **GitHub Repository:** https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/kerberos/requirements/requirements.md +* **Revision History:** https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/kerberos/requirements/requirements.md * **Git:** https://git-scm.com/ * **Kerberos terminology:** https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm [Kerberos]: https://en.wikipedia.org/wiki/Kerberos_(protocol) [SPNEGO]: https://en.wikipedia.org/wiki/SPNEGO [ClickHouse]: https://clickhouse.tech -[GitLab]: https://gitlab.com -[GitLab Repository]: https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/blob/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md -[Revision History]: https://gitlab.com/altinity-qa/documents/qa-srs016-clickhouse-kerberos-authentication/-/commits/master/QA_SRS016_ClickHouse_Kerberos_Authentication.md +[GitHub]: https://gitlab.com +[GitHub Repository]: https://github.com/ClickHouse/ClickHouse/blob/master/tests/testflows/kerberos/requirements/requirements.md +[Revision History]: https://github.com/ClickHouse/ClickHouse/commits/master/tests/testflows/kerberos/requirements/requirements.md [Git]: https://git-scm.com/ [Kerberos terminology]: https://web.mit.edu/kerberos/kfw-4.1/kfw-4.1/kfw-4.1-help/html/kerberos_terminology.htm ''') diff --git a/tests/testflows/kerberos/tests/common.py b/tests/testflows/kerberos/tests/common.py index e768a78cad5..8b72f1c2ffd 100644 --- a/tests/testflows/kerberos/tests/common.py +++ b/tests/testflows/kerberos/tests/common.py @@ -68,8 +68,8 @@ def create_server_principal(self, node): """ try: node.cmd("echo pwd | kinit admin/admin") - node.cmd(f"kadmin -w pwd -q \"add_principal -randkey HTTP/docker-compose_{node.name}_1.docker-compose_default\"") - node.cmd(f"kadmin -w pwd -q \"ktadd -k /etc/krb5.keytab HTTP/docker-compose_{node.name}_1.docker-compose_default\"") + node.cmd(f"kadmin -w pwd -q \"add_principal -randkey HTTP/kerberos_env_{node.name}_1.krbnet\"") + node.cmd(f"kadmin -w pwd -q \"ktadd -k /etc/krb5.keytab HTTP/kerberos_env_{node.name}_1.krbnet\"") yield finally: node.cmd("kdestroy") @@ -170,7 +170,7 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" config_contents = xmltree.tostring(root, encoding='utf8', method='xml').decode('utf-8') command = f"cat < {full_config_path}\n{config_contents}\nHEREDOC" node.command(command, steps=False, exitcode=0) - # time.sleep(1) + time.sleep(1) with Then(f"{preprocessed_name} should be updated", description=f"timeout {timeout}"): started = time.time() @@ -183,11 +183,14 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" assert exitcode == 0, error() with When("I restart ClickHouse to apply the config changes"): + node.cmd("kdestroy") + # time.sleep(1) if output: node.restart(safe=False, wait_healthy=True) else: node.restart(safe=False, wait_healthy=False) + if output != "": with Then(f"check {output} is in output"): time.sleep(5) @@ -201,7 +204,7 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" break time.sleep(1) else: - assert False, error() + assert output in r.output, error() finally: with Finally("I restore original config"): @@ -223,3 +226,19 @@ def check_wrong_config(self, node, client, config_path, modify_file, log_error=" assert exitcode == 0, error() +@TestStep(Given) +def instrument_clickhouse_server_log(self, clickhouse_server_log="/var/log/clickhouse-server/clickhouse-server.log"): + """Instrument clickhouse-server.log for the current test + by adding start and end messages that include + current test name to the clickhouse-server.log of the specified node and + if the test fails then dump the messages from + the clickhouse-server.log for this test. + """ + all_nodes = self.context.ch_nodes + [self.context.krb_server] + + for node in all_nodes: + if node.name != "kerberos": + with When(f"output stats for {node.repr()}"): + node.command(f"echo -e \"\\n-- {current().name} -- top --\\n\" && top -bn1") + node.command(f"echo -e \"\\n-- {current().name} -- df --\\n\" && df -h") + node.command(f"echo -e \"\\n-- {current().name} -- free --\\n\" && free -mh") diff --git a/tests/testflows/kerberos/tests/config.py b/tests/testflows/kerberos/tests/config.py index 3f4bf15deb5..85af0b3214e 100644 --- a/tests/testflows/kerberos/tests/config.py +++ b/tests/testflows/kerberos/tests/config.py @@ -145,12 +145,8 @@ def multiple_principal(self): log_error="Multiple principal sections are not allowed") - - - - - @TestFeature +@Name("config") def config(self): """Perform ClickHouse Kerberos authentication testing for incorrect configuration files """ diff --git a/tests/testflows/kerberos/tests/generic.py b/tests/testflows/kerberos/tests/generic.py index 3276fd5ec5f..642b99b4fc3 100644 --- a/tests/testflows/kerberos/tests/generic.py +++ b/tests/testflows/kerberos/tests/generic.py @@ -3,8 +3,22 @@ from kerberos.tests.common import * from kerberos.requirements.requirements import * import time -import datetime -import itertools + + +@TestScenario +@Requirements( + RQ_SRS_016_Kerberos_Ping("1.0") +) +def ping(self): + """Containers should be reachable + """ + ch_nodes = self.context.ch_nodes + + for i in range(3): + with When(f"curl ch_{i} kerberos"): + r = ch_nodes[i].command(f"curl kerberos -c 1") + with Then(f"return code should be 0"): + assert r.exitcode == 7, error() @TestScenario @@ -84,8 +98,10 @@ def invalid_server_ticket(self): ch_nodes[2].cmd("kdestroy") while True: kinit_no_keytab(node=ch_nodes[2]) + create_server_principal(node=ch_nodes[0]) if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": break + debug(test_select_query(node=ch_nodes[0])) ch_nodes[2].cmd("kdestroy") with And("I expect the user to be default"): @@ -97,8 +113,8 @@ def invalid_server_ticket(self): RQ_SRS_016_Kerberos_KerberosNotAvailable_InvalidClientTicket("1.0") ) def invalid_client_ticket(self): - """ClickHouse SHALL reject Kerberos authentication no Kerberos server is reachable - and client has no valid ticket (or the existing ticket is outdated). + """ClickHouse SHALL reject Kerberos authentication in case client has + no valid ticket (or the existing ticket is outdated). """ ch_nodes = self.context.ch_nodes @@ -108,8 +124,8 @@ def invalid_client_ticket(self): with And("setting up server principal"): create_server_principal(node=ch_nodes[0]) - with And("I kill kerberos-server"): - self.context.krb_server.stop() + # with And("I kill kerberos-server"): + # self.context.krb_server.stop() with And("I wait until client ticket is expired"): time.sleep(10) @@ -120,17 +136,18 @@ def invalid_client_ticket(self): with Then("I expect the user to be default"): assert r.output == "default", error() - with Finally("I start kerberos server again"): - self.context.krb_server.start() - ch_nodes[2].cmd("kdestroy") + with Finally(""): + # self.context.krb_server.start() + time.sleep(1) + ch_nodes[2].cmd(f"echo pwd | kinit -l 10:00 kerberos_user") while True: - kinit_no_keytab(node=ch_nodes[2]) + time.sleep(1) if ch_nodes[2].cmd(test_select_query(node=ch_nodes[0])).output == "kerberos_user": break ch_nodes[2].cmd("kdestroy") -@TestScenario +@TestCase @Requirements( RQ_SRS_016_Kerberos_KerberosNotAvailable_ValidTickets("1.0") ) @@ -316,9 +333,6 @@ def authentication_performance(self): ch_nodes[0].query("DROP USER pwd_user") - - - @TestFeature def generic(self): """Perform ClickHouse Kerberos authentication testing @@ -329,4 +343,4 @@ def generic(self): self.context.clients = [self.context.cluster.node(f"krb-client{i}") for i in range(1, 6)] for scenario in loads(current_module(), Scenario, Suite): - Scenario(run=scenario, flags=TE) + Scenario(run=scenario, flags=TE) #, setup=instrument_clickhouse_server_log) diff --git a/tests/testflows/rbac/helper/tables.py b/tests/testflows/rbac/helper/tables.py index 5d14bb34a83..ee6289bcbb5 100755 --- a/tests/testflows/rbac/helper/tables.py +++ b/tests/testflows/rbac/helper/tables.py @@ -3,39 +3,39 @@ from collections import namedtuple table_tuple = namedtuple("table_tuple", "create_statement cluster") table_types = { - "MergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = MergeTree() PARTITION BY y ORDER BY d", None), - "ReplacingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY d", None), - "SummingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY d", None), - "AggregatingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY d", None), - "CollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY d", None), - "VersionedCollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY d", None), - "GraphiteMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER by d", None), + "MergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = MergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), + "ReplacingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = ReplacingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), + "SummingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) ENGINE = SummingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), + "AggregatingMergeTree": table_tuple("CREATE TABLE {name} (d DATE, a String, b UInt8, x String, y Int8) ENGINE = AggregatingMergeTree() PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), + "CollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) ENGINE = CollapsingMergeTree(sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), + "VersionedCollapsingMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) ENGINE = VersionedCollapsingMergeTree(sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), + "GraphiteMergeTree": table_tuple("CREATE TABLE {name} (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) ENGINE = GraphiteMergeTree('graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", None), "ReplicatedMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), "ReplicatedMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), "ReplicatedReplacingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), "ReplicatedReplacingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), "ReplicatedSummingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), "ReplicatedSummingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8 DEFAULT 1, x String, y Int8) \ - ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + ENGINE = ReplicatedSummingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), "ReplicatedAggregatingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "sharded_cluster"), + ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), "ReplicatedAggregatingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d DATE, a String, b UInt8, x String, y Int8) \ - ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY d", "one_shard_cluster"), + ENGINE = ReplicatedAggregatingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), "ReplicatedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "sharded_cluster"), + ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), "ReplicatedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY d", "one_shard_cluster"), + ENGINE = ReplicatedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), "ReplicatedVersionedCollapsingMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "sharded_cluster"), + ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), "ReplicatedVersionedCollapsingMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, version UInt64, sign Int8 DEFAULT 1) \ - ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY d", "one_shard_cluster"), + ENGINE = ReplicatedVersionedCollapsingMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', sign, version) PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), "ReplicatedGraphiteMergeTree-sharded_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER sharded_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "sharded_cluster"), + ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "sharded_cluster"), "ReplicatedGraphiteMergeTree-one_shard_cluster": table_tuple("CREATE TABLE {name} ON CLUSTER one_shard_cluster (d Date, a String, b UInt8, x String, y Int8, Path String, Time DateTime, Value Float64, col UInt64, Timestamp Int64) \ - ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY d", "one_shard_cluster"), -} \ No newline at end of file + ENGINE = ReplicatedGraphiteMergeTree('/clickhouse/tables/{{shard}}/{name}', '{{replica}}', 'graphite_rollup_example') PARTITION BY y ORDER BY (b, d) PRIMARY KEY b", "one_shard_cluster"), +} diff --git a/tests/testflows/rbac/tests/privileges/alter/alter_index.py b/tests/testflows/rbac/tests/privileges/alter/alter_index.py index 379abd52d8c..78f7134a8b7 100755 --- a/tests/testflows/rbac/tests/privileges/alter/alter_index.py +++ b/tests/testflows/rbac/tests/privileges/alter/alter_index.py @@ -128,10 +128,10 @@ def check_order_by_when_privilege_is_granted(table, user, node): column = "order" with Given("I run sanity check"): - node.query(f"ALTER TABLE {table} MODIFY ORDER BY d", settings = [("user", user)]) + node.query(f"ALTER TABLE {table} MODIFY ORDER BY b", settings = [("user", user)]) with And("I add new column and modify order using that column"): - node.query(f"ALTER TABLE {table} ADD COLUMN {column} UInt32, MODIFY ORDER BY (d, {column})") + node.query(f"ALTER TABLE {table} ADD COLUMN {column} UInt32, MODIFY ORDER BY (b, {column})") with When(f"I insert random data into the ordered-by column {column}"): data = random.sample(range(1,1000),100) @@ -151,7 +151,7 @@ def check_order_by_when_privilege_is_granted(table, user, node): with And("I verify that the sorting key is present in the table"): output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"ORDER BY (d, {column})" in output['statement'], error() + assert f"ORDER BY (b, {column})" in output['statement'], error() with But(f"I cannot drop the required column {column}"): exitcode, message = errors.missing_columns(column) @@ -163,21 +163,13 @@ def check_sample_by_when_privilege_is_granted(table, user, node): """ column = 'sample' - with Given(f"I add new column {column}"): - node.query(f"ALTER TABLE {table} ADD COLUMN {column} UInt32") - with When(f"I add sample by clause"): - node.query(f"ALTER TABLE {table} MODIFY SAMPLE BY (d, {column})", + node.query(f"ALTER TABLE {table} MODIFY SAMPLE BY b", settings = [("user", user)]) with Then("I verify that the sample is in the table"): output = json.loads(node.query(f"SHOW CREATE TABLE {table} FORMAT JSONEachRow").output) - assert f"SAMPLE BY (d, {column})" in output['statement'], error() - - with But(f"I cannot drop the required column {column}"): - exitcode, message = errors.missing_columns(column) - node.query(f"ALTER TABLE {table} DROP COLUMN {column}", - exitcode=exitcode, message=message) + assert f"SAMPLE BY b" in output['statement'], error() def check_add_index_when_privilege_is_granted(table, user, node): """Ensures ADD INDEX runs as expected when the privilege is granted to the specified user @@ -258,7 +250,7 @@ def check_order_by_when_privilege_is_not_granted(table, user, node): """ with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MODIFY ORDER BY d", + node.query(f"ALTER TABLE {table} MODIFY ORDER BY b", settings = [("user", user)], exitcode=exitcode, message=message) def check_sample_by_when_privilege_is_not_granted(table, user, node): @@ -266,7 +258,7 @@ def check_sample_by_when_privilege_is_not_granted(table, user, node): """ with When("I try to use privilege that has not been granted"): exitcode, message = errors.not_enough_privileges(user) - node.query(f"ALTER TABLE {table} MODIFY SAMPLE BY d", + node.query(f"ALTER TABLE {table} MODIFY SAMPLE BY b", settings = [("user", user)], exitcode=exitcode, message=message) def check_add_index_when_privilege_is_not_granted(table, user, node): diff --git a/tests/testflows/regression.py b/tests/testflows/regression.py index 8932e6bcf8f..fe47edcfca6 100755 --- a/tests/testflows/regression.py +++ b/tests/testflows/regression.py @@ -7,9 +7,14 @@ append_path(sys.path, ".") from helpers.common import Pool, join, run_scenario from helpers.argparser import argparser +xfails = { + "kerberos/config/principal and realm specified/:": [(Fail, "https://github.com/ClickHouse/ClickHouse/issues/26197")], +} + @TestModule @Name("clickhouse") @ArgumentParser(argparser) +@XFails(xfails) def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): """ClickHouse regression. """ @@ -23,14 +28,14 @@ def regression(self, local, clickhouse_binary_path, stress=None, parallel=None): with Pool(8) as pool: try: run_scenario(pool, tasks, Feature(test=load("example.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) - #run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args) - run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) + # run_scenario(pool, tasks, Feature(test=load("ldap.regression", "regression")), args) + # run_scenario(pool, tasks, Feature(test=load("rbac.regression", "regression")), args) + # run_scenario(pool, tasks, Feature(test=load("aes_encryption.regression", "regression")), args) + # run_scenario(pool, tasks, Feature(test=load("map_type.regression", "regression")), args) + # run_scenario(pool, tasks, Feature(test=load("window_functions.regression", "regression")), args) + # run_scenario(pool, tasks, Feature(test=load("datetime64_extended_range.regression", "regression")), args) + run_scenario(pool, tasks, Feature(test=load("kerberos.regression", "regression")), args) + # run_scenario(pool, tasks, Feature(test=load("extended_precision_data_types.regression", "regression")), args) finally: join(tasks) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 541dea23698..28c2d7b1523 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,7 +1,11 @@ +v21.7.3.14-stable 2021-07-13 +v21.7.2.7-stable 2021-07-09 +v21.6.7.57-stable 2021-07-09 v21.6.6.51-stable 2021-07-02 v21.6.5.37-stable 2021-06-19 v21.6.4.26-stable 2021-06-11 v21.6.3.14-stable 2021-06-04 +v21.5.9.4-stable 2021-07-10 v21.5.8.21-stable 2021-07-02 v21.5.7.9-stable 2021-06-22 v21.5.6.6-stable 2021-05-29 @@ -11,6 +15,7 @@ v21.4.6.55-stable 2021-04-30 v21.4.5.46-stable 2021-04-24 v21.4.4.30-stable 2021-04-16 v21.4.3.21-stable 2021-04-12 +v21.3.15.4-stable 2021-07-10 v21.3.14.1-lts 2021-07-01 v21.3.13.9-lts 2021-06-22 v21.3.12.2-lts 2021-05-25 @@ -67,6 +72,7 @@ v20.9.5.5-stable 2020-11-13 v20.9.4.76-stable 2020-10-29 v20.9.3.45-stable 2020-10-09 v20.9.2.20-stable 2020-09-22 +v20.8.19.4-stable 2021-07-10 v20.8.18.32-lts 2021-04-16 v20.8.17.25-lts 2021-04-08 v20.8.16.20-lts 2021-04-06