mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into add-sqlite-database-path-check
This commit is contained in:
commit
f20ea87bfb
5
.gitmodules
vendored
5
.gitmodules
vendored
@ -193,7 +193,7 @@
|
||||
url = https://github.com/danlark1/miniselect
|
||||
[submodule "contrib/rocksdb"]
|
||||
path = contrib/rocksdb
|
||||
url = https://github.com/ClickHouse-Extras/rocksdb.git
|
||||
url = https://github.com/ClickHouse-Extras/rocksdb.git
|
||||
[submodule "contrib/xz"]
|
||||
path = contrib/xz
|
||||
url = https://github.com/xz-mirror/xz
|
||||
@ -231,3 +231,6 @@
|
||||
[submodule "contrib/sqlite-amalgamation"]
|
||||
path = contrib/sqlite-amalgamation
|
||||
url = https://github.com/azadkuh/sqlite-amalgamation
|
||||
[submodule "contrib/s2geometry"]
|
||||
path = contrib/s2geometry
|
||||
url = https://github.com/ClickHouse-Extras/s2geometry.git
|
||||
|
@ -541,6 +541,7 @@ include (cmake/find/rocksdb.cmake)
|
||||
include (cmake/find/libpqxx.cmake)
|
||||
include (cmake/find/nuraft.cmake)
|
||||
include (cmake/find/yaml-cpp.cmake)
|
||||
include (cmake/find/s2geometry.cmake)
|
||||
|
||||
if(NOT USE_INTERNAL_PARQUET_LIBRARY)
|
||||
set (ENABLE_ORC OFF CACHE INTERNAL "")
|
||||
|
24
cmake/find/s2geometry.cmake
Normal file
24
cmake/find/s2geometry.cmake
Normal file
@ -0,0 +1,24 @@
|
||||
|
||||
option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES})
|
||||
|
||||
if (ENABLE_S2_GEOMETRY)
|
||||
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/s2geometry")
|
||||
message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init --recursive")
|
||||
set (ENABLE_S2_GEOMETRY 0)
|
||||
set (USE_S2_GEOMETRY 0)
|
||||
else()
|
||||
if (OPENSSL_FOUND)
|
||||
set (S2_GEOMETRY_LIBRARY s2)
|
||||
set (S2_GEOMETRY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src/s2)
|
||||
set (USE_S2_GEOMETRY 1)
|
||||
else()
|
||||
message (WARNING "S2 uses OpenSSL, but the latter is absent.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if (NOT USE_S2_GEOMETRY)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable S2 geometry library")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
message (STATUS "Using s2geometry=${USE_S2_GEOMETRY} : ${S2_GEOMETRY_INCLUDE_DIR}")
|
@ -1,4 +1,4 @@
|
||||
option(ENABLE_STATS "Enalbe StatsLib library" ${ENABLE_LIBRARIES})
|
||||
option(ENABLE_STATS "Enable StatsLib library" ${ENABLE_LIBRARIES})
|
||||
|
||||
if (ENABLE_STATS)
|
||||
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats")
|
||||
|
10
contrib/CMakeLists.txt
vendored
10
contrib/CMakeLists.txt
vendored
@ -1,3 +1,4 @@
|
||||
# Third-party libraries may have substandard code.
|
||||
|
||||
# Put all targets defined here and in added subfolders under "contrib/" folder in GUI-based IDEs by default.
|
||||
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they will
|
||||
@ -10,10 +11,8 @@ else ()
|
||||
endif ()
|
||||
unset (_current_dir_name)
|
||||
|
||||
# Third-party libraries may have substandard code.
|
||||
# Also remove a possible source of nondeterminism.
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -D__DATE__= -D__TIME__= -D__TIMESTAMP__=")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -D__DATE__= -D__TIME__= -D__TIMESTAMP__=")
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
|
||||
|
||||
if (WITH_COVERAGE)
|
||||
set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE})
|
||||
@ -333,3 +332,6 @@ if (USE_SQLITE)
|
||||
add_subdirectory(sqlite-cmake)
|
||||
endif()
|
||||
|
||||
if (USE_S2_GEOMETRY)
|
||||
add_subdirectory(s2geometry-cmake)
|
||||
endif()
|
||||
|
2
contrib/poco
vendored
2
contrib/poco
vendored
@ -1 +1 @@
|
||||
Subproject commit 5994506908028612869fee627d68d8212dfe7c1e
|
||||
Subproject commit 44bed7238e55bbab9b6d8b4f59dbc289927069a0
|
1
contrib/s2geometry
vendored
Submodule
1
contrib/s2geometry
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 20ea540d81f4575a3fc0aea585aac611bcd03ede
|
126
contrib/s2geometry-cmake/CMakeLists.txt
Normal file
126
contrib/s2geometry-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,126 @@
|
||||
set(S2_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src")
|
||||
|
||||
set(S2_SRCS
|
||||
"${S2_SOURCE_DIR}/s2/base/stringprintf.cc"
|
||||
"${S2_SOURCE_DIR}/s2/base/strtoint.cc"
|
||||
"${S2_SOURCE_DIR}/s2/encoded_s2cell_id_vector.cc"
|
||||
"${S2_SOURCE_DIR}/s2/encoded_s2point_vector.cc"
|
||||
"${S2_SOURCE_DIR}/s2/encoded_s2shape_index.cc"
|
||||
"${S2_SOURCE_DIR}/s2/encoded_string_vector.cc"
|
||||
"${S2_SOURCE_DIR}/s2/id_set_lexicon.cc"
|
||||
"${S2_SOURCE_DIR}/s2/mutable_s2shape_index.cc"
|
||||
"${S2_SOURCE_DIR}/s2/r2rect.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s1angle.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s1chord_angle.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s1interval.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2boolean_operation.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builder.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builder_graph.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_closed_set_normalizer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_find_polygon_degeneracies.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_lax_polygon_layer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_s2point_vector_layer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polygon_layer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polyline_layer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polyline_vector_layer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2builderutil_snap_functions.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2cap.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2cell.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2cell_id.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2cell_index.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2cell_union.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2centroids.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2closest_cell_query.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2closest_edge_query.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2closest_point_query.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2contains_vertex_query.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2convex_hull_query.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2coords.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2crossing_edge_query.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2debug.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2earth.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2edge_clipping.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2edge_crosser.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2edge_crossings.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2edge_distances.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2edge_tessellator.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2error.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2furthest_edge_query.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2latlng.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2latlng_rect.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2latlng_rect_bounder.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2lax_loop_shape.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2lax_polygon_shape.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2lax_polyline_shape.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2loop.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2loop_measures.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2measures.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2metrics.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2max_distance_targets.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2min_distance_targets.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2padded_cell.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2point_compression.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2point_region.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2pointutil.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2polygon.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2polyline.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2polyline_alignment.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2polyline_measures.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2polyline_simplifier.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2predicates.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2projections.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2r2rect.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2region.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2region_term_indexer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2region_coverer.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2region_intersection.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2region_union.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shape_index.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shape_index_buffered_region.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shape_index_measures.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shape_measures.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shapeutil_build_polygon_boundaries.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shapeutil_coding.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shapeutil_contains_brute_force.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shapeutil_edge_iterator.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shapeutil_get_reference_point.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shapeutil_range_iterator.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2shapeutil_visit_crossing_edge_pairs.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2text_format.cc"
|
||||
"${S2_SOURCE_DIR}/s2/s2wedge_relations.cc"
|
||||
"${S2_SOURCE_DIR}/s2/strings/ostringstream.cc"
|
||||
"${S2_SOURCE_DIR}/s2/strings/serialize.cc"
|
||||
# ClickHouse doesn't use strings from abseil.
|
||||
# So, there is no duplicate symbols.
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/base/dynamic_annotations.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/raw_logging.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/throw_delegate.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/numeric/int128.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/ascii.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/match.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/numbers.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_cat.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_split.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/string_view.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/strip.cc"
|
||||
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/internal/memutil.cc"
|
||||
"${S2_SOURCE_DIR}/s2/util/bits/bit-interleave.cc"
|
||||
"${S2_SOURCE_DIR}/s2/util/bits/bits.cc"
|
||||
"${S2_SOURCE_DIR}/s2/util/coding/coder.cc"
|
||||
"${S2_SOURCE_DIR}/s2/util/coding/varint.cc"
|
||||
"${S2_SOURCE_DIR}/s2/util/math/exactfloat/exactfloat.cc"
|
||||
"${S2_SOURCE_DIR}/s2/util/math/mathutil.cc"
|
||||
"${S2_SOURCE_DIR}/s2/util/units/length-units.cc"
|
||||
)
|
||||
|
||||
add_library(s2 ${S2_SRCS})
|
||||
|
||||
if (OPENSSL_FOUND)
|
||||
target_link_libraries(s2 PRIVATE ${OPENSSL_LIBRARIES})
|
||||
endif()
|
||||
|
||||
target_include_directories(s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/")
|
||||
|
||||
if(M_LIBRARY)
|
||||
target_link_libraries(s2 PRIVATE ${M_LIBRARY})
|
||||
endif()
|
@ -380,6 +380,14 @@ function run_tests
|
||||
01923_network_receive_time_metric_insert
|
||||
|
||||
01889_sqlite_read_write
|
||||
|
||||
# needs s2
|
||||
01849_geoToS2
|
||||
01851_s2_to_geo
|
||||
01852_s2_get_neighbours
|
||||
01853_s2_cells_intersect
|
||||
01854_s2_cap_contains
|
||||
01854_s2_cap_union
|
||||
)
|
||||
|
||||
time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \
|
||||
|
@ -32,7 +32,7 @@ RUN rm -rf \
|
||||
RUN apt-get clean
|
||||
|
||||
# Install MySQL ODBC driver
|
||||
RUN curl 'https://cdn.mysql.com//Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit.tar.gz' --output 'mysql-connector.tar.gz' && tar -xzf mysql-connector.tar.gz && cd mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit/lib && mv * /usr/local/lib && ln -s /usr/local/lib/libmyodbc8a.so /usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so
|
||||
RUN curl 'https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit.tar.gz' --location --output 'mysql-connector.tar.gz' && tar -xzf mysql-connector.tar.gz && cd mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit/lib && mv * /usr/local/lib && ln -s /usr/local/lib/libmyodbc8a.so /usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so
|
||||
|
||||
# Unfortunately this is required for a single test for conversion data from zookeeper to clickhouse-keeper.
|
||||
# ZooKeeper is not started by default, but consumes some space in containers.
|
||||
@ -49,4 +49,3 @@ RUN mkdir /zookeeper && chmod -R 777 /zookeeper
|
||||
|
||||
ENV TZ=Europe/Moscow
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
|
@ -79,6 +79,7 @@ SELECT library_name, license_type, license_path FROM system.licenses ORDER BY li
|
||||
| re2 | BSD 3-clause | /contrib/re2/LICENSE |
|
||||
| replxx | BSD 3-clause | /contrib/replxx/LICENSE.md |
|
||||
| rocksdb | BSD 3-clause | /contrib/rocksdb/LICENSE.leveldb |
|
||||
| s2geometry | Apache | /contrib/s2geometry/LICENSE |
|
||||
| sentry-native | MIT | /contrib/sentry-native/LICENSE |
|
||||
| simdjson | Apache | /contrib/simdjson/LICENSE |
|
||||
| snappy | Public Domain | /contrib/snappy/COPYING |
|
||||
|
@ -1,6 +1,6 @@
|
||||
---
|
||||
toc_priority: 12
|
||||
toc_title: MateriaziePostgreSQL
|
||||
toc_title: MaterializedPostgreSQL
|
||||
---
|
||||
|
||||
# MaterializedPostgreSQL {#materialize-postgresql}
|
||||
|
@ -157,5 +157,6 @@ toc_title: Adopters
|
||||
| <a href="https://signoz.io/" class="favicon">SigNoz</a> | Observability Platform | Main Product | — | — | [Source code](https://github.com/SigNoz/signoz) |
|
||||
| <a href="https://chelpipegroup.com/" class="favicon">ChelPipe Group</a> | Analytics | — | — | — | [Blog post, June 2021](https://vc.ru/trade/253172-tyazhelomu-proizvodstvu-user-friendly-sayt-internet-magazin-trub-dlya-chtpz) |
|
||||
| <a href="https://zagravagames.com/en/" class="favicon">Zagrava Trading</a> | — | — | — | — | [Job offer, May 2021](https://twitter.com/datastackjobs/status/1394707267082063874) |
|
||||
| <a href="https://beeline.ru/" class="favicon">Beeline</a> | Telecom | Data Platform | — | — | [Blog post, July 2021](https://habr.com/en/company/beeline/blog/567508/) |
|
||||
|
||||
[Original article](https://clickhouse.tech/docs/en/introduction/adopters/) <!--hide-->
|
||||
|
@ -1213,7 +1213,15 @@ Default value: `3`.
|
||||
|
||||
## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
|
||||
|
||||
If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes.
|
||||
Controls quoting of 64-bit or bigger [integers](../../sql-reference/data-types/int-uint.md) (like `UInt64` or `Int128`) when they are output in a [JSON](../../interfaces/formats.md#json) format.
|
||||
Such integers are enclosed in quotes by default. This behavior is compatible with most JavaScript implementations.
|
||||
|
||||
Possible values:
|
||||
|
||||
- 0 — Integers are output without quotes.
|
||||
- 1 — Integers are enclosed in quotes.
|
||||
|
||||
Default value: 1.
|
||||
|
||||
## output_format_json_quote_denormals {#settings-output_format_json_quote_denormals}
|
||||
|
||||
@ -1990,6 +1998,16 @@ Possible values:
|
||||
|
||||
Default value: 16.
|
||||
|
||||
## merge_selecting_sleep_ms {#merge_selecting_sleep_ms}
|
||||
|
||||
Sleep time for merge selecting when no part selected, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters
|
||||
|
||||
Possible values:
|
||||
|
||||
- Any positive integer.
|
||||
|
||||
Default value: 5000
|
||||
|
||||
## parallel_distributed_insert_select {#parallel_distributed_insert_select}
|
||||
|
||||
Enables parallel distributed `INSERT ... SELECT` query.
|
||||
|
@ -8,12 +8,11 @@ Columns:
|
||||
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) — Index name.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — Index type.
|
||||
- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression used to calculate the index.
|
||||
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of granules in the block.
|
||||
- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression for the index calculation.
|
||||
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of granules in the block.
|
||||
|
||||
**Example**
|
||||
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.data_skipping_indices LIMIT 2 FORMAT Vertical;
|
||||
```
|
||||
|
@ -34,7 +34,7 @@ Input table:
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT medianDeterministic(val, 1) FROM t
|
||||
SELECT medianDeterministic(val, 1) FROM t;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
@ -12,9 +12,6 @@ toc_title: Map(key, value)
|
||||
- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md).
|
||||
- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
!!! warning "Warning"
|
||||
Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`.
|
||||
|
||||
To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. This lookup works now with a linear complexity.
|
||||
|
||||
**Examples**
|
||||
|
@ -195,6 +195,41 @@ Result:
|
||||
└────────────────────┘
|
||||
```
|
||||
|
||||
## h3ToGeo {#h3togeo}
|
||||
|
||||
Returns `(lon, lat)` that corresponds to the provided H3 index.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
h3ToGeo(h3Index)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `h3Index` — H3 Index. Type: [UInt64](../../../sql-reference/data-types/int-uint.md).
|
||||
|
||||
**Returned values**
|
||||
|
||||
- `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md).
|
||||
- `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md).
|
||||
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT h3ToGeo(644325524701193974) coordinates;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
┌─coordinates───────────────────────────┐
|
||||
│ (37.79506616830252,55.71290243145668) │
|
||||
└───────────────────────────────────────┘
|
||||
```
|
||||
## h3kRing {#h3kring}
|
||||
|
||||
Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order.
|
||||
|
@ -306,3 +306,49 @@ Result:
|
||||
└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## toJSONString {#tojsonstring}
|
||||
|
||||
Serializes a value to its JSON representation. Various data types and nested structures are supported.
|
||||
64-bit [integers](../../sql-reference/data-types/int-uint.md) or bigger (like `UInt64` or `Int128`) are enclosed in quotes by default. [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) controls this behavior.
|
||||
Special values `NaN` and `inf` are replaced with `null`. Enable [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals) setting to show them.
|
||||
When serializing an [Enum](../../sql-reference/data-types/enum.md) value, the function outputs its name.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
toJSONString(value)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `value` — Value to serialize. Value may be of any data type.
|
||||
|
||||
**Returned value**
|
||||
|
||||
- JSON representation of the value.
|
||||
|
||||
Type: [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Example**
|
||||
|
||||
The first example shows serialization of a [Map](../../sql-reference/data-types/map.md).
|
||||
The second example shows some special values wrapped into a [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT toJSONString(map('key1', 1, 'key2', 2));
|
||||
SELECT toJSONString(tuple(1.25, NULL, NaN, +inf, -inf, [])) SETTINGS output_format_json_quote_denormals = 1;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
``` text
|
||||
{"key1":1,"key2":2}
|
||||
[1.25,null,"nan","inf","-inf",[]]
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
- [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers)
|
||||
- [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals)
|
||||
|
@ -49,6 +49,7 @@ ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'passwor
|
||||
| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) |
|
||||
| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
|
||||
| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) |
|
||||
| ENUM | [Enum](../../sql-reference/data-types/enum.md) |
|
||||
| STRING | [String](../../sql-reference/data-types/string.md) |
|
||||
| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) |
|
||||
| BLOB | [String](../../sql-reference/data-types/string.md) |
|
||||
|
@ -1204,8 +1204,15 @@ load_balancing = round_robin
|
||||
Работает для форматов JSONEachRow и TSKV.
|
||||
|
||||
## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
|
||||
Управляет кавычками при выводе 64-битных или более [целых чисел](../../sql-reference/data-types/int-uint.md) (например, `UInt64` или `Int128`) в формате [JSON](../../interfaces/formats.md#json).
|
||||
По умолчанию такие числа заключаются в кавычки. Это поведение соответствует большинству реализаций JavaScript.
|
||||
|
||||
Если значение истинно, то при использовании JSON\* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек.
|
||||
Возможные значения:
|
||||
|
||||
- 0 — числа выводятся без кавычек.
|
||||
- 1 — числа выводятся в кавычках.
|
||||
|
||||
Значение по умолчанию: 1.
|
||||
|
||||
## output_format_json_quote_denormals {#settings-output_format_json_quote_denormals}
|
||||
|
||||
|
38
docs/ru/operations/system-tables/data_skipping_indices.md
Normal file
38
docs/ru/operations/system-tables/data_skipping_indices.md
Normal file
@ -0,0 +1,38 @@
|
||||
# system.data_skipping_indices {#system-data-skipping-indices}
|
||||
|
||||
Содержит информацию о существующих индексах пропуска данных во всех таблицах.
|
||||
|
||||
Столбцы:
|
||||
|
||||
- `database` ([String](../../sql-reference/data-types/string.md)) — имя базы данных.
|
||||
- `table` ([String](../../sql-reference/data-types/string.md)) — имя таблицы.
|
||||
- `name` ([String](../../sql-reference/data-types/string.md)) — имя индекса.
|
||||
- `type` ([String](../../sql-reference/data-types/string.md)) — тип индекса.
|
||||
- `expr` ([String](../../sql-reference/data-types/string.md)) — выражение, используемое для вычисления индекса.
|
||||
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество гранул в блоке данных.
|
||||
|
||||
**Пример**
|
||||
|
||||
```sql
|
||||
SELECT * FROM system.data_skipping_indices LIMIT 2 FORMAT Vertical;
|
||||
```
|
||||
|
||||
```text
|
||||
Row 1:
|
||||
──────
|
||||
database: default
|
||||
table: user_actions
|
||||
name: clicks_idx
|
||||
type: minmax
|
||||
expr: clicks
|
||||
granularity: 1
|
||||
|
||||
Row 2:
|
||||
──────
|
||||
database: default
|
||||
table: users
|
||||
name: contacts_null_idx
|
||||
type: minmax
|
||||
expr: assumeNotNull(contacts_null)
|
||||
granularity: 1
|
||||
```
|
@ -4,7 +4,6 @@
|
||||
|
||||
Функции:
|
||||
|
||||
|
||||
- `median` — синоним для [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile).
|
||||
- `medianDeterministic` — синоним для [quantileDeterministic](../../../sql-reference/aggregate-functions/reference/quantiledeterministic.md#quantiledeterministic).
|
||||
- `medianExact` — синоним для [quantileExact](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexact).
|
||||
@ -31,7 +30,7 @@
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT medianDeterministic(val, 1) FROM t
|
||||
SELECT medianDeterministic(val, 1) FROM t;
|
||||
```
|
||||
|
||||
Результат:
|
||||
@ -41,4 +40,3 @@ SELECT medianDeterministic(val, 1) FROM t
|
||||
│ 1.5 │
|
||||
└─────────────────────────────┘
|
||||
```
|
||||
|
||||
|
@ -12,9 +12,6 @@ toc_title: Map(key, value)
|
||||
- `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md).
|
||||
- `value` — значение. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) или [Array](../../sql-reference/data-types/array.md).
|
||||
|
||||
!!! warning "Предупреждение"
|
||||
Сейчас использование типа данных `Map` является экспериментальной возможностью. Чтобы использовать этот тип данных, включите настройку `allow_experimental_map_type = 1`.
|
||||
|
||||
Чтобы получить значение из колонки `a Map('key', 'value')`, используйте синтаксис `a['key']`. В настоящее время такая подстановка работает по алгоритму с линейной сложностью.
|
||||
|
||||
**Примеры**
|
||||
|
@ -306,3 +306,51 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello"
|
||||
│ [('d','"hello"'),('f','"world"')] │
|
||||
└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
|
||||
## toJSONString {#tojsonstring}
|
||||
|
||||
Сериализует значение в JSON представление. Поддерживаются различные типы данных и вложенные структуры.
|
||||
По умолчанию 64-битные [целые числа](../../sql-reference/data-types/int-uint.md) и более (например, `UInt64` или `Int128`) заключаются в кавычки. Настройка [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) управляет этим поведением.
|
||||
Специальные значения `NaN` и `inf` заменяются на `null`. Чтобы они отображались, включите настройку [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals).
|
||||
Когда сериализуется значение [Enum](../../sql-reference/data-types/enum.md), то функция выводит его имя.
|
||||
|
||||
**Синтаксис**
|
||||
|
||||
``` sql
|
||||
toJSONString(value)
|
||||
```
|
||||
|
||||
**Аргументы**
|
||||
|
||||
- `value` — значение, которое необходимо сериализовать. Может быть любого типа.
|
||||
|
||||
**Возвращаемое значение**
|
||||
|
||||
- JSON представление значения.
|
||||
|
||||
Тип: [String](../../sql-reference/data-types/string.md).
|
||||
|
||||
**Пример**
|
||||
|
||||
Первый пример показывает сериализацию [Map](../../sql-reference/data-types/map.md).
|
||||
Во втором примере есть специальные значения, обернутые в [Tuple](../../sql-reference/data-types/tuple.md).
|
||||
|
||||
Запрос:
|
||||
|
||||
``` sql
|
||||
SELECT toJSONString(map('key1', 1, 'key2', 2));
|
||||
SELECT toJSONString(tuple(1.25, NULL, NaN, +inf, -inf, [])) SETTINGS output_format_json_quote_denormals = 1;
|
||||
```
|
||||
|
||||
Результат:
|
||||
|
||||
``` text
|
||||
{"key1":1,"key2":2}
|
||||
[1.25,null,"nan","inf","-inf",[]]
|
||||
```
|
||||
|
||||
**Смотрите также**
|
||||
|
||||
- [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers)
|
||||
- [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals)
|
||||
|
@ -9,7 +9,7 @@
|
||||
Do not use any JavaScript or CSS frameworks or preprocessors.
|
||||
This HTML page should not require any build systems (node.js, npm, gulp, etc.)
|
||||
This HTML page should not be minified, instead it should be reasonably minimalistic by itself.
|
||||
This HTML page should not load any external resources
|
||||
This HTML page should not load any external resources on load.
|
||||
(CSS and JavaScript must be embedded directly to the page. No external fonts or images should be loaded).
|
||||
This UI should look as lightweight, clean and fast as possible.
|
||||
All UI elements must be aligned in pixel-perfect way.
|
||||
@ -343,13 +343,18 @@
|
||||
/// Save query in history only if it is different.
|
||||
let previous_query = '';
|
||||
|
||||
/// Substitute the address of the server where the page is served.
|
||||
if (location.protocol != 'file:') {
|
||||
const current_url = new URL(window.location);
|
||||
|
||||
const server_address = current_url.searchParams.get('url');
|
||||
if (server_address) {
|
||||
document.getElementById('url').value = server_address;
|
||||
} else if (location.protocol != 'file:') {
|
||||
/// Substitute the address of the server where the page is served.
|
||||
document.getElementById('url').value = location.origin;
|
||||
}
|
||||
|
||||
/// Substitute user name if it's specified in the query string
|
||||
let user_from_url = (new URL(window.location)).searchParams.get('user');
|
||||
const user_from_url = current_url.searchParams.get('user');
|
||||
if (user_from_url) {
|
||||
document.getElementById('user').value = user_from_url;
|
||||
}
|
||||
@ -361,7 +366,9 @@
|
||||
let user = document.getElementById('user').value;
|
||||
let password = document.getElementById('password').value;
|
||||
|
||||
let url = document.getElementById('url').value +
|
||||
let server_address = document.getElementById('url').value;
|
||||
|
||||
let url = server_address +
|
||||
/// Ask server to allow cross-domain requests.
|
||||
'?add_http_cors_header=1' +
|
||||
'&user=' + encodeURIComponent(user) +
|
||||
@ -390,11 +397,18 @@
|
||||
response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit.
|
||||
};
|
||||
let title = "ClickHouse Query: " + query;
|
||||
let url = window.location.pathname + '?user=' + encodeURIComponent(user) + '#' + window.btoa(query);
|
||||
|
||||
let history_url = window.location.pathname + '?user=' + encodeURIComponent(user);
|
||||
if (server_address != location.origin) {
|
||||
/// Save server's address in URL if it's not identical to the address of the play UI.
|
||||
history_url += '&url=' + encodeURIComponent(server_address);
|
||||
}
|
||||
history_url += '#' + window.btoa(query);
|
||||
|
||||
if (previous_query == '') {
|
||||
history.replaceState(state, title, url);
|
||||
history.replaceState(state, title, history_url);
|
||||
} else {
|
||||
history.pushState(state, title, url);
|
||||
history.pushState(state, title, history_url);
|
||||
}
|
||||
document.title = title;
|
||||
previous_query = query;
|
||||
|
@ -419,6 +419,11 @@ if (USE_AWS_S3)
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if (USE_S2_GEOMETRY)
|
||||
dbms_target_link_libraries (PUBLIC ${S2_GEOMETRY_LIBRARY})
|
||||
dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${S2_GEOMETRY_INCLUDE_DIR})
|
||||
endif()
|
||||
|
||||
if (USE_BROTLI)
|
||||
target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY})
|
||||
target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR})
|
||||
|
@ -559,6 +559,8 @@
|
||||
M(589, DISTRIBUTED_BROKEN_BATCH_FILES) \
|
||||
M(590, CANNOT_SYSCONF) \
|
||||
M(591, SQLITE_ENGINE_ERROR) \
|
||||
M(592, DATA_ENCRYPTION_ERROR) \
|
||||
M(593, ZERO_COPY_REPLICATION_ERROR) \
|
||||
\
|
||||
M(998, POSTGRESQL_CONNECTION_FAILURE) \
|
||||
M(999, KEEPER_EXCEPTION) \
|
||||
|
@ -26,13 +26,14 @@ namespace ErrorCodes
|
||||
MySQLClient::MySQLClient(const String & host_, UInt16 port_, const String & user_, const String & password_)
|
||||
: host(host_), port(port_), user(user_), password(std::move(password_))
|
||||
{
|
||||
client_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_PLUGIN_AUTH | CLIENT_SECURE_CONNECTION;
|
||||
mysql_context.client_capabilities = CLIENT_PROTOCOL_41 | CLIENT_PLUGIN_AUTH | CLIENT_SECURE_CONNECTION;
|
||||
}
|
||||
|
||||
MySQLClient::MySQLClient(MySQLClient && other)
|
||||
: host(std::move(other.host)), port(other.port), user(std::move(other.user)), password(std::move(other.password))
|
||||
, client_capability_flags(other.client_capability_flags)
|
||||
, mysql_context(other.mysql_context)
|
||||
{
|
||||
mysql_context.sequence_id = 0;
|
||||
}
|
||||
|
||||
void MySQLClient::connect()
|
||||
@ -56,7 +57,7 @@ void MySQLClient::connect()
|
||||
|
||||
in = std::make_shared<ReadBufferFromPocoSocket>(*socket);
|
||||
out = std::make_shared<WriteBufferFromPocoSocket>(*socket);
|
||||
packet_endpoint = std::make_shared<PacketEndpoint>(*in, *out, seq);
|
||||
packet_endpoint = mysql_context.makeEndpoint(*in, *out);
|
||||
handshake();
|
||||
}
|
||||
|
||||
@ -68,7 +69,7 @@ void MySQLClient::disconnect()
|
||||
socket->close();
|
||||
socket = nullptr;
|
||||
connected = false;
|
||||
seq = 0;
|
||||
mysql_context.sequence_id = 0;
|
||||
}
|
||||
|
||||
/// https://dev.mysql.com/doc/internals/en/connection-phase-packets.html
|
||||
@ -87,10 +88,10 @@ void MySQLClient::handshake()
|
||||
String auth_plugin_data = native41.getAuthPluginData();
|
||||
|
||||
HandshakeResponse handshake_response(
|
||||
client_capability_flags, MAX_PACKET_LENGTH, charset_utf8, user, "", auth_plugin_data, mysql_native_password);
|
||||
mysql_context.client_capabilities, MAX_PACKET_LENGTH, charset_utf8, user, "", auth_plugin_data, mysql_native_password);
|
||||
packet_endpoint->sendPacket<HandshakeResponse>(handshake_response, true);
|
||||
|
||||
ResponsePacket packet_response(client_capability_flags, true);
|
||||
ResponsePacket packet_response(mysql_context.client_capabilities, true);
|
||||
packet_endpoint->receivePacket(packet_response);
|
||||
packet_endpoint->resetSequenceId();
|
||||
|
||||
@ -105,7 +106,7 @@ void MySQLClient::writeCommand(char command, String query)
|
||||
WriteCommand write_command(command, query);
|
||||
packet_endpoint->sendPacket<WriteCommand>(write_command, true);
|
||||
|
||||
ResponsePacket packet_response(client_capability_flags);
|
||||
ResponsePacket packet_response(mysql_context.client_capabilities);
|
||||
packet_endpoint->receivePacket(packet_response);
|
||||
switch (packet_response.getType())
|
||||
{
|
||||
@ -124,7 +125,7 @@ void MySQLClient::registerSlaveOnMaster(UInt32 slave_id)
|
||||
RegisterSlave register_slave(slave_id);
|
||||
packet_endpoint->sendPacket<RegisterSlave>(register_slave, true);
|
||||
|
||||
ResponsePacket packet_response(client_capability_flags);
|
||||
ResponsePacket packet_response(mysql_context.client_capabilities);
|
||||
packet_endpoint->receivePacket(packet_response);
|
||||
packet_endpoint->resetSequenceId();
|
||||
if (packet_response.getType() == PACKET_ERR)
|
||||
|
@ -45,9 +45,7 @@ private:
|
||||
String password;
|
||||
|
||||
bool connected = false;
|
||||
UInt32 client_capability_flags = 0;
|
||||
|
||||
uint8_t seq = 0;
|
||||
MySQLWireContext mysql_context;
|
||||
const UInt8 charset_utf8 = 33;
|
||||
const String mysql_native_password = "mysql_native_password";
|
||||
|
||||
|
@ -68,4 +68,15 @@ String PacketEndpoint::packetToText(const String & payload)
|
||||
|
||||
}
|
||||
|
||||
|
||||
MySQLProtocol::PacketEndpointPtr MySQLWireContext::makeEndpoint(WriteBuffer & out)
|
||||
{
|
||||
return MySQLProtocol::PacketEndpoint::create(out, sequence_id);
|
||||
}
|
||||
|
||||
MySQLProtocol::PacketEndpointPtr MySQLWireContext::makeEndpoint(ReadBuffer & in, WriteBuffer & out)
|
||||
{
|
||||
return MySQLProtocol::PacketEndpoint::create(in, out, sequence_id);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -5,6 +5,7 @@
|
||||
#include "IMySQLReadPacket.h"
|
||||
#include "IMySQLWritePacket.h"
|
||||
#include "IO/MySQLPacketPayloadReadBuffer.h"
|
||||
#include <common/shared_ptr_helper.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -15,19 +16,13 @@ namespace MySQLProtocol
|
||||
/* Writes and reads packets, keeping sequence-id.
|
||||
* Throws ProtocolError, if packet with incorrect sequence-id was received.
|
||||
*/
|
||||
class PacketEndpoint
|
||||
class PacketEndpoint : public shared_ptr_helper<PacketEndpoint>
|
||||
{
|
||||
public:
|
||||
uint8_t & sequence_id;
|
||||
ReadBuffer * in;
|
||||
WriteBuffer * out;
|
||||
|
||||
/// For writing.
|
||||
PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_);
|
||||
|
||||
/// For reading and writing.
|
||||
PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_);
|
||||
|
||||
MySQLPacketPayloadReadBuffer getPayload();
|
||||
|
||||
void receivePacket(IMySQLReadPacket & packet);
|
||||
@ -48,8 +43,29 @@ public:
|
||||
|
||||
/// Converts packet to text. Is used for debug output.
|
||||
static String packetToText(const String & payload);
|
||||
|
||||
protected:
|
||||
/// For writing.
|
||||
PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_);
|
||||
|
||||
/// For reading and writing.
|
||||
PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_);
|
||||
|
||||
friend struct shared_ptr_helper<PacketEndpoint>;
|
||||
};
|
||||
|
||||
using PacketEndpointPtr = std::shared_ptr<PacketEndpoint>;
|
||||
|
||||
}
|
||||
|
||||
struct MySQLWireContext
|
||||
{
|
||||
uint8_t sequence_id = 0;
|
||||
uint32_t client_capabilities = 0;
|
||||
size_t max_packet_size = 0;
|
||||
|
||||
MySQLProtocol::PacketEndpointPtr makeEndpoint(WriteBuffer & out);
|
||||
MySQLProtocol::PacketEndpointPtr makeEndpoint(ReadBuffer & in, WriteBuffer & out);
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -1,4 +1,7 @@
|
||||
#include "Connection.h"
|
||||
|
||||
#if USE_LIBPQXX
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
namespace postgres
|
||||
@ -72,3 +75,5 @@ void Connection::connect()
|
||||
updateConnection();
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -1,5 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
#include "config_core.h"
|
||||
#endif
|
||||
|
||||
#if USE_LIBPQXX
|
||||
|
||||
#include <pqxx/pqxx> // Y_IGNORE
|
||||
#include <Core/Types.h>
|
||||
#include <boost/noncopyable.hpp>
|
||||
@ -45,3 +51,5 @@ private:
|
||||
Poco::Logger * log;
|
||||
};
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -1,5 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
#include "config_core.h"
|
||||
#endif
|
||||
|
||||
#if USE_LIBPQXX
|
||||
|
||||
#include <pqxx/pqxx> // Y_IGNORE
|
||||
#include <Core/Types.h>
|
||||
#include <common/BorrowedObjectPool.h>
|
||||
@ -35,3 +41,5 @@ private:
|
||||
|
||||
using ConnectionHolderPtr = std::unique_ptr<ConnectionHolder>;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -1,4 +1,7 @@
|
||||
#include "PoolWithFailover.h"
|
||||
|
||||
#if USE_LIBPQXX
|
||||
|
||||
#include "Utils.h"
|
||||
#include <Common/parseRemoteDescription.h>
|
||||
#include <Common/Exception.h>
|
||||
@ -136,3 +139,5 @@ ConnectionHolderPtr PoolWithFailover::get()
|
||||
throw DB::Exception(DB::ErrorCodes::POSTGRESQL_CONNECTION_FAILURE, "Unable to connect to any of the replicas");
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -1,5 +1,12 @@
|
||||
#pragma once
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
#include "config_core.h"
|
||||
#endif
|
||||
|
||||
#if USE_LIBPQXX
|
||||
|
||||
|
||||
#include "ConnectionHolder.h"
|
||||
#include <mutex>
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
@ -63,3 +70,5 @@ private:
|
||||
using PoolWithFailoverPtr = std::shared_ptr<PoolWithFailover>;
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -1,4 +1,7 @@
|
||||
#include "Utils.h"
|
||||
|
||||
#if USE_LIBPQXX
|
||||
|
||||
#include <IO/Operators.h>
|
||||
|
||||
namespace postgres
|
||||
@ -17,3 +20,5 @@ ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, S
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -1,5 +1,11 @@
|
||||
#pragma once
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
#include "config_core.h"
|
||||
#endif
|
||||
|
||||
#if USE_LIBPQXX
|
||||
|
||||
#include <pqxx/pqxx> // Y_IGNORE
|
||||
#include <Core/Types.h>
|
||||
#include "Connection.h"
|
||||
@ -15,3 +21,5 @@ namespace postgres
|
||||
{
|
||||
ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, String user, String password);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
@ -9,6 +9,7 @@
|
||||
#include <DataTypes/IDataType.h>
|
||||
#include <DataTypes/DataTypeNullable.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <Interpreters/convertFieldToType.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
@ -102,7 +103,16 @@ void insertPostgreSQLValue(
|
||||
assert_cast<ColumnUInt32 &>(column).insertValue(time);
|
||||
break;
|
||||
}
|
||||
case ExternalResultDescription::ValueType::vtDateTime64:[[fallthrough]];
|
||||
case ExternalResultDescription::ValueType::vtDateTime64:
|
||||
{
|
||||
ReadBufferFromString in(value);
|
||||
DateTime64 time = 0;
|
||||
readDateTime64Text(time, 6, in, assert_cast<const DataTypeDateTime64 *>(data_type.get())->getTimeZone());
|
||||
if (time < 0)
|
||||
time = 0;
|
||||
assert_cast<ColumnDecimal<Decimal64> &>(column).insertValue(time);
|
||||
break;
|
||||
}
|
||||
case ExternalResultDescription::ValueType::vtDecimal32: [[fallthrough]];
|
||||
case ExternalResultDescription::ValueType::vtDecimal64: [[fallthrough]];
|
||||
case ExternalResultDescription::ValueType::vtDecimal128: [[fallthrough]];
|
||||
@ -206,6 +216,14 @@ void preparePostgreSQLArrayInfo(
|
||||
readDateTimeText(time, in, assert_cast<const DataTypeDateTime *>(nested.get())->getTimeZone());
|
||||
return time;
|
||||
};
|
||||
else if (which.isDateTime64())
|
||||
parser = [nested](std::string & field) -> Field
|
||||
{
|
||||
ReadBufferFromString in(field);
|
||||
DateTime64 time = 0;
|
||||
readDateTime64Text(time, 6, in, assert_cast<const DataTypeDateTime64 *>(nested.get())->getTimeZone());
|
||||
return time;
|
||||
};
|
||||
else if (which.isDecimal32())
|
||||
parser = [nested](std::string & field) -> Field
|
||||
{
|
||||
|
@ -482,6 +482,8 @@ class IColumn;
|
||||
M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \
|
||||
M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \
|
||||
\
|
||||
M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function 'range' per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \
|
||||
\
|
||||
/** Experimental functions */ \
|
||||
M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
|
||||
\
|
||||
|
@ -31,6 +31,10 @@ SRCS(
|
||||
MySQL/PacketsProtocolText.cpp
|
||||
MySQL/PacketsReplication.cpp
|
||||
NamesAndTypes.cpp
|
||||
PostgreSQL/Connection.cpp
|
||||
PostgreSQL/PoolWithFailover.cpp
|
||||
PostgreSQL/Utils.cpp
|
||||
PostgreSQL/insertPostgreSQLValue.cpp
|
||||
PostgreSQLProtocol.cpp
|
||||
QueryProcessingStage.cpp
|
||||
Settings.cpp
|
||||
|
@ -8,7 +8,7 @@
|
||||
#include <Core/ExternalResultDescription.h>
|
||||
#include <DataStreams/IBlockInputStream.h>
|
||||
|
||||
#include <sqlite3.h>
|
||||
#include <sqlite3.h> // Y_IGNORE
|
||||
|
||||
|
||||
namespace DB
|
||||
|
@ -42,11 +42,23 @@ public:
|
||||
return it;
|
||||
}
|
||||
|
||||
/// throws exception if value is not valid
|
||||
const StringRef & getNameForValue(const T & value) const
|
||||
{
|
||||
return findByValue(value)->second;
|
||||
}
|
||||
|
||||
/// returns false if value is not valid
|
||||
bool getNameForValue(const T & value, StringRef & result) const
|
||||
{
|
||||
const auto it = value_to_name_map.find(value);
|
||||
if (it == std::end(value_to_name_map))
|
||||
return false;
|
||||
|
||||
result = it->second;
|
||||
return true;
|
||||
}
|
||||
|
||||
T getValue(StringRef field_name, bool try_treat_as_id = false) const;
|
||||
|
||||
template <typename TValues>
|
||||
|
@ -9,7 +9,7 @@
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <DataTypes/DataTypesDecimal.h>
|
||||
#include <DataTypes/DataTypeDate.h>
|
||||
#include <DataTypes/DataTypeDateTime.h>
|
||||
#include <DataTypes/DataTypeDateTime64.h>
|
||||
#include <boost/algorithm/string/split.hpp>
|
||||
#include <boost/algorithm/string/trim.hpp>
|
||||
#include <Common/quoteString.h>
|
||||
@ -71,7 +71,7 @@ static DataTypePtr convertPostgreSQLDataType(String & type, const std::function<
|
||||
else if (type == "bigserial")
|
||||
res = std::make_shared<DataTypeUInt64>();
|
||||
else if (type.starts_with("timestamp"))
|
||||
res = std::make_shared<DataTypeDateTime>();
|
||||
res = std::make_shared<DataTypeDateTime64>(6);
|
||||
else if (type == "date")
|
||||
res = std::make_shared<DataTypeDate>();
|
||||
else if (type.starts_with("numeric"))
|
||||
|
@ -9,7 +9,7 @@
|
||||
#include <Databases/DatabasesCommon.h>
|
||||
#include <Parsers/ASTCreateQuery.h>
|
||||
|
||||
#include <sqlite3.h>
|
||||
#include <sqlite3.h> // Y_IGNORE
|
||||
|
||||
|
||||
namespace DB
|
||||
|
@ -7,7 +7,7 @@
|
||||
#if USE_SQLITE
|
||||
|
||||
#include <Storages/StorageSQLite.h>
|
||||
#include <sqlite3.h>
|
||||
#include <sqlite3.h> // Y_IGNORE
|
||||
|
||||
|
||||
namespace DB
|
||||
|
@ -206,9 +206,9 @@ void DiskDecorator::startup()
|
||||
delegate->startup();
|
||||
}
|
||||
|
||||
void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context)
|
||||
void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map)
|
||||
{
|
||||
delegate->applyNewSettings(config, context);
|
||||
delegate->applyNewSettings(config, context, config_prefix, map);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -65,11 +65,12 @@ public:
|
||||
String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); }
|
||||
bool checkUniqueId(const String & id) const override { return delegate->checkUniqueId(id); }
|
||||
DiskType::Type getType() const override { return delegate->getType(); }
|
||||
bool supportZeroCopyReplication() const override { return delegate->supportZeroCopyReplication(); }
|
||||
void onFreeze(const String & path) override;
|
||||
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
|
||||
void shutdown() override;
|
||||
void startup() override;
|
||||
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) override;
|
||||
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override;
|
||||
|
||||
protected:
|
||||
Executor & getExecutor() override;
|
||||
|
201
src/Disks/DiskEncrypted.cpp
Normal file
201
src/Disks/DiskEncrypted.cpp
Normal file
@ -0,0 +1,201 @@
|
||||
#include <Disks/DiskEncrypted.h>
|
||||
|
||||
#if USE_SSL
|
||||
#include <Disks/DiskFactory.h>
|
||||
#include <IO/FileEncryptionCommon.h>
|
||||
#include <IO/ReadBufferFromEncryptedFile.h>
|
||||
#include <IO/WriteBufferFromEncryptedFile.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int INCORRECT_DISK_INDEX;
|
||||
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
using DiskEncryptedPtr = std::shared_ptr<DiskEncrypted>;
|
||||
using namespace FileEncryption;
|
||||
|
||||
class DiskEncryptedReservation : public IReservation
|
||||
{
|
||||
public:
|
||||
DiskEncryptedReservation(DiskEncryptedPtr disk_, std::unique_ptr<IReservation> reservation_)
|
||||
: disk(std::move(disk_)), reservation(std::move(reservation_))
|
||||
{
|
||||
}
|
||||
|
||||
UInt64 getSize() const override { return reservation->getSize(); }
|
||||
|
||||
DiskPtr getDisk(size_t i) const override
|
||||
{
|
||||
if (i != 0)
|
||||
throw Exception("Can't use i != 0 with single disk reservation", ErrorCodes::INCORRECT_DISK_INDEX);
|
||||
return disk;
|
||||
}
|
||||
|
||||
Disks getDisks() const override { return {disk}; }
|
||||
|
||||
void update(UInt64 new_size) override { reservation->update(new_size); }
|
||||
|
||||
private:
|
||||
DiskEncryptedPtr disk;
|
||||
std::unique_ptr<IReservation> reservation;
|
||||
};
|
||||
|
||||
ReservationPtr DiskEncrypted::reserve(UInt64 bytes)
|
||||
{
|
||||
auto reservation = delegate->reserve(bytes);
|
||||
if (!reservation)
|
||||
return {};
|
||||
return std::make_unique<DiskEncryptedReservation>(std::static_pointer_cast<DiskEncrypted>(shared_from_this()), std::move(reservation));
|
||||
}
|
||||
|
||||
DiskEncrypted::DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_)
|
||||
: DiskDecorator(disk_)
|
||||
, name(name_), key(key_), disk_path(path_)
|
||||
, disk_absolute_path(delegate->getPath() + disk_path)
|
||||
{
|
||||
initialize();
|
||||
}
|
||||
|
||||
void DiskEncrypted::initialize()
|
||||
{
|
||||
// use wrapped_disk as an EncryptedDisk store
|
||||
if (disk_path.empty())
|
||||
return;
|
||||
|
||||
if (disk_path.back() != '/')
|
||||
throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
|
||||
|
||||
delegate->createDirectories(disk_path);
|
||||
}
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
auto buffer = delegate->readFile(wrapped_path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
|
||||
String iv;
|
||||
size_t offset = 0;
|
||||
|
||||
if (exists(path) && getFileSize(path))
|
||||
{
|
||||
iv = readIV(kIVSize, *buffer);
|
||||
offset = kIVSize;
|
||||
}
|
||||
else
|
||||
iv = randomString(kIVSize);
|
||||
|
||||
return std::make_unique<ReadBufferFromEncryptedFile>(buf_size, std::move(buffer), iv, key, offset);
|
||||
}
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase> DiskEncrypted::writeFile(const String & path, size_t buf_size, WriteMode mode)
|
||||
{
|
||||
String iv;
|
||||
size_t start_offset = 0;
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
|
||||
if (mode == WriteMode::Append && exists(path) && getFileSize(path))
|
||||
{
|
||||
auto read_buffer = delegate->readFile(wrapped_path, kIVSize);
|
||||
iv = readIV(kIVSize, *read_buffer);
|
||||
start_offset = getFileSize(path);
|
||||
}
|
||||
else
|
||||
iv = randomString(kIVSize);
|
||||
|
||||
auto buffer = delegate->writeFile(wrapped_path, buf_size, mode);
|
||||
return std::make_unique<WriteBufferFromEncryptedFile>(buf_size, std::move(buffer), iv, key, start_offset);
|
||||
}
|
||||
|
||||
|
||||
size_t DiskEncrypted::getFileSize(const String & path) const
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
size_t size = delegate->getFileSize(wrapped_path);
|
||||
return size > kIVSize ? (size - kIVSize) : 0;
|
||||
}
|
||||
|
||||
void DiskEncrypted::truncateFile(const String & path, size_t size)
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->truncateFile(wrapped_path, size ? (size + kIVSize) : 0);
|
||||
}
|
||||
|
||||
SyncGuardPtr DiskEncrypted::getDirectorySyncGuard(const String & path) const
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
return delegate->getDirectorySyncGuard(wrapped_path);
|
||||
}
|
||||
|
||||
void DiskEncrypted::applyNewSettings(
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
ContextPtr /*context*/,
|
||||
const String & config_prefix,
|
||||
const DisksMap & map)
|
||||
{
|
||||
String wrapped_disk_name = config.getString(config_prefix + ".disk", "");
|
||||
if (wrapped_disk_name.empty())
|
||||
throw Exception("The wrapped disk name can not be empty. An encrypted disk is a wrapper over another disk. "
|
||||
"Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
|
||||
key = config.getString(config_prefix + ".key", "");
|
||||
if (key.empty())
|
||||
throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
|
||||
auto wrapped_disk = map.find(wrapped_disk_name);
|
||||
if (wrapped_disk == map.end())
|
||||
throw Exception("The wrapped disk must have been announced earlier. No disk with name " + wrapped_disk_name + ". Disk " + name,
|
||||
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
delegate = wrapped_disk->second;
|
||||
|
||||
disk_path = config.getString(config_prefix + ".path", "");
|
||||
initialize();
|
||||
}
|
||||
|
||||
void registerDiskEncrypted(DiskFactory & factory)
|
||||
{
|
||||
auto creator = [](const String & name,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_prefix,
|
||||
ContextPtr /*context*/,
|
||||
const DisksMap & map) -> DiskPtr {
|
||||
|
||||
String wrapped_disk_name = config.getString(config_prefix + ".disk", "");
|
||||
if (wrapped_disk_name.empty())
|
||||
throw Exception("The wrapped disk name can not be empty. An encrypted disk is a wrapper over another disk. "
|
||||
"Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
|
||||
String key = config.getString(config_prefix + ".key", "");
|
||||
if (key.empty())
|
||||
throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
if (key.size() != cipherKeyLength(defaultCipher()))
|
||||
throw Exception("Expected key with size " + std::to_string(cipherKeyLength(defaultCipher())) + ", got key with size " + std::to_string(key.size()),
|
||||
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
|
||||
auto wrapped_disk = map.find(wrapped_disk_name);
|
||||
if (wrapped_disk == map.end())
|
||||
throw Exception("The wrapped disk must have been announced earlier. No disk with name " + wrapped_disk_name + ". Disk " + name,
|
||||
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
|
||||
String relative_path = config.getString(config_prefix + ".path", "");
|
||||
|
||||
return std::make_shared<DiskEncrypted>(name, wrapped_disk->second, key, relative_path);
|
||||
};
|
||||
factory.registerDiskType("encrypted", creator);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
#endif
|
229
src/Disks/DiskEncrypted.h
Normal file
229
src/Disks/DiskEncrypted.h
Normal file
@ -0,0 +1,229 @@
|
||||
#pragma once
|
||||
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
#include <Common/config.h>
|
||||
#endif
|
||||
|
||||
#if USE_SSL
|
||||
#include <Disks/IDisk.h>
|
||||
#include <Disks/DiskDecorator.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
class ReadBufferFromFileBase;
|
||||
class WriteBufferFromFileBase;
|
||||
|
||||
class DiskEncrypted : public DiskDecorator
|
||||
{
|
||||
public:
|
||||
DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_);
|
||||
|
||||
const String & getName() const override { return name; }
|
||||
const String & getPath() const override { return disk_absolute_path; }
|
||||
|
||||
ReservationPtr reserve(UInt64 bytes) override;
|
||||
|
||||
bool exists(const String & path) const override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
return delegate->exists(wrapped_path);
|
||||
}
|
||||
|
||||
bool isFile(const String & path) const override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
return delegate->isFile(wrapped_path);
|
||||
}
|
||||
|
||||
bool isDirectory(const String & path) const override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
return delegate->isDirectory(wrapped_path);
|
||||
}
|
||||
|
||||
size_t getFileSize(const String & path) const override;
|
||||
|
||||
void createDirectory(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->createDirectory(wrapped_path);
|
||||
}
|
||||
|
||||
void createDirectories(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->createDirectories(wrapped_path);
|
||||
}
|
||||
|
||||
|
||||
void clearDirectory(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->clearDirectory(wrapped_path);
|
||||
}
|
||||
|
||||
void moveDirectory(const String & from_path, const String & to_path) override
|
||||
{
|
||||
auto wrapped_from_path = wrappedPath(from_path);
|
||||
auto wrapped_to_path = wrappedPath(to_path);
|
||||
delegate->moveDirectory(wrapped_from_path, wrapped_to_path);
|
||||
}
|
||||
|
||||
DiskDirectoryIteratorPtr iterateDirectory(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
return delegate->iterateDirectory(wrapped_path);
|
||||
}
|
||||
|
||||
void createFile(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->createFile(wrapped_path);
|
||||
}
|
||||
|
||||
void moveFile(const String & from_path, const String & to_path) override
|
||||
{
|
||||
auto wrapped_from_path = wrappedPath(from_path);
|
||||
auto wrapped_to_path = wrappedPath(to_path);
|
||||
delegate->moveFile(wrapped_from_path, wrapped_to_path);
|
||||
}
|
||||
|
||||
void replaceFile(const String & from_path, const String & to_path) override
|
||||
{
|
||||
auto wrapped_from_path = wrappedPath(from_path);
|
||||
auto wrapped_to_path = wrappedPath(to_path);
|
||||
delegate->replaceFile(wrapped_from_path, wrapped_to_path);
|
||||
}
|
||||
|
||||
void listFiles(const String & path, std::vector<String> & file_names) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->listFiles(wrapped_path, file_names);
|
||||
}
|
||||
|
||||
void copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path) override
|
||||
{
|
||||
IDisk::copy(from_path, to_disk, to_path);
|
||||
}
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readFile(
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
size_t estimated_size,
|
||||
size_t aio_threshold,
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const override;
|
||||
|
||||
std::unique_ptr<WriteBufferFromFileBase> writeFile(
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
WriteMode mode) override;
|
||||
|
||||
void removeFile(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->removeFile(wrapped_path);
|
||||
}
|
||||
|
||||
void removeFileIfExists(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->removeFileIfExists(wrapped_path);
|
||||
}
|
||||
|
||||
void removeDirectory(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->removeDirectory(wrapped_path);
|
||||
}
|
||||
|
||||
void removeRecursive(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->removeRecursive(wrapped_path);
|
||||
}
|
||||
|
||||
void removeSharedFile(const String & path, bool flag) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->removeSharedFile(wrapped_path, flag);
|
||||
}
|
||||
|
||||
void removeSharedRecursive(const String & path, bool flag) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->removeSharedRecursive(wrapped_path, flag);
|
||||
}
|
||||
|
||||
void removeSharedFileIfExists(const String & path, bool flag) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->removeSharedFileIfExists(wrapped_path, flag);
|
||||
}
|
||||
|
||||
void setLastModified(const String & path, const Poco::Timestamp & timestamp) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->setLastModified(wrapped_path, timestamp);
|
||||
}
|
||||
|
||||
Poco::Timestamp getLastModified(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
return delegate->getLastModified(wrapped_path);
|
||||
}
|
||||
|
||||
void setReadOnly(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->setReadOnly(wrapped_path);
|
||||
}
|
||||
|
||||
void createHardLink(const String & src_path, const String & dst_path) override
|
||||
{
|
||||
auto wrapped_src_path = wrappedPath(src_path);
|
||||
auto wrapped_dst_path = wrappedPath(dst_path);
|
||||
delegate->createHardLink(wrapped_src_path, wrapped_dst_path);
|
||||
}
|
||||
|
||||
void truncateFile(const String & path, size_t size) override;
|
||||
|
||||
String getUniqueId(const String & path) const override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
return delegate->getUniqueId(wrapped_path);
|
||||
}
|
||||
|
||||
void onFreeze(const String & path) override
|
||||
{
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
delegate->onFreeze(wrapped_path);
|
||||
}
|
||||
|
||||
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override;
|
||||
|
||||
DiskType::Type getType() const override { return DiskType::Type::Encrypted; }
|
||||
|
||||
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
|
||||
|
||||
private:
|
||||
void initialize();
|
||||
|
||||
String wrappedPath(const String & path) const
|
||||
{
|
||||
// if path starts_with disk_path -> got already wrapped path
|
||||
if (!disk_path.empty() && path.starts_with(disk_path))
|
||||
return path;
|
||||
return disk_path + path;
|
||||
}
|
||||
|
||||
String name;
|
||||
String key;
|
||||
String disk_path;
|
||||
String disk_absolute_path;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
#endif
|
@ -24,7 +24,8 @@ DiskPtr DiskFactory::create(
|
||||
const String & name,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_prefix,
|
||||
ContextPtr context) const
|
||||
ContextPtr context,
|
||||
const DisksMap & map) const
|
||||
{
|
||||
const auto disk_type = config.getString(config_prefix + ".type", "local");
|
||||
|
||||
@ -33,7 +34,7 @@ DiskPtr DiskFactory::create(
|
||||
throw Exception{"DiskFactory: the disk '" + name + "' has unknown disk type: " + disk_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
|
||||
|
||||
const auto & disk_creator = found->second;
|
||||
return disk_creator(name, config, config_prefix, context);
|
||||
return disk_creator(name, config, config_prefix, context, map);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -8,12 +8,14 @@
|
||||
#include <Poco/Util/AbstractConfiguration.h>
|
||||
|
||||
#include <functional>
|
||||
#include <map>
|
||||
#include <unordered_map>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
using DisksMap = std::map<String, DiskPtr>;
|
||||
/**
|
||||
* Disk factory. Responsible for creating new disk objects.
|
||||
*/
|
||||
@ -24,7 +26,8 @@ public:
|
||||
const String & name,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_prefix,
|
||||
ContextPtr context)>;
|
||||
ContextPtr context,
|
||||
const DisksMap & map)>;
|
||||
|
||||
static DiskFactory & instance();
|
||||
|
||||
@ -34,7 +37,8 @@ public:
|
||||
const String & name,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_prefix,
|
||||
ContextPtr context) const;
|
||||
ContextPtr context,
|
||||
const DisksMap & map) const;
|
||||
|
||||
private:
|
||||
using DiskTypeRegistry = std::unordered_map<String, Creator>;
|
||||
|
@ -367,7 +367,8 @@ void registerDiskLocal(DiskFactory & factory)
|
||||
auto creator = [](const String & name,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_prefix,
|
||||
ContextPtr context) -> DiskPtr {
|
||||
ContextPtr context,
|
||||
const DisksMap & /*map*/) -> DiskPtr {
|
||||
String path = config.getString(config_prefix + ".path", "");
|
||||
if (name == "default")
|
||||
{
|
||||
|
@ -100,6 +100,8 @@ public:
|
||||
|
||||
DiskType::Type getType() const override { return DiskType::Type::Local; }
|
||||
|
||||
bool supportZeroCopyReplication() const override { return false; }
|
||||
|
||||
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
|
||||
|
||||
private:
|
||||
|
@ -450,7 +450,8 @@ void registerDiskMemory(DiskFactory & factory)
|
||||
auto creator = [](const String & name,
|
||||
const Poco::Util::AbstractConfiguration & /*config*/,
|
||||
const String & /*config_prefix*/,
|
||||
ContextPtr /*context*/) -> DiskPtr { return std::make_shared<DiskMemory>(name); };
|
||||
ContextPtr /*context*/,
|
||||
const DisksMap & /*map*/) -> DiskPtr { return std::make_shared<DiskMemory>(name); };
|
||||
factory.registerDiskType("memory", creator);
|
||||
}
|
||||
|
||||
|
@ -92,6 +92,8 @@ public:
|
||||
|
||||
DiskType::Type getType() const override { return DiskType::Type::RAM; }
|
||||
|
||||
bool supportZeroCopyReplication() const override { return false; }
|
||||
|
||||
private:
|
||||
void createDirectoriesImpl(const String & path);
|
||||
void replaceFileImpl(const String & from_path, const String & to_path);
|
||||
|
@ -37,7 +37,7 @@ DiskSelector::DiskSelector(const Poco::Util::AbstractConfiguration & config, con
|
||||
|
||||
auto disk_config_prefix = config_prefix + "." + disk_name;
|
||||
|
||||
disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context));
|
||||
disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context, disks));
|
||||
}
|
||||
if (!has_default_disk)
|
||||
disks.emplace(default_disk_name, std::make_shared<DiskLocal>(default_disk_name, context->getPath(), 0));
|
||||
@ -62,16 +62,16 @@ DiskSelectorPtr DiskSelector::updateFromConfig(
|
||||
if (!std::all_of(disk_name.begin(), disk_name.end(), isWordCharASCII))
|
||||
throw Exception("Disk name can contain only alphanumeric and '_' (" + disk_name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
|
||||
|
||||
auto disk_config_prefix = config_prefix + "." + disk_name;
|
||||
if (result->getDisksMap().count(disk_name) == 0)
|
||||
{
|
||||
auto disk_config_prefix = config_prefix + "." + disk_name;
|
||||
result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context));
|
||||
result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context, result->getDisksMap()));
|
||||
}
|
||||
else
|
||||
{
|
||||
auto disk = old_disks_minus_new_disks[disk_name];
|
||||
|
||||
disk->applyNewSettings(config, context);
|
||||
disk->applyNewSettings(config, context, disk_config_prefix, result->getDisksMap());
|
||||
|
||||
old_disks_minus_new_disks.erase(disk_name);
|
||||
}
|
||||
|
@ -12,7 +12,6 @@ namespace DB
|
||||
|
||||
class DiskSelector;
|
||||
using DiskSelectorPtr = std::shared_ptr<const DiskSelector>;
|
||||
using DisksMap = std::map<String, DiskPtr>;
|
||||
|
||||
/// Parse .xml configuration and store information about disks
|
||||
/// Mostly used for introspection.
|
||||
|
@ -12,7 +12,8 @@ struct DiskType
|
||||
Local,
|
||||
RAM,
|
||||
S3,
|
||||
HDFS
|
||||
HDFS,
|
||||
Encrypted
|
||||
};
|
||||
static String toString(Type disk_type)
|
||||
{
|
||||
@ -26,6 +27,8 @@ struct DiskType
|
||||
return "s3";
|
||||
case Type::HDFS:
|
||||
return "hdfs";
|
||||
case Type::Encrypted:
|
||||
return "encrypted";
|
||||
}
|
||||
__builtin_unreachable();
|
||||
}
|
||||
|
@ -115,7 +115,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskHDFS::writeFile(const String & path
|
||||
auto hdfs_path = remote_fs_root_path + file_name;
|
||||
|
||||
LOG_DEBUG(log, "{} to file by path: {}. HDFS path: {}", mode == WriteMode::Rewrite ? "Write" : "Append",
|
||||
backQuote(metadata_path + path), remote_fs_root_path + hdfs_path);
|
||||
backQuote(metadata_path + path), hdfs_path);
|
||||
|
||||
/// Single O_WRONLY in libhdfs adds O_TRUNC
|
||||
auto hdfs_buffer = std::make_unique<WriteBufferFromHDFS>(hdfs_path,
|
||||
@ -153,6 +153,14 @@ void DiskHDFS::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper)
|
||||
});
|
||||
}
|
||||
|
||||
bool DiskHDFS::checkUniqueId(const String & hdfs_uri) const
|
||||
{
|
||||
if (!boost::algorithm::starts_with(hdfs_uri, remote_fs_root_path))
|
||||
return false;
|
||||
const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2);
|
||||
const String remote_fs_object_path = hdfs_uri.substr(begin_of_path);
|
||||
return (0 == hdfsExists(hdfs_fs.get(), remote_fs_object_path.c_str()));
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
@ -170,7 +178,8 @@ void registerDiskHDFS(DiskFactory & factory)
|
||||
auto creator = [](const String & name,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_prefix,
|
||||
ContextPtr context_) -> DiskPtr
|
||||
ContextPtr context_,
|
||||
const DisksMap & /*map*/) -> DiskPtr
|
||||
{
|
||||
fs::path disk = fs::path(context_->getPath()) / "disks" / name;
|
||||
fs::create_directories(disk);
|
||||
|
@ -44,6 +44,8 @@ public:
|
||||
|
||||
DiskType::Type getType() const override { return DiskType::Type::HDFS; }
|
||||
|
||||
bool supportZeroCopyReplication() const override { return true; }
|
||||
|
||||
std::unique_ptr<ReadBufferFromFileBase> readFile(
|
||||
const String & path,
|
||||
size_t buf_size,
|
||||
@ -58,6 +60,11 @@ public:
|
||||
|
||||
RemoteFSPathKeeperPtr createFSPathKeeper() const override;
|
||||
|
||||
/// Check file exists and ClickHouse has an access to it
|
||||
/// Overrode in remote disk
|
||||
/// Required for remote disk to ensure that replica has access to data written by other node
|
||||
bool checkUniqueId(const String & hdfs_uri) const override;
|
||||
|
||||
private:
|
||||
String getRandomName() { return toString(UUIDHelpers::generateV4()); }
|
||||
|
||||
|
@ -1,6 +1,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <Interpreters/Context_fwd.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <Core/Defines.h>
|
||||
#include <common/types.h>
|
||||
#include <Common/CurrentMetrics.h>
|
||||
@ -211,19 +212,23 @@ public:
|
||||
/// Return disk type - "local", "s3", etc.
|
||||
virtual DiskType::Type getType() const = 0;
|
||||
|
||||
/// Whether this disk support zero-copy replication.
|
||||
/// Overrode in remote fs disks.
|
||||
virtual bool supportZeroCopyReplication() const = 0;
|
||||
|
||||
/// Invoked when Global Context is shutdown.
|
||||
virtual void shutdown() {}
|
||||
|
||||
/// Performs action on disk startup.
|
||||
virtual void startup() {}
|
||||
|
||||
/// Return some uniq string for file, overrode for S3
|
||||
/// Required for distinguish different copies of the same part on S3
|
||||
/// Return some uniq string for file, overrode for IDiskRemote
|
||||
/// Required for distinguish different copies of the same part on remote disk
|
||||
virtual String getUniqueId(const String & path) const { return path; }
|
||||
|
||||
/// Check file exists and ClickHouse has an access to it
|
||||
/// Overrode in DiskS3
|
||||
/// Required for S3 to ensure that replica has access to data written by other node
|
||||
/// Overrode in remote FS disks (s3/hdfs)
|
||||
/// Required for remote disk to ensure that replica has access to data written by other node
|
||||
virtual bool checkUniqueId(const String & id) const { return exists(id); }
|
||||
|
||||
/// Invoked on partitions freeze query.
|
||||
@ -233,7 +238,7 @@ public:
|
||||
virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const;
|
||||
|
||||
/// Applies new settings for disk in runtime.
|
||||
virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr) {}
|
||||
virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr, const String &, const DisksMap &) { }
|
||||
|
||||
protected:
|
||||
friend class DiskDecorator;
|
||||
|
@ -344,17 +344,6 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path)
|
||||
}
|
||||
|
||||
|
||||
void IDiskRemote::removeFileIfExists(const String & path)
|
||||
{
|
||||
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
|
||||
if (fs::exists(fs::path(metadata_path) / path))
|
||||
{
|
||||
removeMeta(path, fs_paths_keeper);
|
||||
removeFromRemoteFS(fs_paths_keeper);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs)
|
||||
{
|
||||
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
|
||||
@ -364,6 +353,18 @@ void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs)
|
||||
}
|
||||
|
||||
|
||||
void IDiskRemote::removeSharedFileIfExists(const String & path, bool keep_in_remote_fs)
|
||||
{
|
||||
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
|
||||
if (fs::exists(fs::path(metadata_path) / path))
|
||||
{
|
||||
removeMeta(path, fs_paths_keeper);
|
||||
if (!keep_in_remote_fs)
|
||||
removeFromRemoteFS(fs_paths_keeper);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void IDiskRemote::removeSharedRecursive(const String & path, bool keep_in_remote_fs)
|
||||
{
|
||||
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
|
||||
@ -488,4 +489,13 @@ bool IDiskRemote::tryReserve(UInt64 bytes)
|
||||
return false;
|
||||
}
|
||||
|
||||
String IDiskRemote::getUniqueId(const String & path) const
|
||||
{
|
||||
Metadata metadata(remote_fs_root_path, metadata_path, path);
|
||||
String id;
|
||||
if (!metadata.remote_fs_objects.empty())
|
||||
id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first;
|
||||
return id;
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -83,12 +83,14 @@ public:
|
||||
|
||||
void removeFile(const String & path) override { removeSharedFile(path, false); }
|
||||
|
||||
void removeFileIfExists(const String & path) override;
|
||||
void removeFileIfExists(const String & path) override { removeSharedFileIfExists(path, false); }
|
||||
|
||||
void removeRecursive(const String & path) override { removeSharedRecursive(path, false); }
|
||||
|
||||
void removeSharedFile(const String & path, bool keep_in_remote_fs) override;
|
||||
|
||||
void removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) override;
|
||||
|
||||
void removeSharedRecursive(const String & path, bool keep_in_remote_fs) override;
|
||||
|
||||
void listFiles(const String & path, std::vector<String> & file_names) override;
|
||||
@ -117,6 +119,10 @@ public:
|
||||
|
||||
ReservationPtr reserve(UInt64 bytes) override;
|
||||
|
||||
String getUniqueId(const String & path) const override;
|
||||
|
||||
bool checkUniqueId(const String & id) const override = 0;
|
||||
|
||||
virtual void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) = 0;
|
||||
|
||||
virtual RemoteFSPathKeeperPtr createFSPathKeeper() const = 0;
|
||||
|
@ -158,15 +158,6 @@ DiskS3::DiskS3(
|
||||
{
|
||||
}
|
||||
|
||||
String DiskS3::getUniqueId(const String & path) const
|
||||
{
|
||||
Metadata metadata(remote_fs_root_path, metadata_path, path);
|
||||
String id;
|
||||
if (!metadata.remote_fs_objects.empty())
|
||||
id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first;
|
||||
return id;
|
||||
}
|
||||
|
||||
RemoteFSPathKeeperPtr DiskS3::createFSPathKeeper() const
|
||||
{
|
||||
auto settings = current_settings.get();
|
||||
@ -930,7 +921,7 @@ void DiskS3::onFreeze(const String & path)
|
||||
revision_file_buf.finalize();
|
||||
}
|
||||
|
||||
void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context)
|
||||
void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &)
|
||||
{
|
||||
auto new_settings = settings_getter(config, "storage_configuration.disks." + name, context);
|
||||
|
||||
|
@ -98,22 +98,21 @@ public:
|
||||
|
||||
DiskType::Type getType() const override { return DiskType::Type::S3; }
|
||||
|
||||
bool supportZeroCopyReplication() const override { return true; }
|
||||
|
||||
void shutdown() override;
|
||||
|
||||
void startup() override;
|
||||
|
||||
/// Return some uniq string for file
|
||||
/// Required for distinguish different copies of the same part on S3
|
||||
String getUniqueId(const String & path) const override;
|
||||
|
||||
/// Check file exists and ClickHouse has an access to it
|
||||
/// Required for S3 to ensure that replica has access to data wroten by other node
|
||||
/// Overrode in remote disk
|
||||
/// Required for remote disk to ensure that replica has access to data written by other node
|
||||
bool checkUniqueId(const String & id) const override;
|
||||
|
||||
/// Dumps current revision counter into file 'revision.txt' at given path.
|
||||
void onFreeze(const String & path) override;
|
||||
|
||||
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) override;
|
||||
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &) override;
|
||||
|
||||
private:
|
||||
void createFileOperationObject(const String & operation_name, UInt64 revision, const ObjectMetadata & metadata);
|
||||
|
@ -167,7 +167,8 @@ void registerDiskS3(DiskFactory & factory)
|
||||
auto creator = [](const String & name,
|
||||
const Poco::Util::AbstractConfiguration & config,
|
||||
const String & config_prefix,
|
||||
ContextPtr context) -> DiskPtr {
|
||||
ContextPtr context,
|
||||
const DisksMap & /*map*/) -> DiskPtr {
|
||||
S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint")));
|
||||
if (uri.key.back() != '/')
|
||||
throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS);
|
||||
|
@ -16,6 +16,10 @@ void registerDiskMemory(DiskFactory & factory);
|
||||
void registerDiskS3(DiskFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_SSL
|
||||
void registerDiskEncrypted(DiskFactory & factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
void registerDiskHDFS(DiskFactory & factory);
|
||||
#endif
|
||||
@ -32,6 +36,10 @@ void registerDisks()
|
||||
registerDiskS3(factory);
|
||||
#endif
|
||||
|
||||
#if USE_SSL
|
||||
registerDiskEncrypted(factory);
|
||||
#endif
|
||||
|
||||
#if USE_HDFS
|
||||
registerDiskHDFS(factory);
|
||||
#endif
|
||||
|
@ -10,6 +10,7 @@ PEERDIR(
|
||||
SRCS(
|
||||
DiskCacheWrapper.cpp
|
||||
DiskDecorator.cpp
|
||||
DiskEncrypted.cpp
|
||||
DiskFactory.cpp
|
||||
DiskLocal.cpp
|
||||
DiskMemory.cpp
|
||||
|
@ -33,6 +33,7 @@ namespace ErrorCodes
|
||||
extern const int LOGICAL_ERROR;
|
||||
extern const int FORMAT_IS_NOT_SUITABLE_FOR_INPUT;
|
||||
extern const int FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT;
|
||||
extern const int UNSUPPORTED_METHOD;
|
||||
}
|
||||
|
||||
const FormatFactory::Creators & FormatFactory::getCreators(const String & name) const
|
||||
@ -207,6 +208,9 @@ BlockOutputStreamPtr FormatFactory::getOutputStreamParallelIfPossible(
|
||||
WriteCallback callback,
|
||||
const std::optional<FormatSettings> & _format_settings) const
|
||||
{
|
||||
if (context->getMySQLProtocolContext() && name != "MySQLWire")
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats");
|
||||
|
||||
const auto & output_getter = getCreators(name).output_processor_creator;
|
||||
|
||||
const Settings & settings = context->getSettingsRef();
|
||||
@ -309,7 +313,10 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible(
|
||||
{
|
||||
const auto & output_getter = getCreators(name).output_processor_creator;
|
||||
if (!output_getter)
|
||||
throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
|
||||
throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output (with processors)", name);
|
||||
|
||||
if (context->getMySQLProtocolContext() && name != "MySQLWire")
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats");
|
||||
|
||||
auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context);
|
||||
|
||||
@ -344,7 +351,7 @@ OutputFormatPtr FormatFactory::getOutputFormat(
|
||||
{
|
||||
const auto & output_getter = getCreators(name).output_processor_creator;
|
||||
if (!output_getter)
|
||||
throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
|
||||
throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output (with processors)", name);
|
||||
|
||||
if (context->hasQueryContext() && context->getSettingsRef().log_queries)
|
||||
context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Format, name);
|
||||
|
@ -124,3 +124,6 @@ endif()
|
||||
|
||||
# Signed integer overflow on user-provided data inside boost::geometry - ignore.
|
||||
set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow)
|
||||
|
||||
# target_link_libraries(clickhouse_functions PRIVATE ${S2_LIBRARY})
|
||||
target_include_directories(clickhouse_functions SYSTEM PUBLIC ${S2_GEOMETRY_INCLUDE_DIR})
|
||||
|
@ -49,47 +49,48 @@ Columns convertConstTupleToConstantElements(const ColumnConst & column)
|
||||
return res;
|
||||
}
|
||||
|
||||
ColumnWithTypeAndName columnGetNested(const ColumnWithTypeAndName & col)
|
||||
{
|
||||
if (col.type->isNullable())
|
||||
{
|
||||
const DataTypePtr & nested_type = static_cast<const DataTypeNullable &>(*col.type).getNestedType();
|
||||
|
||||
if (!col.column)
|
||||
{
|
||||
return ColumnWithTypeAndName{nullptr, nested_type, col.name};
|
||||
}
|
||||
else if (const auto * nullable = checkAndGetColumn<ColumnNullable>(*col.column))
|
||||
{
|
||||
const auto & nested_col = nullable->getNestedColumnPtr();
|
||||
return ColumnWithTypeAndName{nested_col, nested_type, col.name};
|
||||
}
|
||||
else if (const auto * const_column = checkAndGetColumn<ColumnConst>(*col.column))
|
||||
{
|
||||
const auto * nullable_column = checkAndGetColumn<ColumnNullable>(const_column->getDataColumn());
|
||||
|
||||
ColumnPtr nullable_res;
|
||||
if (nullable_column)
|
||||
{
|
||||
const auto & nested_col = nullable_column->getNestedColumnPtr();
|
||||
nullable_res = ColumnConst::create(nested_col, col.column->size());
|
||||
}
|
||||
else
|
||||
{
|
||||
nullable_res = makeNullable(col.column);
|
||||
}
|
||||
return ColumnWithTypeAndName{ nullable_res, nested_type, col.name };
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
return col;
|
||||
}
|
||||
|
||||
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns)
|
||||
{
|
||||
ColumnsWithTypeAndName res;
|
||||
for (const auto & col : columns)
|
||||
{
|
||||
if (col.type->isNullable())
|
||||
{
|
||||
const DataTypePtr & nested_type = static_cast<const DataTypeNullable &>(*col.type).getNestedType();
|
||||
|
||||
if (!col.column)
|
||||
{
|
||||
res.emplace_back(ColumnWithTypeAndName{nullptr, nested_type, col.name});
|
||||
}
|
||||
else if (const auto * nullable = checkAndGetColumn<ColumnNullable>(*col.column))
|
||||
{
|
||||
const auto & nested_col = nullable->getNestedColumnPtr();
|
||||
res.emplace_back(ColumnWithTypeAndName{nested_col, nested_type, col.name});
|
||||
}
|
||||
else if (const auto * const_column = checkAndGetColumn<ColumnConst>(*col.column))
|
||||
{
|
||||
const auto * nullable_column = checkAndGetColumn<ColumnNullable>(const_column->getDataColumn());
|
||||
|
||||
ColumnPtr nullable_res;
|
||||
if (nullable_column)
|
||||
{
|
||||
const auto & nested_col = nullable_column->getNestedColumnPtr();
|
||||
nullable_res = ColumnConst::create(nested_col, col.column->size());
|
||||
}
|
||||
else
|
||||
{
|
||||
nullable_res = makeNullable(col.column);
|
||||
}
|
||||
res.emplace_back(ColumnWithTypeAndName{ nullable_res, nested_type, col.name });
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN);
|
||||
}
|
||||
else
|
||||
res.emplace_back(col);
|
||||
}
|
||||
res.emplace_back(columnGetNested(col));
|
||||
|
||||
return res;
|
||||
}
|
||||
|
@ -81,6 +81,8 @@ inline std::enable_if_t<IsDecimalNumber<T>, Field> toField(const T & x, UInt32 s
|
||||
|
||||
Columns convertConstTupleToConstantElements(const ColumnConst & column);
|
||||
|
||||
/// Returns nested column with corrected type if nullable
|
||||
ColumnWithTypeAndName columnGetNested(const ColumnWithTypeAndName & col);
|
||||
|
||||
/// Returns the copy of a given columns in which each column is replaced with its respective nested
|
||||
/// column if it is nullable.
|
||||
|
@ -580,36 +580,44 @@ template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDateTime64
|
||||
template <typename DataType>
|
||||
struct FormatImpl
|
||||
{
|
||||
static void execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType *, const DateLUTImpl *)
|
||||
template <typename ReturnType = void>
|
||||
static ReturnType execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType *, const DateLUTImpl *)
|
||||
{
|
||||
writeText(x, wb);
|
||||
return ReturnType(true);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FormatImpl<DataTypeDate>
|
||||
{
|
||||
static void execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl *)
|
||||
template <typename ReturnType = void>
|
||||
static ReturnType execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl *)
|
||||
{
|
||||
writeDateText(DayNum(x), wb);
|
||||
return ReturnType(true);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FormatImpl<DataTypeDateTime>
|
||||
{
|
||||
static void execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime *, const DateLUTImpl * time_zone)
|
||||
template <typename ReturnType = void>
|
||||
static ReturnType execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime *, const DateLUTImpl * time_zone)
|
||||
{
|
||||
writeDateTimeText(x, wb, *time_zone);
|
||||
return ReturnType(true);
|
||||
}
|
||||
};
|
||||
|
||||
template <>
|
||||
struct FormatImpl<DataTypeDateTime64>
|
||||
{
|
||||
static void execute(const DataTypeDateTime64::FieldType x, WriteBuffer & wb, const DataTypeDateTime64 * type, const DateLUTImpl * time_zone)
|
||||
template <typename ReturnType = void>
|
||||
static ReturnType execute(const DataTypeDateTime64::FieldType x, WriteBuffer & wb, const DataTypeDateTime64 * type, const DateLUTImpl * time_zone)
|
||||
{
|
||||
writeDateTimeText(DateTime64(x), type->getScale(), wb, *time_zone);
|
||||
return ReturnType(true);
|
||||
}
|
||||
};
|
||||
|
||||
@ -617,18 +625,34 @@ struct FormatImpl<DataTypeDateTime64>
|
||||
template <typename FieldType>
|
||||
struct FormatImpl<DataTypeEnum<FieldType>>
|
||||
{
|
||||
static void execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum<FieldType> * type, const DateLUTImpl *)
|
||||
template <typename ReturnType = void>
|
||||
static ReturnType execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum<FieldType> * type, const DateLUTImpl *)
|
||||
{
|
||||
writeString(type->getNameForValue(x), wb);
|
||||
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
|
||||
|
||||
if constexpr (throw_exception)
|
||||
{
|
||||
writeString(type->getNameForValue(x), wb);
|
||||
}
|
||||
else
|
||||
{
|
||||
StringRef res;
|
||||
bool is_ok = type->getNameForValue(x, res);
|
||||
if (is_ok)
|
||||
writeString(res, wb);
|
||||
return ReturnType(is_ok);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
template <typename FieldType>
|
||||
struct FormatImpl<DataTypeDecimal<FieldType>>
|
||||
{
|
||||
static void execute(const FieldType x, WriteBuffer & wb, const DataTypeDecimal<FieldType> * type, const DateLUTImpl *)
|
||||
template <typename ReturnType = void>
|
||||
static ReturnType execute(const FieldType x, WriteBuffer & wb, const DataTypeDecimal<FieldType> * type, const DateLUTImpl *)
|
||||
{
|
||||
writeText(x, type->getScale(), wb);
|
||||
return ReturnType(true);
|
||||
}
|
||||
};
|
||||
|
||||
@ -643,6 +667,16 @@ struct ConvertImpl<DataTypeEnum<FieldType>, DataTypeNumber<FieldType>, Name, Con
|
||||
}
|
||||
};
|
||||
|
||||
static ColumnUInt8::MutablePtr copyNullMap(ColumnPtr col)
|
||||
{
|
||||
ColumnUInt8::MutablePtr null_map = nullptr;
|
||||
if (const auto * col_null = checkAndGetColumn<ColumnNullable>(col.get()))
|
||||
{
|
||||
null_map = ColumnUInt8::create();
|
||||
null_map->insertRangeFrom(col_null->getNullMapColumn(), 0, col_null->size());
|
||||
}
|
||||
return null_map;
|
||||
}
|
||||
|
||||
template <typename FromDataType, typename Name>
|
||||
struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType, DataTypeString>, DataTypeString>, Name, ConvertDefaultBehaviorTag>
|
||||
@ -652,13 +686,18 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
|
||||
|
||||
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/)
|
||||
{
|
||||
const auto & col_with_type_and_name = arguments[0];
|
||||
ColumnUInt8::MutablePtr null_map = copyNullMap(arguments[0].column);
|
||||
|
||||
const auto & col_with_type_and_name = columnGetNested(arguments[0]);
|
||||
const auto & type = static_cast<const FromDataType &>(*col_with_type_and_name.type);
|
||||
|
||||
const DateLUTImpl * time_zone = nullptr;
|
||||
/// For argument of DateTime type, second argument with time zone could be specified.
|
||||
if constexpr (std::is_same_v<FromDataType, DataTypeDateTime> || std::is_same_v<FromDataType, DataTypeDateTime64>)
|
||||
time_zone = &extractTimeZoneFromFunctionArguments(arguments, 1, 0);
|
||||
{
|
||||
auto non_null_args = createBlockWithNestedColumns(arguments);
|
||||
time_zone = &extractTimeZoneFromFunctionArguments(non_null_args, 1, 0);
|
||||
}
|
||||
|
||||
if (const auto col_from = checkAndGetColumn<ColVecType>(col_with_type_and_name.column.get()))
|
||||
{
|
||||
@ -684,14 +723,30 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
|
||||
|
||||
WriteBufferFromVector<ColumnString::Chars> write_buffer(data_to);
|
||||
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
if (null_map)
|
||||
{
|
||||
FormatImpl<FromDataType>::execute(vec_from[i], write_buffer, &type, time_zone);
|
||||
writeChar(0, write_buffer);
|
||||
offsets_to[i] = write_buffer.count();
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
bool is_ok = FormatImpl<FromDataType>::template execute<bool>(vec_from[i], write_buffer, &type, time_zone);
|
||||
null_map->getData()[i] |= !is_ok;
|
||||
writeChar(0, write_buffer);
|
||||
offsets_to[i] = write_buffer.count();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
{
|
||||
FormatImpl<FromDataType>::template execute<void>(vec_from[i], write_buffer, &type, time_zone);
|
||||
writeChar(0, write_buffer);
|
||||
offsets_to[i] = write_buffer.count();
|
||||
}
|
||||
}
|
||||
|
||||
write_buffer.finalize();
|
||||
|
||||
if (null_map)
|
||||
return ColumnNullable::create(std::move(col_to), std::move(null_map));
|
||||
return col_to;
|
||||
}
|
||||
else
|
||||
@ -705,9 +760,11 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
|
||||
/// Generic conversion of any type to String.
|
||||
struct ConvertImplGenericToString
|
||||
{
|
||||
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments)
|
||||
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type)
|
||||
{
|
||||
const auto & col_with_type_and_name = arguments[0];
|
||||
ColumnUInt8::MutablePtr null_map = copyNullMap(arguments[0].column);
|
||||
|
||||
const auto & col_with_type_and_name = columnGetNested(arguments[0]);
|
||||
const IDataType & type = *col_with_type_and_name.type;
|
||||
const IColumn & col_from = *col_with_type_and_name.column;
|
||||
|
||||
@ -733,6 +790,9 @@ struct ConvertImplGenericToString
|
||||
}
|
||||
|
||||
write_buffer.finalize();
|
||||
|
||||
if (result_type->isNullable() && null_map)
|
||||
return ColumnNullable::create(std::move(col_to), std::move(null_map));
|
||||
return col_to;
|
||||
}
|
||||
};
|
||||
@ -1420,7 +1480,11 @@ public:
|
||||
/// Function actually uses default implementation for nulls,
|
||||
/// but we need to know if return type is Nullable or not,
|
||||
/// so we use checked_return_type only to intercept the first call to getReturnTypeImpl(...).
|
||||
bool useDefaultImplementationForNulls() const override { return checked_return_type; }
|
||||
bool useDefaultImplementationForNulls() const override
|
||||
{
|
||||
bool to_nullable_string = to_nullable && std::is_same_v<ToDataType, DataTypeString>;
|
||||
return checked_return_type && !to_nullable_string;
|
||||
}
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override
|
||||
@ -1485,7 +1549,10 @@ private:
|
||||
throw Exception{"Function " + getName() + " expects at least 1 argument",
|
||||
ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION};
|
||||
|
||||
const IDataType * from_type = arguments[0].type.get();
|
||||
if (result_type->onlyNull())
|
||||
return result_type->createColumnConstWithDefaultValue(input_rows_count);
|
||||
|
||||
const DataTypePtr from_type = removeNullable(arguments[0].type);
|
||||
ColumnPtr result_column;
|
||||
|
||||
auto call = [&](const auto & types, const auto & tag) -> bool
|
||||
@ -1581,7 +1648,7 @@ private:
|
||||
/// Generic conversion of any type to String.
|
||||
if (std::is_same_v<ToDataType, DataTypeString>)
|
||||
{
|
||||
return ConvertImplGenericToString::execute(arguments);
|
||||
return ConvertImplGenericToString::execute(arguments, result_type);
|
||||
}
|
||||
else
|
||||
throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(),
|
||||
|
@ -58,10 +58,10 @@ struct CountEqualAction
|
||||
namespace Impl
|
||||
{
|
||||
template <
|
||||
class ConcreteAction,
|
||||
typename ConcreteAction,
|
||||
bool RightArgIsConstant = false,
|
||||
class IntegralInitial = UInt64,
|
||||
class IntegralResult = UInt64>
|
||||
typename IntegralInitial = UInt64,
|
||||
typename IntegralResult = UInt64>
|
||||
struct Main
|
||||
{
|
||||
private:
|
||||
@ -94,13 +94,13 @@ private:
|
||||
}
|
||||
|
||||
/// LowCardinality
|
||||
static bool compare(const IColumn & left, const Result& right, size_t i, size_t)
|
||||
static bool compare(const IColumn & left, const Result & right, size_t i, size_t)
|
||||
{
|
||||
return left.getUInt(i) == right;
|
||||
}
|
||||
|
||||
/// Generic
|
||||
static bool compare(const IColumn& left, const IColumn& right, size_t i, size_t j)
|
||||
static bool compare(const IColumn & left, const IColumn & right, size_t i, size_t j)
|
||||
{
|
||||
return 0 == left.compareAt(i, RightArgIsConstant ? 0 : j, right, 1);
|
||||
}
|
||||
@ -109,7 +109,7 @@ private:
|
||||
|
||||
static constexpr bool hasNull(const NullMap * const null_map, size_t i) noexcept { return (*null_map)[i]; }
|
||||
|
||||
template <size_t Case, class Data, class Target>
|
||||
template <size_t Case, typename Data, typename Target>
|
||||
static void process(
|
||||
const Data & data, const ArrOffsets & offsets, const Target & target, ResultArr & result,
|
||||
[[maybe_unused]] const NullMap * const null_map_data,
|
||||
@ -148,7 +148,7 @@ private:
|
||||
continue;
|
||||
}
|
||||
else if (!compare(data, target, current_offset + j, i))
|
||||
continue;
|
||||
continue;
|
||||
|
||||
ConcreteAction::apply(current, j);
|
||||
|
||||
@ -162,7 +162,7 @@ private:
|
||||
}
|
||||
|
||||
public:
|
||||
template <class Data, class Target>
|
||||
template <typename Data, typename Target>
|
||||
static void vector(
|
||||
const Data & data,
|
||||
const ArrOffsets & offsets,
|
||||
@ -183,7 +183,7 @@ public:
|
||||
};
|
||||
|
||||
/// When the 2nd function argument is a NULL value.
|
||||
template <class ConcreteAction>
|
||||
template <typename ConcreteAction>
|
||||
struct Null
|
||||
{
|
||||
using ResultType = typename ConcreteAction::ResultType;
|
||||
@ -227,7 +227,7 @@ struct Null
|
||||
}
|
||||
};
|
||||
|
||||
template <class ConcreteAction>
|
||||
template <typename ConcreteAction>
|
||||
struct String
|
||||
{
|
||||
private:
|
||||
@ -350,7 +350,7 @@ public:
|
||||
};
|
||||
}
|
||||
|
||||
template <class ConcreteAction, class Name>
|
||||
template <typename ConcreteAction, typename Name>
|
||||
class FunctionArrayIndex : public IFunction
|
||||
{
|
||||
public:
|
||||
@ -565,7 +565,7 @@ private:
|
||||
* Integral s = {s1, s2, ...}
|
||||
* (s1, s1, s2, ...), (s2, s1, s2, ...), (s3, s1, s2, ...)
|
||||
*/
|
||||
template <class ...Integral>
|
||||
template <typename... Integral>
|
||||
static inline ColumnPtr executeIntegral(const ColumnsWithTypeAndName & arguments)
|
||||
{
|
||||
const ColumnArray * const left = checkAndGetColumn<ColumnArray>(arguments[0].column.get());
|
||||
@ -590,14 +590,14 @@ private:
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
template <class ...Integral>
|
||||
template <typename... Integral>
|
||||
static inline bool executeIntegral(ExecutionData& data)
|
||||
{
|
||||
return (executeIntegralExpanded<Integral, Integral...>(data) || ...);
|
||||
}
|
||||
|
||||
/// Invoke executeIntegralImpl with such parameters: (A, other1), (A, other2), ...
|
||||
template <class A, class ...Other>
|
||||
template <typename A, typename... Other>
|
||||
static inline bool executeIntegralExpanded(ExecutionData& data)
|
||||
{
|
||||
return (executeIntegralImpl<A, Other>(data) || ...);
|
||||
@ -608,7 +608,7 @@ private:
|
||||
* second argument, namely, the @e value, so it's possible to invoke the <tt>has(Array(Int8), UInt64)</tt> e.g.
|
||||
* so we have to check all possible variants for #Initial and #Resulting types.
|
||||
*/
|
||||
template <class Initial, class Resulting>
|
||||
template <typename Initial, typename Resulting>
|
||||
static bool executeIntegralImpl(ExecutionData& data)
|
||||
{
|
||||
const ColumnVector<Initial> * col_nested = checkAndGetColumn<ColumnVector<Initial>>(&data.left);
|
||||
@ -647,7 +647,7 @@ private:
|
||||
}
|
||||
|
||||
/**
|
||||
* Catches arguments of type LC(T) (left) and U (right).
|
||||
* Catches arguments of type LowCardinality(T) (left) and U (right).
|
||||
*
|
||||
* The perftests
|
||||
* https://clickhouse-test-reports.s3.yandex.net/12550/2d27fa0fa8c198a82bf1fe3625050ccf56695976/integration_tests_(release).html
|
||||
@ -726,7 +726,7 @@ private:
|
||||
|
||||
return col_result;
|
||||
}
|
||||
else if (col_lc->nestedIsNullable()) // LC(Nullable(T)) and U
|
||||
else if (col_lc->nestedIsNullable()) // LowCardinality(Nullable(T)) and U
|
||||
{
|
||||
const ColumnPtr left_casted = col_lc->convertToFullColumnIfLowCardinality(); // Nullable(T)
|
||||
const ColumnNullable& left_nullable = *checkAndGetColumn<ColumnNullable>(left_casted.get());
|
||||
@ -746,16 +746,17 @@ private:
|
||||
? right_nullable->getNestedColumn()
|
||||
: *right_casted.get();
|
||||
|
||||
ExecutionData data = {
|
||||
ExecutionData data =
|
||||
{
|
||||
left_ptr, right_ptr,
|
||||
col_array->getOffsets(),
|
||||
nullptr,
|
||||
{null_map_left_casted, null_map_right_casted}};
|
||||
|
||||
if (dispatchConvertedLCColumns(data))
|
||||
if (dispatchConvertedLowCardinalityColumns(data))
|
||||
return data.result_column;
|
||||
}
|
||||
else // LC(T) and U, T not Nullable
|
||||
else // LowCardinality(T) and U, T not Nullable
|
||||
{
|
||||
if (col_arg.isNullable())
|
||||
return nullptr;
|
||||
@ -764,24 +765,25 @@ private:
|
||||
arg_lc && arg_lc->isNullable())
|
||||
return nullptr;
|
||||
|
||||
// LC(T) and U (possibly LC(V))
|
||||
// LowCardinality(T) and U (possibly LowCardinality(V))
|
||||
|
||||
const ColumnPtr left_casted = col_lc->convertToFullColumnIfLowCardinality();
|
||||
const ColumnPtr right_casted = col_arg.convertToFullColumnIfLowCardinality();
|
||||
|
||||
ExecutionData data = {
|
||||
ExecutionData data =
|
||||
{
|
||||
*left_casted.get(), *right_casted.get(), col_array->getOffsets(),
|
||||
nullptr, {null_map_data, null_map_item}
|
||||
};
|
||||
|
||||
if (dispatchConvertedLCColumns(data))
|
||||
if (dispatchConvertedLowCardinalityColumns(data))
|
||||
return data.result_column;
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
static bool dispatchConvertedLCColumns(ExecutionData& data)
|
||||
static bool dispatchConvertedLowCardinalityColumns(ExecutionData & data)
|
||||
{
|
||||
if (data.left.isNumeric() && data.right.isNumeric()) // ColumnArrays
|
||||
return executeIntegral<INTEGRAL_TPL_PACK>(data);
|
||||
|
@ -7,6 +7,7 @@
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnVector.h>
|
||||
#include <Interpreters/castColumn.h>
|
||||
#include <Interpreters/Context.h>
|
||||
#include <numeric>
|
||||
|
||||
|
||||
@ -31,8 +32,10 @@ class FunctionRange : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "range";
|
||||
static constexpr size_t max_elements = 100'000'000;
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionRange>(); }
|
||||
|
||||
const size_t max_elements;
|
||||
static FunctionPtr create(ContextPtr context_) { return std::make_shared<FunctionRange>(std::move(context_)); }
|
||||
explicit FunctionRange(ContextPtr context) : max_elements(context->getSettingsRef().function_range_max_elements_in_block) {}
|
||||
|
||||
private:
|
||||
String getName() const override { return name; }
|
||||
|
@ -6,4 +6,5 @@
|
||||
#cmakedefine01 USE_SIMDJSON
|
||||
#cmakedefine01 USE_RAPIDJSON
|
||||
#cmakedefine01 USE_H3
|
||||
#cmakedefine01 USE_S2_GEOMETRY
|
||||
#cmakedefine01 USE_FASTOPS
|
||||
|
@ -46,20 +46,23 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isFloat64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be Float64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be Float64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
arg = arguments[1].get();
|
||||
if (!WhichDataType(arg).isFloat64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be Float64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be Float64",
|
||||
arg->getName(), 2, getName());
|
||||
|
||||
arg = arguments[2].get();
|
||||
if (!WhichDataType(arg).isUInt8())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(3) + " of function " + getName() + ". Must be UInt8",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt8",
|
||||
arg->getName(), 3, getName());
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
111
src/Functions/geoToS2.cpp
Normal file
111
src/Functions/geoToS2.cpp
Normal file
@ -0,0 +1,111 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
class S2CellId;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/**
|
||||
* Accepts points of the form (longitude, latitude)
|
||||
* Returns s2 identifier
|
||||
*/
|
||||
class FunctionGeoToS2 : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "geoToS2";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionGeoToS2>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 2; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
for (size_t i = 0; i < getNumberOfArguments(); ++i)
|
||||
{
|
||||
const auto * arg = arguments[i].get();
|
||||
if (!WhichDataType(arg).isFloat64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be Float64",
|
||||
arg->getName(), i, getName());
|
||||
}
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_lon = arguments[0].column.get();
|
||||
const auto * col_lat = arguments[1].column.get();
|
||||
|
||||
auto dst = ColumnVector<UInt64>::create();
|
||||
auto & dst_data = dst->getData();
|
||||
dst_data.resize(input_rows_count);
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const Float64 lon = col_lon->getFloat64(row);
|
||||
const Float64 lat = col_lat->getFloat64(row);
|
||||
|
||||
if (isNaN(lon) || isNaN(lat))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Arguments must not be NaN");
|
||||
|
||||
if (!(isFinite(lon) && isFinite(lat)))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Arguments must not be infinite");
|
||||
|
||||
/// S2 acceptes point as (latitude, longitude)
|
||||
S2LatLng lat_lng = S2LatLng::FromDegrees(lat, lon);
|
||||
S2CellId id(lat_lng);
|
||||
|
||||
dst_data[row] = id.id();
|
||||
}
|
||||
|
||||
return dst;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionGeoToS2(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionGeoToS2>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
@ -44,8 +44,9 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt8())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt8",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeFloat64>();
|
||||
}
|
||||
@ -62,8 +63,10 @@ public:
|
||||
{
|
||||
const int resolution = col_hindex->getUInt(row);
|
||||
if (resolution > MAX_H3_RES)
|
||||
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
throw Exception(
|
||||
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
|
||||
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
|
||||
resolution, getName(), MAX_H3_RES);
|
||||
|
||||
// Numerical constant is 180 degrees / pi / Earth radius, Earth radius is from h3 sources
|
||||
Float64 res = 8.99320592271288084e-6 * getHexagonEdgeLengthAvgM(resolution);
|
||||
|
@ -49,8 +49,9 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt8())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt8",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeFloat64>();
|
||||
}
|
||||
@ -67,8 +68,10 @@ public:
|
||||
{
|
||||
const UInt64 resolution = col_hindex->getUInt(row);
|
||||
if (resolution > MAX_H3_RES)
|
||||
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
throw Exception(
|
||||
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
|
||||
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
|
||||
resolution, getName(), MAX_H3_RES);
|
||||
|
||||
Float64 res = getHexagonEdgeLengthAvgM(resolution);
|
||||
|
||||
|
@ -41,8 +41,9 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
@ -41,8 +41,9 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
@ -44,8 +44,9 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt8())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt8",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeFloat64>();
|
||||
}
|
||||
@ -62,8 +63,10 @@ public:
|
||||
{
|
||||
const UInt64 resolution = col_hindex->getUInt(row);
|
||||
if (resolution > MAX_H3_RES)
|
||||
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
throw Exception(
|
||||
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
|
||||
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
|
||||
resolution, getName(), MAX_H3_RES);
|
||||
|
||||
Float64 res = getHexagonAreaAvgM2(resolution);
|
||||
|
||||
|
@ -41,14 +41,16 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
arg = arguments[1].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 2, getName());
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
@ -41,8 +41,9 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
@ -50,14 +50,16 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
arg = arguments[1].get();
|
||||
if (!WhichDataType(arg).isUInt8())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt8",
|
||||
arg->getName(), 2, getName());
|
||||
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
|
||||
}
|
||||
@ -81,14 +83,17 @@ public:
|
||||
const UInt8 child_resolution = col_resolution->getUInt(row);
|
||||
|
||||
if (child_resolution > MAX_H3_RES)
|
||||
throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
throw Exception(
|
||||
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
|
||||
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}",
|
||||
toString(child_resolution), getName(), toString(MAX_H3_RES));
|
||||
|
||||
const size_t vec_size = cellToChildrenSize(parent_hindex, child_resolution);
|
||||
if (vec_size > MAX_ARRAY_SIZE)
|
||||
throw Exception("The result of function" + getName()
|
||||
+ " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = "
|
||||
+ toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE);
|
||||
throw Exception(
|
||||
ErrorCodes::TOO_LARGE_ARRAY_SIZE,
|
||||
"The result of function {} (array of {} elements) will be too large with resolution argument = {}",
|
||||
getName(), toString(vec_size), toString(child_resolution));
|
||||
|
||||
hindex_vec.resize(vec_size);
|
||||
cellToChildren(parent_hindex, child_resolution, hindex_vec.data());
|
||||
|
@ -44,14 +44,16 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
arg = arguments[1].get();
|
||||
if (!WhichDataType(arg).isUInt8())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt8",
|
||||
arg->getName(), 2, getName());
|
||||
|
||||
return std::make_shared<DataTypeUInt64>();
|
||||
}
|
||||
@ -71,8 +73,10 @@ public:
|
||||
const UInt8 resolution = col_resolution->getUInt(row);
|
||||
|
||||
if (resolution > MAX_H3_RES)
|
||||
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
|
||||
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
throw Exception(
|
||||
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
|
||||
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}",
|
||||
toString(resolution), getName(), toString(MAX_H3_RES));
|
||||
|
||||
UInt64 res = cellToParent(hindex, resolution);
|
||||
|
||||
|
@ -42,8 +42,9 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeString>();
|
||||
}
|
||||
@ -67,16 +68,14 @@ public:
|
||||
const UInt64 hindex = col_hindex->getUInt(i);
|
||||
|
||||
if (!isValidCell(hindex))
|
||||
{
|
||||
throw Exception("Invalid H3 index: " + std::to_string(hindex), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
}
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Invalid H3 index: {}", hindex);
|
||||
|
||||
h3ToString(hindex, pos, H3_INDEX_STRING_LENGTH);
|
||||
|
||||
// move to end of the index
|
||||
while (*pos != '\0')
|
||||
{
|
||||
pos++;
|
||||
}
|
||||
|
||||
vec_offsets[i] = ++pos - begin;
|
||||
}
|
||||
vec_res.resize(pos - begin);
|
||||
|
@ -47,14 +47,16 @@ public:
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
arg = arguments[1].get();
|
||||
if (!isInteger(arg))
|
||||
throw Exception(
|
||||
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be integer",
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be integer",
|
||||
arg->getName(), 2, getName());
|
||||
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
|
||||
}
|
||||
|
96
src/Functions/h3toGeo.cpp
Normal file
96
src/Functions/h3toGeo.cpp
Normal file
@ -0,0 +1,96 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_H3
|
||||
|
||||
#include <array>
|
||||
#include <math.h>
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Functions/IFunction.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
|
||||
#include <h3api.h>
|
||||
|
||||
|
||||
namespace DB
|
||||
{
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// Implements the function h3ToGeo which takes a single argument (h3Index)
|
||||
/// and returns the longitude and latitude that correspond to the provided h3 index
|
||||
class FunctionH3ToGeo : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "h3ToGeo";
|
||||
|
||||
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionH3ToGeo>(); }
|
||||
|
||||
std::string getName() const override { return name; }
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
const auto * arg = arguments[0].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeTuple>(
|
||||
DataTypes{std::make_shared<DataTypeFloat64>(), std::make_shared<DataTypeFloat64>()},
|
||||
Strings{"longitude", "latitude"});
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_index = arguments[0].column.get();
|
||||
|
||||
auto latitude = ColumnFloat64::create(input_rows_count);
|
||||
auto longitude = ColumnFloat64::create(input_rows_count);
|
||||
|
||||
ColumnFloat64::Container & lon_data = longitude->getData();
|
||||
ColumnFloat64::Container & lat_data = latitude->getData();
|
||||
|
||||
|
||||
for (size_t row = 0; row < input_rows_count; ++row)
|
||||
{
|
||||
H3Index h3index = col_index->getUInt(row);
|
||||
LatLng coord{};
|
||||
|
||||
cellToLatLng(h3index,&coord);
|
||||
lon_data[row] = radsToDegs(coord.lng);
|
||||
lat_data[row] = radsToDegs(coord.lat);
|
||||
}
|
||||
|
||||
MutableColumns columns;
|
||||
columns.emplace_back(std::move(longitude));
|
||||
columns.emplace_back(std::move(latitude));
|
||||
return ColumnTuple::create(std::move(columns));
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionH3ToGeo(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionH3ToGeo>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
#endif
|
@ -28,6 +28,7 @@ void registerFunctionSvg(FunctionFactory & factory);
|
||||
|
||||
#if USE_H3
|
||||
void registerFunctionGeoToH3(FunctionFactory &);
|
||||
void registerFunctionH3ToGeo(FunctionFactory &);
|
||||
void registerFunctionH3EdgeAngle(FunctionFactory &);
|
||||
void registerFunctionH3EdgeLengthM(FunctionFactory &);
|
||||
void registerFunctionH3GetResolution(FunctionFactory &);
|
||||
@ -42,6 +43,19 @@ void registerFunctionH3ToString(FunctionFactory &);
|
||||
void registerFunctionH3HexAreaM2(FunctionFactory &);
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
void registerFunctionGeoToS2(FunctionFactory &);
|
||||
void registerFunctionS2ToGeo(FunctionFactory &);
|
||||
void registerFunctionS2GetNeighbors(FunctionFactory &);
|
||||
void registerFunctionS2CellsIntersect(FunctionFactory &);
|
||||
void registerFunctionS2CapContains(FunctionFactory &);
|
||||
void registerFunctionS2CapUnion(FunctionFactory &);
|
||||
void registerFunctionS2RectAdd(FunctionFactory &);
|
||||
void registerFunctionS2RectContains(FunctionFactory &);
|
||||
void registerFunctionS2RectUnion(FunctionFactory &);
|
||||
void registerFunctionS2RectIntersection(FunctionFactory &);
|
||||
#endif
|
||||
|
||||
|
||||
void registerFunctionsGeo(FunctionFactory & factory)
|
||||
{
|
||||
@ -66,6 +80,7 @@ void registerFunctionsGeo(FunctionFactory & factory)
|
||||
|
||||
#if USE_H3
|
||||
registerFunctionGeoToH3(factory);
|
||||
registerFunctionH3ToGeo(factory);
|
||||
registerFunctionH3EdgeAngle(factory);
|
||||
registerFunctionH3EdgeLengthM(factory);
|
||||
registerFunctionH3GetResolution(factory);
|
||||
@ -79,6 +94,19 @@ void registerFunctionsGeo(FunctionFactory & factory)
|
||||
registerFunctionH3ToString(factory);
|
||||
registerFunctionH3HexAreaM2(factory);
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
registerFunctionGeoToS2(factory);
|
||||
registerFunctionS2ToGeo(factory);
|
||||
registerFunctionS2GetNeighbors(factory);
|
||||
registerFunctionS2CellsIntersect(factory);
|
||||
registerFunctionS2CapContains(factory);
|
||||
registerFunctionS2CapUnion(factory);
|
||||
registerFunctionS2RectAdd(factory);
|
||||
registerFunctionS2RectContains(factory);
|
||||
registerFunctionS2RectUnion(factory);
|
||||
registerFunctionS2RectIntersection(factory);
|
||||
#endif
|
||||
}
|
||||
|
||||
}
|
||||
|
132
src/Functions/s2CapContains.cpp
Normal file
132
src/Functions/s2CapContains.cpp
Normal file
@ -0,0 +1,132 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/**
|
||||
* The cap represents a portion of the sphere that has been cut off by a plane.
|
||||
* It is defined by a point on a sphere and a radius in degrees.
|
||||
* Imagine that we draw a line through the center of the sphere and our point.
|
||||
* An infinite number of planes pass through this line, but any plane will intersect the cap in two points.
|
||||
* Thus the angle is defined by one of this points and the entire line.
|
||||
* So, the radius of Pi/2 defines a hemisphere and the radius of Pi defines a whole sphere.
|
||||
*
|
||||
* This function returns whether a cap contains a point.
|
||||
*/
|
||||
class FunctionS2CapContains : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "s2CapContains";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionS2CapContains>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 3; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
for (size_t index = 0; index < getNumberOfArguments(); ++index)
|
||||
{
|
||||
const auto * arg = arguments[index].get();
|
||||
|
||||
/// Radius
|
||||
if (index == 1)
|
||||
{
|
||||
if (!WhichDataType(arg).isFloat64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be Float64",
|
||||
arg->getName(), 2, getName());
|
||||
}
|
||||
else if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), index + 1, getName());
|
||||
}
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_center = arguments[0].column.get();
|
||||
const auto * col_degrees = arguments[1].column.get();
|
||||
const auto * col_point = arguments[2].column.get();
|
||||
|
||||
auto dst = ColumnUInt8::create();
|
||||
auto & dst_data = dst->getData();
|
||||
dst_data.reserve(input_rows_count);
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const auto center = S2CellId(col_center->getUInt(row));
|
||||
const Float64 degrees = col_degrees->getFloat64(row);
|
||||
const auto point = S2CellId(col_point->getUInt(row));
|
||||
|
||||
if (isNaN(degrees))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan");
|
||||
|
||||
if (std::isinf(degrees))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be infinite");
|
||||
|
||||
if (!center.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Center is not valid");
|
||||
|
||||
if (!point.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid");
|
||||
|
||||
S1Angle angle = S1Angle::Degrees(degrees);
|
||||
S2Cap cap(center.ToPoint(), angle);
|
||||
|
||||
dst_data.emplace_back(cap.Contains(point.ToPoint()));
|
||||
}
|
||||
|
||||
return dst;
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionS2CapContains(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionS2CapContains>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
141
src/Functions/s2CapUnion.cpp
Normal file
141
src/Functions/s2CapUnion.cpp
Normal file
@ -0,0 +1,141 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/NaNUtils.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/**
|
||||
* The cap represents a portion of the sphere that has been cut off by a plane.
|
||||
* See comment for s2CapContains function.
|
||||
* This function returns the smallest cap that contains both of input caps.
|
||||
* It is represented by identifier of the center and a radius.
|
||||
*/
|
||||
class FunctionS2CapUnion : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "s2CapUnion";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionS2CapUnion>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 4; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
for (size_t index = 0; index < getNumberOfArguments(); ++index)
|
||||
{
|
||||
const auto * arg = arguments[index].get();
|
||||
if (index == 1 || index == 3)
|
||||
{
|
||||
if (!WhichDataType(arg).isFloat64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be Float64",
|
||||
arg->getName(), index + 1, getName());
|
||||
}
|
||||
else if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), index + 1, getName()
|
||||
);
|
||||
}
|
||||
|
||||
DataTypePtr center = std::make_shared<DataTypeUInt64>();
|
||||
DataTypePtr radius = std::make_shared<DataTypeFloat64>();
|
||||
|
||||
return std::make_shared<DataTypeTuple>(DataTypes{center, radius});
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_center1 = arguments[0].column.get();
|
||||
const auto * col_radius1 = arguments[1].column.get();
|
||||
const auto * col_center2 = arguments[2].column.get();
|
||||
const auto * col_radius2 = arguments[3].column.get();
|
||||
|
||||
auto col_res_center = ColumnUInt64::create();
|
||||
auto col_res_radius = ColumnFloat64::create();
|
||||
|
||||
auto & vec_res_center = col_res_center->getData();
|
||||
vec_res_center.reserve(input_rows_count);
|
||||
|
||||
auto & vec_res_radius = col_res_radius->getData();
|
||||
vec_res_radius.reserve(input_rows_count);
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const UInt64 first_center = col_center1->getUInt(row);
|
||||
const Float64 first_radius = col_radius1->getFloat64(row);
|
||||
const UInt64 second_center = col_center2->getUInt(row);
|
||||
const Float64 second_radius = col_radius2->getFloat64(row);
|
||||
|
||||
if (isNaN(first_radius) || isNaN(second_radius))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan");
|
||||
|
||||
if (std::isinf(first_radius) || std::isinf(second_radius))
|
||||
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be infinite");
|
||||
|
||||
auto first_center_cell = S2CellId(first_center);
|
||||
auto second_center_cell = S2CellId(second_center);
|
||||
|
||||
if (!first_center_cell.is_valid() || !second_center_cell.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Center of the cap is not valid");
|
||||
|
||||
S2Cap cap1(first_center_cell.ToPoint(), S1Angle::Degrees(first_radius));
|
||||
S2Cap cap2(second_center_cell.ToPoint(), S1Angle::Degrees(second_radius));
|
||||
|
||||
S2Cap cap_union = cap1.Union(cap2);
|
||||
|
||||
vec_res_center.emplace_back(S2CellId(cap_union.center()).id());
|
||||
vec_res_radius.emplace_back(cap_union.GetRadius().degrees());
|
||||
}
|
||||
|
||||
return ColumnTuple::create(Columns{std::move(col_res_center), std::move(col_res_radius)});
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionS2CapUnion(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionS2CapUnion>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
104
src/Functions/s2CellsIntersect.cpp
Normal file
104
src/Functions/s2CellsIntersect.cpp
Normal file
@ -0,0 +1,104 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/**
|
||||
* Each cell in s2 library is a quadrilateral bounded by four geodesics.
|
||||
*/
|
||||
class FunctionS2CellsIntersect : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "s2CellsIntersect";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionS2CellsIntersect>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 2; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
for (size_t i = 0; i < getNumberOfArguments(); ++i)
|
||||
{
|
||||
const auto * arg = arguments[i].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), i, getName());
|
||||
}
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_id_first = arguments[0].column.get();
|
||||
const auto * col_id_second = arguments[1].column.get();
|
||||
|
||||
auto dst = ColumnUInt8::create();
|
||||
auto & dst_data = dst->getData();
|
||||
dst_data.reserve(input_rows_count);
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const UInt64 id_first = col_id_first->getInt(row);
|
||||
const UInt64 id_second = col_id_second->getInt(row);
|
||||
|
||||
auto first_cell = S2CellId(id_first);
|
||||
auto second_cell = S2CellId(id_second);
|
||||
|
||||
if (!first_cell.is_valid() || !second_cell.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cell is not valid");
|
||||
|
||||
dst_data.emplace_back(S2CellId(id_first).intersects(S2CellId(id_second)));
|
||||
}
|
||||
|
||||
return dst;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionS2CellsIntersect(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionS2CellsIntersect>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
111
src/Functions/s2GetNeighbors.cpp
Normal file
111
src/Functions/s2GetNeighbors.cpp
Normal file
@ -0,0 +1,111 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnArray.h>
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeArray.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/**
|
||||
* Each cell in s2 library is a quadrilateral bounded by four geodesics.
|
||||
* So, each cell has 4 neighbors
|
||||
*/
|
||||
class FunctionS2GetNeighbors : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "s2GetNeighbors";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionS2GetNeighbors>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 1; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
const auto * arg = arguments[0].get();
|
||||
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be Float64",
|
||||
arg->getName(), 1, getName());
|
||||
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_id = arguments[0].column.get();
|
||||
|
||||
auto dst = ColumnArray::create(ColumnUInt64::create());
|
||||
auto & dst_data = dst->getData();
|
||||
auto & dst_offsets = dst->getOffsets();
|
||||
dst_offsets.resize(input_rows_count);
|
||||
size_t current_offset = 0;
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const UInt64 id = col_id->getUInt(row);
|
||||
|
||||
S2CellId cell_id(id);
|
||||
|
||||
if (!cell_id.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cell is not valid");
|
||||
|
||||
S2CellId neighbors[4];
|
||||
cell_id.GetEdgeNeighbors(neighbors);
|
||||
|
||||
dst_data.reserve(dst_data.size() + 4);
|
||||
for (auto & neighbor : neighbors)
|
||||
{
|
||||
++current_offset;
|
||||
dst_data.insert(neighbor.id());
|
||||
}
|
||||
dst_offsets[row] = current_offset;
|
||||
}
|
||||
|
||||
return dst;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionS2GetNeighbors(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionS2GetNeighbors>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
115
src/Functions/s2RectAdd.cpp
Normal file
115
src/Functions/s2RectAdd.cpp
Normal file
@ -0,0 +1,115 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class FunctionS2RectAdd : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "s2RectAdd";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionS2RectAdd>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 4; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
for (size_t index = 0; index < getNumberOfArguments(); ++index)
|
||||
{
|
||||
const auto * arg = arguments[index].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), index, getName());
|
||||
}
|
||||
|
||||
DataTypePtr element = std::make_shared<DataTypeUInt64>();
|
||||
|
||||
return std::make_shared<DataTypeTuple>(DataTypes{element, element});
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_lo = arguments[0].column.get();
|
||||
const auto * col_hi = arguments[1].column.get();
|
||||
const auto * col_point = arguments[2].column.get();
|
||||
|
||||
auto col_res_first = ColumnUInt64::create();
|
||||
auto col_res_second = ColumnUInt64::create();
|
||||
|
||||
auto & vec_res_first = col_res_first->getData();
|
||||
vec_res_first.reserve(input_rows_count);
|
||||
|
||||
auto & vec_res_second = col_res_second->getData();
|
||||
vec_res_second.reserve(input_rows_count);
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const auto lo = S2CellId(col_lo->getUInt(row));
|
||||
const auto hi = S2CellId(col_hi->getUInt(row));
|
||||
const auto point = S2CellId(col_point->getUInt(row));
|
||||
|
||||
if (!lo.is_valid() || !hi.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid");
|
||||
|
||||
if (!point.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid");
|
||||
|
||||
S2LatLngRect rect(lo.ToLatLng(), hi.ToLatLng());
|
||||
|
||||
rect.AddPoint(point.ToPoint());
|
||||
|
||||
vec_res_first.emplace_back(S2CellId(rect.lo()).id());
|
||||
vec_res_second.emplace_back(S2CellId(rect.hi()).id());
|
||||
}
|
||||
|
||||
return ColumnTuple::create(Columns{std::move(col_res_first), std::move(col_res_second)});
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionS2RectAdd(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionS2RectAdd>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
105
src/Functions/s2RectContains.cpp
Normal file
105
src/Functions/s2RectContains.cpp
Normal file
@ -0,0 +1,105 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
class FunctionS2RectContains : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "s2RectContains";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionS2RectContains>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 4; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
for (size_t i = 0; i < getNumberOfArguments(); ++i)
|
||||
{
|
||||
const auto * arg = arguments[i].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), i, getName());
|
||||
}
|
||||
|
||||
return std::make_shared<DataTypeUInt8>();
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_lo = arguments[0].column.get();
|
||||
const auto * col_hi = arguments[1].column.get();
|
||||
const auto * col_point = arguments[2].column.get();
|
||||
|
||||
auto dst = ColumnVector<UInt8>::create();
|
||||
auto & dst_data = dst->getData();
|
||||
dst_data.reserve(input_rows_count);
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const auto lo = S2CellId(col_lo->getUInt(row));
|
||||
const auto hi = S2CellId(col_hi->getUInt(row));
|
||||
const auto point = S2CellId(col_point->getUInt(row));
|
||||
|
||||
if (!lo.is_valid() || !hi.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid");
|
||||
|
||||
if (!point.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid");
|
||||
|
||||
S2LatLngRect rect(lo.ToLatLng(), hi.ToLatLng());
|
||||
|
||||
dst_data.emplace_back(rect.Contains(point.ToLatLng()));
|
||||
}
|
||||
|
||||
return dst;
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionS2RectContains(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionS2RectContains>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
121
src/Functions/s2RectIntersection.cpp
Normal file
121
src/Functions/s2RectIntersection.cpp
Normal file
@ -0,0 +1,121 @@
|
||||
#if !defined(ARCADIA_BUILD)
|
||||
# include "config_functions.h"
|
||||
#endif
|
||||
|
||||
#if USE_S2_GEOMETRY
|
||||
|
||||
#include <Columns/ColumnsNumber.h>
|
||||
#include <Columns/ColumnTuple.h>
|
||||
#include <DataTypes/DataTypesNumber.h>
|
||||
#include <DataTypes/DataTypeTuple.h>
|
||||
#include <Functions/FunctionFactory.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <common/range.h>
|
||||
|
||||
#include "s2_fwd.h"
|
||||
|
||||
class S2CellId;
|
||||
|
||||
namespace DB
|
||||
{
|
||||
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
|
||||
extern const int BAD_ARGUMENTS;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
|
||||
class FunctionS2RectIntersection : public IFunction
|
||||
{
|
||||
public:
|
||||
static constexpr auto name = "s2RectIntersection";
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionS2RectIntersection>();
|
||||
}
|
||||
|
||||
std::string getName() const override
|
||||
{
|
||||
return name;
|
||||
}
|
||||
|
||||
size_t getNumberOfArguments() const override { return 4; }
|
||||
|
||||
bool useDefaultImplementationForConstants() const override { return true; }
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
|
||||
{
|
||||
for (size_t i = 0; i < getNumberOfArguments(); ++i)
|
||||
{
|
||||
const auto * arg = arguments[i].get();
|
||||
if (!WhichDataType(arg).isUInt64())
|
||||
throw Exception(
|
||||
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
|
||||
"Illegal type {} of argument {} of function {}. Must be UInt64",
|
||||
arg->getName(), i, getName());
|
||||
}
|
||||
|
||||
DataTypePtr element = std::make_shared<DataTypeUInt64>();
|
||||
|
||||
return std::make_shared<DataTypeTuple>(DataTypes{element, element});
|
||||
}
|
||||
|
||||
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
|
||||
{
|
||||
const auto * col_lo1 = arguments[0].column.get();
|
||||
const auto * col_hi1 = arguments[1].column.get();
|
||||
const auto * col_lo2 = arguments[2].column.get();
|
||||
const auto * col_hi2 = arguments[3].column.get();
|
||||
|
||||
auto col_res_first = ColumnUInt64::create();
|
||||
auto col_res_second = ColumnUInt64::create();
|
||||
|
||||
auto & vec_res_first = col_res_first->getData();
|
||||
vec_res_first.reserve(input_rows_count);
|
||||
|
||||
auto & vec_res_second = col_res_second->getData();
|
||||
vec_res_second.reserve(input_rows_count);
|
||||
|
||||
for (const auto row : collections::range(0, input_rows_count))
|
||||
{
|
||||
const auto lo1 = S2CellId(col_lo1->getUInt(row));
|
||||
const auto hi1 = S2CellId(col_hi1->getUInt(row));
|
||||
const auto lo2 = S2CellId(col_lo2->getUInt(row));
|
||||
const auto hi2 = S2CellId(col_hi2->getUInt(row));
|
||||
|
||||
if (!lo1.is_valid() || !hi1.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "First rectangle is not valid");
|
||||
|
||||
if (!lo2.is_valid() || !hi2.is_valid())
|
||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second rectangle is not valid");
|
||||
|
||||
S2LatLngRect rect1(lo1.ToLatLng(), hi1.ToLatLng());
|
||||
S2LatLngRect rect2(lo2.ToLatLng(), hi2.ToLatLng());
|
||||
|
||||
S2LatLngRect rect_intersection = rect1.Intersection(rect2);
|
||||
|
||||
vec_res_first.emplace_back(S2CellId(rect_intersection.lo()).id());
|
||||
vec_res_second.emplace_back(S2CellId(rect_intersection.hi()).id());
|
||||
}
|
||||
|
||||
return ColumnTuple::create(Columns{std::move(col_res_first), std::move(col_res_second)});
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
void registerFunctionS2RectIntersection(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionS2RectIntersection>();
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
#endif
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user