Merge branch 'master' of https://github.com/ClickHouse/ClickHouse into add-sqlite-database-path-check

This commit is contained in:
kssenii 2021-07-14 20:49:05 +00:00
commit f20ea87bfb
192 changed files with 4695 additions and 578 deletions

5
.gitmodules vendored
View File

@ -193,7 +193,7 @@
url = https://github.com/danlark1/miniselect
[submodule "contrib/rocksdb"]
path = contrib/rocksdb
url = https://github.com/ClickHouse-Extras/rocksdb.git
url = https://github.com/ClickHouse-Extras/rocksdb.git
[submodule "contrib/xz"]
path = contrib/xz
url = https://github.com/xz-mirror/xz
@ -231,3 +231,6 @@
[submodule "contrib/sqlite-amalgamation"]
path = contrib/sqlite-amalgamation
url = https://github.com/azadkuh/sqlite-amalgamation
[submodule "contrib/s2geometry"]
path = contrib/s2geometry
url = https://github.com/ClickHouse-Extras/s2geometry.git

View File

@ -541,6 +541,7 @@ include (cmake/find/rocksdb.cmake)
include (cmake/find/libpqxx.cmake)
include (cmake/find/nuraft.cmake)
include (cmake/find/yaml-cpp.cmake)
include (cmake/find/s2geometry.cmake)
if(NOT USE_INTERNAL_PARQUET_LIBRARY)
set (ENABLE_ORC OFF CACHE INTERNAL "")

View File

@ -0,0 +1,24 @@
option(ENABLE_S2_GEOMETRY "Enable S2 geometry library" ${ENABLE_LIBRARIES})
if (ENABLE_S2_GEOMETRY)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/s2geometry")
message (WARNING "submodule contrib/s2geometry is missing. to fix try run: \n git submodule update --init --recursive")
set (ENABLE_S2_GEOMETRY 0)
set (USE_S2_GEOMETRY 0)
else()
if (OPENSSL_FOUND)
set (S2_GEOMETRY_LIBRARY s2)
set (S2_GEOMETRY_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src/s2)
set (USE_S2_GEOMETRY 1)
else()
message (WARNING "S2 uses OpenSSL, but the latter is absent.")
endif()
endif()
if (NOT USE_S2_GEOMETRY)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't enable S2 geometry library")
endif()
endif()
message (STATUS "Using s2geometry=${USE_S2_GEOMETRY} : ${S2_GEOMETRY_INCLUDE_DIR}")

View File

@ -1,4 +1,4 @@
option(ENABLE_STATS "Enalbe StatsLib library" ${ENABLE_LIBRARIES})
option(ENABLE_STATS "Enable StatsLib library" ${ENABLE_LIBRARIES})
if (ENABLE_STATS)
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/stats")

View File

@ -1,3 +1,4 @@
# Third-party libraries may have substandard code.
# Put all targets defined here and in added subfolders under "contrib/" folder in GUI-based IDEs by default.
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they will
@ -10,10 +11,8 @@ else ()
endif ()
unset (_current_dir_name)
# Third-party libraries may have substandard code.
# Also remove a possible source of nondeterminism.
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w -D__DATE__= -D__TIME__= -D__TIMESTAMP__=")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w -D__DATE__= -D__TIME__= -D__TIMESTAMP__=")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
if (WITH_COVERAGE)
set (WITHOUT_COVERAGE_LIST ${WITHOUT_COVERAGE})
@ -333,3 +332,6 @@ if (USE_SQLITE)
add_subdirectory(sqlite-cmake)
endif()
if (USE_S2_GEOMETRY)
add_subdirectory(s2geometry-cmake)
endif()

2
contrib/poco vendored

@ -1 +1 @@
Subproject commit 5994506908028612869fee627d68d8212dfe7c1e
Subproject commit 44bed7238e55bbab9b6d8b4f59dbc289927069a0

1
contrib/s2geometry vendored Submodule

@ -0,0 +1 @@
Subproject commit 20ea540d81f4575a3fc0aea585aac611bcd03ede

View File

@ -0,0 +1,126 @@
set(S2_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src")
set(S2_SRCS
"${S2_SOURCE_DIR}/s2/base/stringprintf.cc"
"${S2_SOURCE_DIR}/s2/base/strtoint.cc"
"${S2_SOURCE_DIR}/s2/encoded_s2cell_id_vector.cc"
"${S2_SOURCE_DIR}/s2/encoded_s2point_vector.cc"
"${S2_SOURCE_DIR}/s2/encoded_s2shape_index.cc"
"${S2_SOURCE_DIR}/s2/encoded_string_vector.cc"
"${S2_SOURCE_DIR}/s2/id_set_lexicon.cc"
"${S2_SOURCE_DIR}/s2/mutable_s2shape_index.cc"
"${S2_SOURCE_DIR}/s2/r2rect.cc"
"${S2_SOURCE_DIR}/s2/s1angle.cc"
"${S2_SOURCE_DIR}/s2/s1chord_angle.cc"
"${S2_SOURCE_DIR}/s2/s1interval.cc"
"${S2_SOURCE_DIR}/s2/s2boolean_operation.cc"
"${S2_SOURCE_DIR}/s2/s2builder.cc"
"${S2_SOURCE_DIR}/s2/s2builder_graph.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_closed_set_normalizer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_find_polygon_degeneracies.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_lax_polygon_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_s2point_vector_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polygon_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polyline_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polyline_vector_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_snap_functions.cc"
"${S2_SOURCE_DIR}/s2/s2cap.cc"
"${S2_SOURCE_DIR}/s2/s2cell.cc"
"${S2_SOURCE_DIR}/s2/s2cell_id.cc"
"${S2_SOURCE_DIR}/s2/s2cell_index.cc"
"${S2_SOURCE_DIR}/s2/s2cell_union.cc"
"${S2_SOURCE_DIR}/s2/s2centroids.cc"
"${S2_SOURCE_DIR}/s2/s2closest_cell_query.cc"
"${S2_SOURCE_DIR}/s2/s2closest_edge_query.cc"
"${S2_SOURCE_DIR}/s2/s2closest_point_query.cc"
"${S2_SOURCE_DIR}/s2/s2contains_vertex_query.cc"
"${S2_SOURCE_DIR}/s2/s2convex_hull_query.cc"
"${S2_SOURCE_DIR}/s2/s2coords.cc"
"${S2_SOURCE_DIR}/s2/s2crossing_edge_query.cc"
"${S2_SOURCE_DIR}/s2/s2debug.cc"
"${S2_SOURCE_DIR}/s2/s2earth.cc"
"${S2_SOURCE_DIR}/s2/s2edge_clipping.cc"
"${S2_SOURCE_DIR}/s2/s2edge_crosser.cc"
"${S2_SOURCE_DIR}/s2/s2edge_crossings.cc"
"${S2_SOURCE_DIR}/s2/s2edge_distances.cc"
"${S2_SOURCE_DIR}/s2/s2edge_tessellator.cc"
"${S2_SOURCE_DIR}/s2/s2error.cc"
"${S2_SOURCE_DIR}/s2/s2furthest_edge_query.cc"
"${S2_SOURCE_DIR}/s2/s2latlng.cc"
"${S2_SOURCE_DIR}/s2/s2latlng_rect.cc"
"${S2_SOURCE_DIR}/s2/s2latlng_rect_bounder.cc"
"${S2_SOURCE_DIR}/s2/s2lax_loop_shape.cc"
"${S2_SOURCE_DIR}/s2/s2lax_polygon_shape.cc"
"${S2_SOURCE_DIR}/s2/s2lax_polyline_shape.cc"
"${S2_SOURCE_DIR}/s2/s2loop.cc"
"${S2_SOURCE_DIR}/s2/s2loop_measures.cc"
"${S2_SOURCE_DIR}/s2/s2measures.cc"
"${S2_SOURCE_DIR}/s2/s2metrics.cc"
"${S2_SOURCE_DIR}/s2/s2max_distance_targets.cc"
"${S2_SOURCE_DIR}/s2/s2min_distance_targets.cc"
"${S2_SOURCE_DIR}/s2/s2padded_cell.cc"
"${S2_SOURCE_DIR}/s2/s2point_compression.cc"
"${S2_SOURCE_DIR}/s2/s2point_region.cc"
"${S2_SOURCE_DIR}/s2/s2pointutil.cc"
"${S2_SOURCE_DIR}/s2/s2polygon.cc"
"${S2_SOURCE_DIR}/s2/s2polyline.cc"
"${S2_SOURCE_DIR}/s2/s2polyline_alignment.cc"
"${S2_SOURCE_DIR}/s2/s2polyline_measures.cc"
"${S2_SOURCE_DIR}/s2/s2polyline_simplifier.cc"
"${S2_SOURCE_DIR}/s2/s2predicates.cc"
"${S2_SOURCE_DIR}/s2/s2projections.cc"
"${S2_SOURCE_DIR}/s2/s2r2rect.cc"
"${S2_SOURCE_DIR}/s2/s2region.cc"
"${S2_SOURCE_DIR}/s2/s2region_term_indexer.cc"
"${S2_SOURCE_DIR}/s2/s2region_coverer.cc"
"${S2_SOURCE_DIR}/s2/s2region_intersection.cc"
"${S2_SOURCE_DIR}/s2/s2region_union.cc"
"${S2_SOURCE_DIR}/s2/s2shape_index.cc"
"${S2_SOURCE_DIR}/s2/s2shape_index_buffered_region.cc"
"${S2_SOURCE_DIR}/s2/s2shape_index_measures.cc"
"${S2_SOURCE_DIR}/s2/s2shape_measures.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_build_polygon_boundaries.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_coding.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_contains_brute_force.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_edge_iterator.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_get_reference_point.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_range_iterator.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_visit_crossing_edge_pairs.cc"
"${S2_SOURCE_DIR}/s2/s2text_format.cc"
"${S2_SOURCE_DIR}/s2/s2wedge_relations.cc"
"${S2_SOURCE_DIR}/s2/strings/ostringstream.cc"
"${S2_SOURCE_DIR}/s2/strings/serialize.cc"
# ClickHouse doesn't use strings from abseil.
# So, there is no duplicate symbols.
"${S2_SOURCE_DIR}/s2/third_party/absl/base/dynamic_annotations.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/raw_logging.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/throw_delegate.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/numeric/int128.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/ascii.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/match.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/numbers.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_cat.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_split.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/string_view.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/strip.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/internal/memutil.cc"
"${S2_SOURCE_DIR}/s2/util/bits/bit-interleave.cc"
"${S2_SOURCE_DIR}/s2/util/bits/bits.cc"
"${S2_SOURCE_DIR}/s2/util/coding/coder.cc"
"${S2_SOURCE_DIR}/s2/util/coding/varint.cc"
"${S2_SOURCE_DIR}/s2/util/math/exactfloat/exactfloat.cc"
"${S2_SOURCE_DIR}/s2/util/math/mathutil.cc"
"${S2_SOURCE_DIR}/s2/util/units/length-units.cc"
)
add_library(s2 ${S2_SRCS})
if (OPENSSL_FOUND)
target_link_libraries(s2 PRIVATE ${OPENSSL_LIBRARIES})
endif()
target_include_directories(s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/")
if(M_LIBRARY)
target_link_libraries(s2 PRIVATE ${M_LIBRARY})
endif()

View File

@ -380,6 +380,14 @@ function run_tests
01923_network_receive_time_metric_insert
01889_sqlite_read_write
# needs s2
01849_geoToS2
01851_s2_to_geo
01852_s2_get_neighbours
01853_s2_cells_intersect
01854_s2_cap_contains
01854_s2_cap_union
)
time clickhouse-test --hung-check -j 8 --order=random --use-skip-list \

View File

@ -32,7 +32,7 @@ RUN rm -rf \
RUN apt-get clean
# Install MySQL ODBC driver
RUN curl 'https://cdn.mysql.com//Downloads/Connector-ODBC/8.0/mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit.tar.gz' --output 'mysql-connector.tar.gz' && tar -xzf mysql-connector.tar.gz && cd mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit/lib && mv * /usr/local/lib && ln -s /usr/local/lib/libmyodbc8a.so /usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so
RUN curl 'https://downloads.mysql.com/archives/get/p/10/file/mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit.tar.gz' --location --output 'mysql-connector.tar.gz' && tar -xzf mysql-connector.tar.gz && cd mysql-connector-odbc-8.0.21-linux-glibc2.12-x86-64bit/lib && mv * /usr/local/lib && ln -s /usr/local/lib/libmyodbc8a.so /usr/lib/x86_64-linux-gnu/odbc/libmyodbc.so
# Unfortunately this is required for a single test for conversion data from zookeeper to clickhouse-keeper.
# ZooKeeper is not started by default, but consumes some space in containers.
@ -49,4 +49,3 @@ RUN mkdir /zookeeper && chmod -R 777 /zookeeper
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone

View File

@ -79,6 +79,7 @@ SELECT library_name, license_type, license_path FROM system.licenses ORDER BY li
| re2 | BSD 3-clause | /contrib/re2/LICENSE |
| replxx | BSD 3-clause | /contrib/replxx/LICENSE.md |
| rocksdb | BSD 3-clause | /contrib/rocksdb/LICENSE.leveldb |
| s2geometry | Apache | /contrib/s2geometry/LICENSE |
| sentry-native | MIT | /contrib/sentry-native/LICENSE |
| simdjson | Apache | /contrib/simdjson/LICENSE |
| snappy | Public Domain | /contrib/snappy/COPYING |

View File

@ -1,6 +1,6 @@
---
toc_priority: 12
toc_title: MateriaziePostgreSQL
toc_title: MaterializedPostgreSQL
---
# MaterializedPostgreSQL {#materialize-postgresql}

View File

@ -157,5 +157,6 @@ toc_title: Adopters
| <a href="https://signoz.io/" class="favicon">SigNoz</a> | Observability Platform | Main Product | — | — | [Source code](https://github.com/SigNoz/signoz) |
| <a href="https://chelpipegroup.com/" class="favicon">ChelPipe Group</a> | Analytics | — | — | — | [Blog post, June 2021](https://vc.ru/trade/253172-tyazhelomu-proizvodstvu-user-friendly-sayt-internet-magazin-trub-dlya-chtpz) |
| <a href="https://zagravagames.com/en/" class="favicon">Zagrava Trading</a> | — | — | — | — | [Job offer, May 2021](https://twitter.com/datastackjobs/status/1394707267082063874) |
| <a href="https://beeline.ru/" class="favicon">Beeline</a> | Telecom | Data Platform | — | — | [Blog post, July 2021](https://habr.com/en/company/beeline/blog/567508/) |
[Original article](https://clickhouse.tech/docs/en/introduction/adopters/) <!--hide-->

View File

@ -1213,7 +1213,15 @@ Default value: `3`.
## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
If the value is true, integers appear in quotes when using JSON\* Int64 and UInt64 formats (for compatibility with most JavaScript implementations); otherwise, integers are output without the quotes.
Controls quoting of 64-bit or bigger [integers](../../sql-reference/data-types/int-uint.md) (like `UInt64` or `Int128`) when they are output in a [JSON](../../interfaces/formats.md#json) format.
Such integers are enclosed in quotes by default. This behavior is compatible with most JavaScript implementations.
Possible values:
- 0 — Integers are output without quotes.
- 1 — Integers are enclosed in quotes.
Default value: 1.
## output_format_json_quote_denormals {#settings-output_format_json_quote_denormals}
@ -1990,6 +1998,16 @@ Possible values:
Default value: 16.
## merge_selecting_sleep_ms {#merge_selecting_sleep_ms}
Sleep time for merge selecting when no part selected, a lower setting will trigger selecting tasks in background_schedule_pool frequently which result in large amount of requests to zookeeper in large-scale clusters
Possible values:
- Any positive integer.
Default value: 5000
## parallel_distributed_insert_select {#parallel_distributed_insert_select}
Enables parallel distributed `INSERT ... SELECT` query.

View File

@ -8,12 +8,11 @@ Columns:
- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
- `name` ([String](../../sql-reference/data-types/string.md)) — Index name.
- `type` ([String](../../sql-reference/data-types/string.md)) — Index type.
- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression used to calculate the index.
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Number of granules in the block.
- `expr` ([String](../../sql-reference/data-types/string.md)) — Expression for the index calculation.
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of granules in the block.
**Example**
```sql
SELECT * FROM system.data_skipping_indices LIMIT 2 FORMAT Vertical;
```

View File

@ -34,7 +34,7 @@ Input table:
Query:
``` sql
SELECT medianDeterministic(val, 1) FROM t
SELECT medianDeterministic(val, 1) FROM t;
```
Result:

View File

@ -12,9 +12,6 @@ toc_title: Map(key, value)
- `key` — The key part of the pair. [String](../../sql-reference/data-types/string.md) or [Integer](../../sql-reference/data-types/int-uint.md).
- `value` — The value part of the pair. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) or [Array](../../sql-reference/data-types/array.md).
!!! warning "Warning"
Currently `Map` data type is an experimental feature. To work with it you must set `allow_experimental_map_type = 1`.
To get the value from an `a Map('key', 'value')` column, use `a['key']` syntax. This lookup works now with a linear complexity.
**Examples**

View File

@ -195,6 +195,41 @@ Result:
└────────────────────┘
```
## h3ToGeo {#h3togeo}
Returns `(lon, lat)` that corresponds to the provided H3 index.
**Syntax**
``` sql
h3ToGeo(h3Index)
```
**Arguments**
- `h3Index` — H3 Index. Type: [UInt64](../../../sql-reference/data-types/int-uint.md).
**Returned values**
- `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md).
- `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md).
**Example**
Query:
``` sql
SELECT h3ToGeo(644325524701193974) coordinates;
```
Result:
``` text
┌─coordinates───────────────────────────┐
│ (37.79506616830252,55.71290243145668) │
└───────────────────────────────────────┘
```
## h3kRing {#h3kring}
Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order.

View File

@ -306,3 +306,49 @@ Result:
└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
## toJSONString {#tojsonstring}
Serializes a value to its JSON representation. Various data types and nested structures are supported.
64-bit [integers](../../sql-reference/data-types/int-uint.md) or bigger (like `UInt64` or `Int128`) are enclosed in quotes by default. [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) controls this behavior.
Special values `NaN` and `inf` are replaced with `null`. Enable [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals) setting to show them.
When serializing an [Enum](../../sql-reference/data-types/enum.md) value, the function outputs its name.
**Syntax**
``` sql
toJSONString(value)
```
**Arguments**
- `value` — Value to serialize. Value may be of any data type.
**Returned value**
- JSON representation of the value.
Type: [String](../../sql-reference/data-types/string.md).
**Example**
The first example shows serialization of a [Map](../../sql-reference/data-types/map.md).
The second example shows some special values wrapped into a [Tuple](../../sql-reference/data-types/tuple.md).
Query:
``` sql
SELECT toJSONString(map('key1', 1, 'key2', 2));
SELECT toJSONString(tuple(1.25, NULL, NaN, +inf, -inf, [])) SETTINGS output_format_json_quote_denormals = 1;
```
Result:
``` text
{"key1":1,"key2":2}
[1.25,null,"nan","inf","-inf",[]]
```
**See Also**
- [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers)
- [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals)

View File

@ -49,6 +49,7 @@ ENGINE = MaterializeMySQL('host:port', ['database' | database], 'user', 'passwor
| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) |
| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) |
| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) |
| ENUM | [Enum](../../sql-reference/data-types/enum.md) |
| STRING | [String](../../sql-reference/data-types/string.md) |
| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) |
| BLOB | [String](../../sql-reference/data-types/string.md) |

View File

@ -1204,8 +1204,15 @@ load_balancing = round_robin
Работает для форматов JSONEachRow и TSKV.
## output_format_json_quote_64bit_integers {#session_settings-output_format_json_quote_64bit_integers}
Управляет кавычками при выводе 64-битных или более [целых чисел](../../sql-reference/data-types/int-uint.md) (например, `UInt64` или `Int128`) в формате [JSON](../../interfaces/formats.md#json).
По умолчанию такие числа заключаются в кавычки. Это поведение соответствует большинству реализаций JavaScript.
Если значение истинно, то при использовании JSON\* форматов UInt64 и Int64 числа выводятся в кавычках (из соображений совместимости с большинством реализаций JavaScript), иначе - без кавычек.
Возможные значения:
- 0 — числа выводятся без кавычек.
- 1 — числа выводятся в кавычках.
Значение по умолчанию: 1.
## output_format_json_quote_denormals {#settings-output_format_json_quote_denormals}

View File

@ -0,0 +1,38 @@
# system.data_skipping_indices {#system-data-skipping-indices}
Содержит информацию о существующих индексах пропуска данных во всех таблицах.
Столбцы:
- `database` ([String](../../sql-reference/data-types/string.md)) — имя базы данных.
- `table` ([String](../../sql-reference/data-types/string.md)) — имя таблицы.
- `name` ([String](../../sql-reference/data-types/string.md)) — имя индекса.
- `type` ([String](../../sql-reference/data-types/string.md)) — тип индекса.
- `expr` ([String](../../sql-reference/data-types/string.md)) — выражение, используемое для вычисления индекса.
- `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — количество гранул в блоке данных.
**Пример**
```sql
SELECT * FROM system.data_skipping_indices LIMIT 2 FORMAT Vertical;
```
```text
Row 1:
──────
database: default
table: user_actions
name: clicks_idx
type: minmax
expr: clicks
granularity: 1
Row 2:
──────
database: default
table: users
name: contacts_null_idx
type: minmax
expr: assumeNotNull(contacts_null)
granularity: 1
```

View File

@ -4,7 +4,6 @@
Функции:
- `median` — синоним для [quantile](../../../sql-reference/aggregate-functions/reference/quantile.md#quantile).
- `medianDeterministic` — синоним для [quantileDeterministic](../../../sql-reference/aggregate-functions/reference/quantiledeterministic.md#quantiledeterministic).
- `medianExact` — синоним для [quantileExact](../../../sql-reference/aggregate-functions/reference/quantileexact.md#quantileexact).
@ -31,7 +30,7 @@
Запрос:
``` sql
SELECT medianDeterministic(val, 1) FROM t
SELECT medianDeterministic(val, 1) FROM t;
```
Результат:
@ -41,4 +40,3 @@ SELECT medianDeterministic(val, 1) FROM t
│ 1.5 │
└─────────────────────────────┘
```

View File

@ -12,9 +12,6 @@ toc_title: Map(key, value)
- `key` — ключ. [String](../../sql-reference/data-types/string.md) или [Integer](../../sql-reference/data-types/int-uint.md).
- `value` — значение. [String](../../sql-reference/data-types/string.md), [Integer](../../sql-reference/data-types/int-uint.md) или [Array](../../sql-reference/data-types/array.md).
!!! warning "Предупреждение"
Сейчас использование типа данных `Map` является экспериментальной возможностью. Чтобы использовать этот тип данных, включите настройку `allow_experimental_map_type = 1`.
Чтобы получить значение из колонки `a Map('key', 'value')`, используйте синтаксис `a['key']`. В настоящее время такая подстановка работает по алгоритму с линейной сложностью.
**Примеры**

View File

@ -306,3 +306,51 @@ SELECT JSONExtractKeysAndValuesRaw('{"a": [-100, 200.0], "b":{"c": {"d": "hello"
│ [('d','"hello"'),('f','"world"')] │
└───────────────────────────────────────────────────────────────────────────────────────────────────────┘
```
## toJSONString {#tojsonstring}
Сериализует значение в JSON представление. Поддерживаются различные типы данных и вложенные структуры.
По умолчанию 64-битные [целые числа](../../sql-reference/data-types/int-uint.md) и более (например, `UInt64` или `Int128`) заключаются в кавычки. Настройка [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers) управляет этим поведением.
Специальные значения `NaN` и `inf` заменяются на `null`. Чтобы они отображались, включите настройку [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals).
Когда сериализуется значение [Enum](../../sql-reference/data-types/enum.md), то функция выводит его имя.
**Синтаксис**
``` sql
toJSONString(value)
```
**Аргументы**
- `value` — значение, которое необходимо сериализовать. Может быть любого типа.
**Возвращаемое значение**
- JSON представление значения.
Тип: [String](../../sql-reference/data-types/string.md).
**Пример**
Первый пример показывает сериализацию [Map](../../sql-reference/data-types/map.md).
Во втором примере есть специальные значения, обернутые в [Tuple](../../sql-reference/data-types/tuple.md).
Запрос:
``` sql
SELECT toJSONString(map('key1', 1, 'key2', 2));
SELECT toJSONString(tuple(1.25, NULL, NaN, +inf, -inf, [])) SETTINGS output_format_json_quote_denormals = 1;
```
Результат:
``` text
{"key1":1,"key2":2}
[1.25,null,"nan","inf","-inf",[]]
```
**Смотрите также**
- [output_format_json_quote_64bit_integers](../../operations/settings/settings.md#session_settings-output_format_json_quote_64bit_integers)
- [output_format_json_quote_denormals](../../operations/settings/settings.md#settings-output_format_json_quote_denormals)

View File

@ -9,7 +9,7 @@
Do not use any JavaScript or CSS frameworks or preprocessors.
This HTML page should not require any build systems (node.js, npm, gulp, etc.)
This HTML page should not be minified, instead it should be reasonably minimalistic by itself.
This HTML page should not load any external resources
This HTML page should not load any external resources on load.
(CSS and JavaScript must be embedded directly to the page. No external fonts or images should be loaded).
This UI should look as lightweight, clean and fast as possible.
All UI elements must be aligned in pixel-perfect way.
@ -343,13 +343,18 @@
/// Save query in history only if it is different.
let previous_query = '';
/// Substitute the address of the server where the page is served.
if (location.protocol != 'file:') {
const current_url = new URL(window.location);
const server_address = current_url.searchParams.get('url');
if (server_address) {
document.getElementById('url').value = server_address;
} else if (location.protocol != 'file:') {
/// Substitute the address of the server where the page is served.
document.getElementById('url').value = location.origin;
}
/// Substitute user name if it's specified in the query string
let user_from_url = (new URL(window.location)).searchParams.get('user');
const user_from_url = current_url.searchParams.get('user');
if (user_from_url) {
document.getElementById('user').value = user_from_url;
}
@ -361,7 +366,9 @@
let user = document.getElementById('user').value;
let password = document.getElementById('password').value;
let url = document.getElementById('url').value +
let server_address = document.getElementById('url').value;
let url = server_address +
/// Ask server to allow cross-domain requests.
'?add_http_cors_header=1' +
'&user=' + encodeURIComponent(user) +
@ -390,11 +397,18 @@
response: this.response.length > 100000 ? null : this.response /// Lower than the browser's limit.
};
let title = "ClickHouse Query: " + query;
let url = window.location.pathname + '?user=' + encodeURIComponent(user) + '#' + window.btoa(query);
let history_url = window.location.pathname + '?user=' + encodeURIComponent(user);
if (server_address != location.origin) {
/// Save server's address in URL if it's not identical to the address of the play UI.
history_url += '&url=' + encodeURIComponent(server_address);
}
history_url += '#' + window.btoa(query);
if (previous_query == '') {
history.replaceState(state, title, url);
history.replaceState(state, title, history_url);
} else {
history.pushState(state, title, url);
history.pushState(state, title, history_url);
}
document.title = title;
previous_query = query;

View File

@ -419,6 +419,11 @@ if (USE_AWS_S3)
target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${AWS_S3_INCLUDE_DIR})
endif()
if (USE_S2_GEOMETRY)
dbms_target_link_libraries (PUBLIC ${S2_GEOMETRY_LIBRARY})
dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${S2_GEOMETRY_INCLUDE_DIR})
endif()
if (USE_BROTLI)
target_link_libraries (clickhouse_common_io PRIVATE ${BROTLI_LIBRARY})
target_include_directories (clickhouse_common_io SYSTEM BEFORE PRIVATE ${BROTLI_INCLUDE_DIR})

View File

@ -559,6 +559,8 @@
M(589, DISTRIBUTED_BROKEN_BATCH_FILES) \
M(590, CANNOT_SYSCONF) \
M(591, SQLITE_ENGINE_ERROR) \
M(592, DATA_ENCRYPTION_ERROR) \
M(593, ZERO_COPY_REPLICATION_ERROR) \
\
M(998, POSTGRESQL_CONNECTION_FAILURE) \
M(999, KEEPER_EXCEPTION) \

View File

@ -26,13 +26,14 @@ namespace ErrorCodes
MySQLClient::MySQLClient(const String & host_, UInt16 port_, const String & user_, const String & password_)
: host(host_), port(port_), user(user_), password(std::move(password_))
{
client_capability_flags = CLIENT_PROTOCOL_41 | CLIENT_PLUGIN_AUTH | CLIENT_SECURE_CONNECTION;
mysql_context.client_capabilities = CLIENT_PROTOCOL_41 | CLIENT_PLUGIN_AUTH | CLIENT_SECURE_CONNECTION;
}
MySQLClient::MySQLClient(MySQLClient && other)
: host(std::move(other.host)), port(other.port), user(std::move(other.user)), password(std::move(other.password))
, client_capability_flags(other.client_capability_flags)
, mysql_context(other.mysql_context)
{
mysql_context.sequence_id = 0;
}
void MySQLClient::connect()
@ -56,7 +57,7 @@ void MySQLClient::connect()
in = std::make_shared<ReadBufferFromPocoSocket>(*socket);
out = std::make_shared<WriteBufferFromPocoSocket>(*socket);
packet_endpoint = std::make_shared<PacketEndpoint>(*in, *out, seq);
packet_endpoint = mysql_context.makeEndpoint(*in, *out);
handshake();
}
@ -68,7 +69,7 @@ void MySQLClient::disconnect()
socket->close();
socket = nullptr;
connected = false;
seq = 0;
mysql_context.sequence_id = 0;
}
/// https://dev.mysql.com/doc/internals/en/connection-phase-packets.html
@ -87,10 +88,10 @@ void MySQLClient::handshake()
String auth_plugin_data = native41.getAuthPluginData();
HandshakeResponse handshake_response(
client_capability_flags, MAX_PACKET_LENGTH, charset_utf8, user, "", auth_plugin_data, mysql_native_password);
mysql_context.client_capabilities, MAX_PACKET_LENGTH, charset_utf8, user, "", auth_plugin_data, mysql_native_password);
packet_endpoint->sendPacket<HandshakeResponse>(handshake_response, true);
ResponsePacket packet_response(client_capability_flags, true);
ResponsePacket packet_response(mysql_context.client_capabilities, true);
packet_endpoint->receivePacket(packet_response);
packet_endpoint->resetSequenceId();
@ -105,7 +106,7 @@ void MySQLClient::writeCommand(char command, String query)
WriteCommand write_command(command, query);
packet_endpoint->sendPacket<WriteCommand>(write_command, true);
ResponsePacket packet_response(client_capability_flags);
ResponsePacket packet_response(mysql_context.client_capabilities);
packet_endpoint->receivePacket(packet_response);
switch (packet_response.getType())
{
@ -124,7 +125,7 @@ void MySQLClient::registerSlaveOnMaster(UInt32 slave_id)
RegisterSlave register_slave(slave_id);
packet_endpoint->sendPacket<RegisterSlave>(register_slave, true);
ResponsePacket packet_response(client_capability_flags);
ResponsePacket packet_response(mysql_context.client_capabilities);
packet_endpoint->receivePacket(packet_response);
packet_endpoint->resetSequenceId();
if (packet_response.getType() == PACKET_ERR)

View File

@ -45,9 +45,7 @@ private:
String password;
bool connected = false;
UInt32 client_capability_flags = 0;
uint8_t seq = 0;
MySQLWireContext mysql_context;
const UInt8 charset_utf8 = 33;
const String mysql_native_password = "mysql_native_password";

View File

@ -68,4 +68,15 @@ String PacketEndpoint::packetToText(const String & payload)
}
MySQLProtocol::PacketEndpointPtr MySQLWireContext::makeEndpoint(WriteBuffer & out)
{
return MySQLProtocol::PacketEndpoint::create(out, sequence_id);
}
MySQLProtocol::PacketEndpointPtr MySQLWireContext::makeEndpoint(ReadBuffer & in, WriteBuffer & out)
{
return MySQLProtocol::PacketEndpoint::create(in, out, sequence_id);
}
}

View File

@ -5,6 +5,7 @@
#include "IMySQLReadPacket.h"
#include "IMySQLWritePacket.h"
#include "IO/MySQLPacketPayloadReadBuffer.h"
#include <common/shared_ptr_helper.h>
namespace DB
{
@ -15,19 +16,13 @@ namespace MySQLProtocol
/* Writes and reads packets, keeping sequence-id.
* Throws ProtocolError, if packet with incorrect sequence-id was received.
*/
class PacketEndpoint
class PacketEndpoint : public shared_ptr_helper<PacketEndpoint>
{
public:
uint8_t & sequence_id;
ReadBuffer * in;
WriteBuffer * out;
/// For writing.
PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_);
/// For reading and writing.
PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_);
MySQLPacketPayloadReadBuffer getPayload();
void receivePacket(IMySQLReadPacket & packet);
@ -48,8 +43,29 @@ public:
/// Converts packet to text. Is used for debug output.
static String packetToText(const String & payload);
protected:
/// For writing.
PacketEndpoint(WriteBuffer & out_, uint8_t & sequence_id_);
/// For reading and writing.
PacketEndpoint(ReadBuffer & in_, WriteBuffer & out_, uint8_t & sequence_id_);
friend struct shared_ptr_helper<PacketEndpoint>;
};
using PacketEndpointPtr = std::shared_ptr<PacketEndpoint>;
}
struct MySQLWireContext
{
uint8_t sequence_id = 0;
uint32_t client_capabilities = 0;
size_t max_packet_size = 0;
MySQLProtocol::PacketEndpointPtr makeEndpoint(WriteBuffer & out);
MySQLProtocol::PacketEndpointPtr makeEndpoint(ReadBuffer & in, WriteBuffer & out);
};
}
}

View File

@ -1,4 +1,7 @@
#include "Connection.h"
#if USE_LIBPQXX
#include <common/logger_useful.h>
namespace postgres
@ -72,3 +75,5 @@ void Connection::connect()
updateConnection();
}
}
#endif

View File

@ -1,5 +1,11 @@
#pragma once
#if !defined(ARCADIA_BUILD)
#include "config_core.h"
#endif
#if USE_LIBPQXX
#include <pqxx/pqxx> // Y_IGNORE
#include <Core/Types.h>
#include <boost/noncopyable.hpp>
@ -45,3 +51,5 @@ private:
Poco::Logger * log;
};
}
#endif

View File

@ -1,5 +1,11 @@
#pragma once
#if !defined(ARCADIA_BUILD)
#include "config_core.h"
#endif
#if USE_LIBPQXX
#include <pqxx/pqxx> // Y_IGNORE
#include <Core/Types.h>
#include <common/BorrowedObjectPool.h>
@ -35,3 +41,5 @@ private:
using ConnectionHolderPtr = std::unique_ptr<ConnectionHolder>;
}
#endif

View File

@ -1,4 +1,7 @@
#include "PoolWithFailover.h"
#if USE_LIBPQXX
#include "Utils.h"
#include <Common/parseRemoteDescription.h>
#include <Common/Exception.h>
@ -136,3 +139,5 @@ ConnectionHolderPtr PoolWithFailover::get()
throw DB::Exception(DB::ErrorCodes::POSTGRESQL_CONNECTION_FAILURE, "Unable to connect to any of the replicas");
}
}
#endif

View File

@ -1,5 +1,12 @@
#pragma once
#if !defined(ARCADIA_BUILD)
#include "config_core.h"
#endif
#if USE_LIBPQXX
#include "ConnectionHolder.h"
#include <mutex>
#include <Poco/Util/AbstractConfiguration.h>
@ -63,3 +70,5 @@ private:
using PoolWithFailoverPtr = std::shared_ptr<PoolWithFailover>;
}
#endif

View File

@ -1,4 +1,7 @@
#include "Utils.h"
#if USE_LIBPQXX
#include <IO/Operators.h>
namespace postgres
@ -17,3 +20,5 @@ ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, S
}
}
#endif

View File

@ -1,5 +1,11 @@
#pragma once
#if !defined(ARCADIA_BUILD)
#include "config_core.h"
#endif
#if USE_LIBPQXX
#include <pqxx/pqxx> // Y_IGNORE
#include <Core/Types.h>
#include "Connection.h"
@ -15,3 +21,5 @@ namespace postgres
{
ConnectionInfo formatConnectionString(String dbname, String host, UInt16 port, String user, String password);
}
#endif

View File

@ -9,6 +9,7 @@
#include <DataTypes/IDataType.h>
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypesDecimal.h>
#include <Interpreters/convertFieldToType.h>
#include <IO/ReadHelpers.h>
@ -102,7 +103,16 @@ void insertPostgreSQLValue(
assert_cast<ColumnUInt32 &>(column).insertValue(time);
break;
}
case ExternalResultDescription::ValueType::vtDateTime64:[[fallthrough]];
case ExternalResultDescription::ValueType::vtDateTime64:
{
ReadBufferFromString in(value);
DateTime64 time = 0;
readDateTime64Text(time, 6, in, assert_cast<const DataTypeDateTime64 *>(data_type.get())->getTimeZone());
if (time < 0)
time = 0;
assert_cast<ColumnDecimal<Decimal64> &>(column).insertValue(time);
break;
}
case ExternalResultDescription::ValueType::vtDecimal32: [[fallthrough]];
case ExternalResultDescription::ValueType::vtDecimal64: [[fallthrough]];
case ExternalResultDescription::ValueType::vtDecimal128: [[fallthrough]];
@ -206,6 +216,14 @@ void preparePostgreSQLArrayInfo(
readDateTimeText(time, in, assert_cast<const DataTypeDateTime *>(nested.get())->getTimeZone());
return time;
};
else if (which.isDateTime64())
parser = [nested](std::string & field) -> Field
{
ReadBufferFromString in(field);
DateTime64 time = 0;
readDateTime64Text(time, 6, in, assert_cast<const DataTypeDateTime64 *>(nested.get())->getTimeZone());
return time;
};
else if (which.isDecimal32())
parser = [nested](std::string & field) -> Field
{

View File

@ -482,6 +482,8 @@ class IColumn;
M(UInt64, limit, 0, "Limit on read rows from the most 'end' result for select query, default 0 means no limit length", 0) \
M(UInt64, offset, 0, "Offset on read rows from the most 'end' result for select query", 0) \
\
M(UInt64, function_range_max_elements_in_block, 500000000, "Maximum number of values generated by function 'range' per block of data (sum of array sizes for every row in a block, see also 'max_block_size' and 'min_insert_block_size_rows'). It is a safety threshold.", 0) \
\
/** Experimental functions */ \
M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \
\

View File

@ -31,6 +31,10 @@ SRCS(
MySQL/PacketsProtocolText.cpp
MySQL/PacketsReplication.cpp
NamesAndTypes.cpp
PostgreSQL/Connection.cpp
PostgreSQL/PoolWithFailover.cpp
PostgreSQL/Utils.cpp
PostgreSQL/insertPostgreSQLValue.cpp
PostgreSQLProtocol.cpp
QueryProcessingStage.cpp
Settings.cpp

View File

@ -8,7 +8,7 @@
#include <Core/ExternalResultDescription.h>
#include <DataStreams/IBlockInputStream.h>
#include <sqlite3.h>
#include <sqlite3.h> // Y_IGNORE
namespace DB

View File

@ -42,11 +42,23 @@ public:
return it;
}
/// throws exception if value is not valid
const StringRef & getNameForValue(const T & value) const
{
return findByValue(value)->second;
}
/// returns false if value is not valid
bool getNameForValue(const T & value, StringRef & result) const
{
const auto it = value_to_name_map.find(value);
if (it == std::end(value_to_name_map))
return false;
result = it->second;
return true;
}
T getValue(StringRef field_name, bool try_treat_as_id = false) const;
template <typename TValues>

View File

@ -9,7 +9,7 @@
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypesDecimal.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <boost/algorithm/string/split.hpp>
#include <boost/algorithm/string/trim.hpp>
#include <Common/quoteString.h>
@ -71,7 +71,7 @@ static DataTypePtr convertPostgreSQLDataType(String & type, const std::function<
else if (type == "bigserial")
res = std::make_shared<DataTypeUInt64>();
else if (type.starts_with("timestamp"))
res = std::make_shared<DataTypeDateTime>();
res = std::make_shared<DataTypeDateTime64>(6);
else if (type == "date")
res = std::make_shared<DataTypeDate>();
else if (type.starts_with("numeric"))

View File

@ -9,7 +9,7 @@
#include <Databases/DatabasesCommon.h>
#include <Parsers/ASTCreateQuery.h>
#include <sqlite3.h>
#include <sqlite3.h> // Y_IGNORE
namespace DB

View File

@ -7,7 +7,7 @@
#if USE_SQLITE
#include <Storages/StorageSQLite.h>
#include <sqlite3.h>
#include <sqlite3.h> // Y_IGNORE
namespace DB

View File

@ -206,9 +206,9 @@ void DiskDecorator::startup()
delegate->startup();
}
void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context)
void DiskDecorator::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map)
{
delegate->applyNewSettings(config, context);
delegate->applyNewSettings(config, context, config_prefix, map);
}
}

View File

@ -65,11 +65,12 @@ public:
String getUniqueId(const String & path) const override { return delegate->getUniqueId(path); }
bool checkUniqueId(const String & id) const override { return delegate->checkUniqueId(id); }
DiskType::Type getType() const override { return delegate->getType(); }
bool supportZeroCopyReplication() const override { return delegate->supportZeroCopyReplication(); }
void onFreeze(const String & path) override;
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
void shutdown() override;
void startup() override;
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) override;
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override;
protected:
Executor & getExecutor() override;

201
src/Disks/DiskEncrypted.cpp Normal file
View File

@ -0,0 +1,201 @@
#include <Disks/DiskEncrypted.h>
#if USE_SSL
#include <Disks/DiskFactory.h>
#include <IO/FileEncryptionCommon.h>
#include <IO/ReadBufferFromEncryptedFile.h>
#include <IO/WriteBufferFromEncryptedFile.h>
namespace DB
{
namespace ErrorCodes
{
extern const int INCORRECT_DISK_INDEX;
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
extern const int LOGICAL_ERROR;
}
using DiskEncryptedPtr = std::shared_ptr<DiskEncrypted>;
using namespace FileEncryption;
class DiskEncryptedReservation : public IReservation
{
public:
DiskEncryptedReservation(DiskEncryptedPtr disk_, std::unique_ptr<IReservation> reservation_)
: disk(std::move(disk_)), reservation(std::move(reservation_))
{
}
UInt64 getSize() const override { return reservation->getSize(); }
DiskPtr getDisk(size_t i) const override
{
if (i != 0)
throw Exception("Can't use i != 0 with single disk reservation", ErrorCodes::INCORRECT_DISK_INDEX);
return disk;
}
Disks getDisks() const override { return {disk}; }
void update(UInt64 new_size) override { reservation->update(new_size); }
private:
DiskEncryptedPtr disk;
std::unique_ptr<IReservation> reservation;
};
ReservationPtr DiskEncrypted::reserve(UInt64 bytes)
{
auto reservation = delegate->reserve(bytes);
if (!reservation)
return {};
return std::make_unique<DiskEncryptedReservation>(std::static_pointer_cast<DiskEncrypted>(shared_from_this()), std::move(reservation));
}
DiskEncrypted::DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_)
: DiskDecorator(disk_)
, name(name_), key(key_), disk_path(path_)
, disk_absolute_path(delegate->getPath() + disk_path)
{
initialize();
}
void DiskEncrypted::initialize()
{
// use wrapped_disk as an EncryptedDisk store
if (disk_path.empty())
return;
if (disk_path.back() != '/')
throw Exception("Disk path must ends with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR);
delegate->createDirectories(disk_path);
}
std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
const String & path,
size_t buf_size,
size_t estimated_size,
size_t aio_threshold,
size_t mmap_threshold,
MMappedFileCache * mmap_cache) const
{
auto wrapped_path = wrappedPath(path);
auto buffer = delegate->readFile(wrapped_path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
String iv;
size_t offset = 0;
if (exists(path) && getFileSize(path))
{
iv = readIV(kIVSize, *buffer);
offset = kIVSize;
}
else
iv = randomString(kIVSize);
return std::make_unique<ReadBufferFromEncryptedFile>(buf_size, std::move(buffer), iv, key, offset);
}
std::unique_ptr<WriteBufferFromFileBase> DiskEncrypted::writeFile(const String & path, size_t buf_size, WriteMode mode)
{
String iv;
size_t start_offset = 0;
auto wrapped_path = wrappedPath(path);
if (mode == WriteMode::Append && exists(path) && getFileSize(path))
{
auto read_buffer = delegate->readFile(wrapped_path, kIVSize);
iv = readIV(kIVSize, *read_buffer);
start_offset = getFileSize(path);
}
else
iv = randomString(kIVSize);
auto buffer = delegate->writeFile(wrapped_path, buf_size, mode);
return std::make_unique<WriteBufferFromEncryptedFile>(buf_size, std::move(buffer), iv, key, start_offset);
}
size_t DiskEncrypted::getFileSize(const String & path) const
{
auto wrapped_path = wrappedPath(path);
size_t size = delegate->getFileSize(wrapped_path);
return size > kIVSize ? (size - kIVSize) : 0;
}
void DiskEncrypted::truncateFile(const String & path, size_t size)
{
auto wrapped_path = wrappedPath(path);
delegate->truncateFile(wrapped_path, size ? (size + kIVSize) : 0);
}
SyncGuardPtr DiskEncrypted::getDirectorySyncGuard(const String & path) const
{
auto wrapped_path = wrappedPath(path);
return delegate->getDirectorySyncGuard(wrapped_path);
}
void DiskEncrypted::applyNewSettings(
const Poco::Util::AbstractConfiguration & config,
ContextPtr /*context*/,
const String & config_prefix,
const DisksMap & map)
{
String wrapped_disk_name = config.getString(config_prefix + ".disk", "");
if (wrapped_disk_name.empty())
throw Exception("The wrapped disk name can not be empty. An encrypted disk is a wrapper over another disk. "
"Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
key = config.getString(config_prefix + ".key", "");
if (key.empty())
throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
auto wrapped_disk = map.find(wrapped_disk_name);
if (wrapped_disk == map.end())
throw Exception("The wrapped disk must have been announced earlier. No disk with name " + wrapped_disk_name + ". Disk " + name,
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
delegate = wrapped_disk->second;
disk_path = config.getString(config_prefix + ".path", "");
initialize();
}
void registerDiskEncrypted(DiskFactory & factory)
{
auto creator = [](const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr /*context*/,
const DisksMap & map) -> DiskPtr {
String wrapped_disk_name = config.getString(config_prefix + ".disk", "");
if (wrapped_disk_name.empty())
throw Exception("The wrapped disk name can not be empty. An encrypted disk is a wrapper over another disk. "
"Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
String key = config.getString(config_prefix + ".key", "");
if (key.empty())
throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
if (key.size() != cipherKeyLength(defaultCipher()))
throw Exception("Expected key with size " + std::to_string(cipherKeyLength(defaultCipher())) + ", got key with size " + std::to_string(key.size()),
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
auto wrapped_disk = map.find(wrapped_disk_name);
if (wrapped_disk == map.end())
throw Exception("The wrapped disk must have been announced earlier. No disk with name " + wrapped_disk_name + ". Disk " + name,
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
String relative_path = config.getString(config_prefix + ".path", "");
return std::make_shared<DiskEncrypted>(name, wrapped_disk->second, key, relative_path);
};
factory.registerDiskType("encrypted", creator);
}
}
#endif

229
src/Disks/DiskEncrypted.h Normal file
View File

@ -0,0 +1,229 @@
#pragma once
#if !defined(ARCADIA_BUILD)
#include <Common/config.h>
#endif
#if USE_SSL
#include <Disks/IDisk.h>
#include <Disks/DiskDecorator.h>
namespace DB
{
class ReadBufferFromFileBase;
class WriteBufferFromFileBase;
class DiskEncrypted : public DiskDecorator
{
public:
DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_);
const String & getName() const override { return name; }
const String & getPath() const override { return disk_absolute_path; }
ReservationPtr reserve(UInt64 bytes) override;
bool exists(const String & path) const override
{
auto wrapped_path = wrappedPath(path);
return delegate->exists(wrapped_path);
}
bool isFile(const String & path) const override
{
auto wrapped_path = wrappedPath(path);
return delegate->isFile(wrapped_path);
}
bool isDirectory(const String & path) const override
{
auto wrapped_path = wrappedPath(path);
return delegate->isDirectory(wrapped_path);
}
size_t getFileSize(const String & path) const override;
void createDirectory(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->createDirectory(wrapped_path);
}
void createDirectories(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->createDirectories(wrapped_path);
}
void clearDirectory(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->clearDirectory(wrapped_path);
}
void moveDirectory(const String & from_path, const String & to_path) override
{
auto wrapped_from_path = wrappedPath(from_path);
auto wrapped_to_path = wrappedPath(to_path);
delegate->moveDirectory(wrapped_from_path, wrapped_to_path);
}
DiskDirectoryIteratorPtr iterateDirectory(const String & path) override
{
auto wrapped_path = wrappedPath(path);
return delegate->iterateDirectory(wrapped_path);
}
void createFile(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->createFile(wrapped_path);
}
void moveFile(const String & from_path, const String & to_path) override
{
auto wrapped_from_path = wrappedPath(from_path);
auto wrapped_to_path = wrappedPath(to_path);
delegate->moveFile(wrapped_from_path, wrapped_to_path);
}
void replaceFile(const String & from_path, const String & to_path) override
{
auto wrapped_from_path = wrappedPath(from_path);
auto wrapped_to_path = wrappedPath(to_path);
delegate->replaceFile(wrapped_from_path, wrapped_to_path);
}
void listFiles(const String & path, std::vector<String> & file_names) override
{
auto wrapped_path = wrappedPath(path);
delegate->listFiles(wrapped_path, file_names);
}
void copy(const String & from_path, const std::shared_ptr<IDisk> & to_disk, const String & to_path) override
{
IDisk::copy(from_path, to_disk, to_path);
}
std::unique_ptr<ReadBufferFromFileBase> readFile(
const String & path,
size_t buf_size,
size_t estimated_size,
size_t aio_threshold,
size_t mmap_threshold,
MMappedFileCache * mmap_cache) const override;
std::unique_ptr<WriteBufferFromFileBase> writeFile(
const String & path,
size_t buf_size,
WriteMode mode) override;
void removeFile(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->removeFile(wrapped_path);
}
void removeFileIfExists(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->removeFileIfExists(wrapped_path);
}
void removeDirectory(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->removeDirectory(wrapped_path);
}
void removeRecursive(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->removeRecursive(wrapped_path);
}
void removeSharedFile(const String & path, bool flag) override
{
auto wrapped_path = wrappedPath(path);
delegate->removeSharedFile(wrapped_path, flag);
}
void removeSharedRecursive(const String & path, bool flag) override
{
auto wrapped_path = wrappedPath(path);
delegate->removeSharedRecursive(wrapped_path, flag);
}
void removeSharedFileIfExists(const String & path, bool flag) override
{
auto wrapped_path = wrappedPath(path);
delegate->removeSharedFileIfExists(wrapped_path, flag);
}
void setLastModified(const String & path, const Poco::Timestamp & timestamp) override
{
auto wrapped_path = wrappedPath(path);
delegate->setLastModified(wrapped_path, timestamp);
}
Poco::Timestamp getLastModified(const String & path) override
{
auto wrapped_path = wrappedPath(path);
return delegate->getLastModified(wrapped_path);
}
void setReadOnly(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->setReadOnly(wrapped_path);
}
void createHardLink(const String & src_path, const String & dst_path) override
{
auto wrapped_src_path = wrappedPath(src_path);
auto wrapped_dst_path = wrappedPath(dst_path);
delegate->createHardLink(wrapped_src_path, wrapped_dst_path);
}
void truncateFile(const String & path, size_t size) override;
String getUniqueId(const String & path) const override
{
auto wrapped_path = wrappedPath(path);
return delegate->getUniqueId(wrapped_path);
}
void onFreeze(const String & path) override
{
auto wrapped_path = wrappedPath(path);
delegate->onFreeze(wrapped_path);
}
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override;
DiskType::Type getType() const override { return DiskType::Type::Encrypted; }
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
private:
void initialize();
String wrappedPath(const String & path) const
{
// if path starts_with disk_path -> got already wrapped path
if (!disk_path.empty() && path.starts_with(disk_path))
return path;
return disk_path + path;
}
String name;
String key;
String disk_path;
String disk_absolute_path;
};
}
#endif

View File

@ -24,7 +24,8 @@ DiskPtr DiskFactory::create(
const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr context) const
ContextPtr context,
const DisksMap & map) const
{
const auto disk_type = config.getString(config_prefix + ".type", "local");
@ -33,7 +34,7 @@ DiskPtr DiskFactory::create(
throw Exception{"DiskFactory: the disk '" + name + "' has unknown disk type: " + disk_type, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG};
const auto & disk_creator = found->second;
return disk_creator(name, config, config_prefix, context);
return disk_creator(name, config, config_prefix, context, map);
}
}

View File

@ -8,12 +8,14 @@
#include <Poco/Util/AbstractConfiguration.h>
#include <functional>
#include <map>
#include <unordered_map>
namespace DB
{
using DisksMap = std::map<String, DiskPtr>;
/**
* Disk factory. Responsible for creating new disk objects.
*/
@ -24,7 +26,8 @@ public:
const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr context)>;
ContextPtr context,
const DisksMap & map)>;
static DiskFactory & instance();
@ -34,7 +37,8 @@ public:
const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr context) const;
ContextPtr context,
const DisksMap & map) const;
private:
using DiskTypeRegistry = std::unordered_map<String, Creator>;

View File

@ -367,7 +367,8 @@ void registerDiskLocal(DiskFactory & factory)
auto creator = [](const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr context) -> DiskPtr {
ContextPtr context,
const DisksMap & /*map*/) -> DiskPtr {
String path = config.getString(config_prefix + ".path", "");
if (name == "default")
{

View File

@ -100,6 +100,8 @@ public:
DiskType::Type getType() const override { return DiskType::Type::Local; }
bool supportZeroCopyReplication() const override { return false; }
SyncGuardPtr getDirectorySyncGuard(const String & path) const override;
private:

View File

@ -450,7 +450,8 @@ void registerDiskMemory(DiskFactory & factory)
auto creator = [](const String & name,
const Poco::Util::AbstractConfiguration & /*config*/,
const String & /*config_prefix*/,
ContextPtr /*context*/) -> DiskPtr { return std::make_shared<DiskMemory>(name); };
ContextPtr /*context*/,
const DisksMap & /*map*/) -> DiskPtr { return std::make_shared<DiskMemory>(name); };
factory.registerDiskType("memory", creator);
}

View File

@ -92,6 +92,8 @@ public:
DiskType::Type getType() const override { return DiskType::Type::RAM; }
bool supportZeroCopyReplication() const override { return false; }
private:
void createDirectoriesImpl(const String & path);
void replaceFileImpl(const String & from_path, const String & to_path);

View File

@ -37,7 +37,7 @@ DiskSelector::DiskSelector(const Poco::Util::AbstractConfiguration & config, con
auto disk_config_prefix = config_prefix + "." + disk_name;
disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context));
disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context, disks));
}
if (!has_default_disk)
disks.emplace(default_disk_name, std::make_shared<DiskLocal>(default_disk_name, context->getPath(), 0));
@ -62,16 +62,16 @@ DiskSelectorPtr DiskSelector::updateFromConfig(
if (!std::all_of(disk_name.begin(), disk_name.end(), isWordCharASCII))
throw Exception("Disk name can contain only alphanumeric and '_' (" + disk_name + ")", ErrorCodes::EXCESSIVE_ELEMENT_IN_CONFIG);
auto disk_config_prefix = config_prefix + "." + disk_name;
if (result->getDisksMap().count(disk_name) == 0)
{
auto disk_config_prefix = config_prefix + "." + disk_name;
result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context));
result->addToDiskMap(disk_name, factory.create(disk_name, config, disk_config_prefix, context, result->getDisksMap()));
}
else
{
auto disk = old_disks_minus_new_disks[disk_name];
disk->applyNewSettings(config, context);
disk->applyNewSettings(config, context, disk_config_prefix, result->getDisksMap());
old_disks_minus_new_disks.erase(disk_name);
}

View File

@ -12,7 +12,6 @@ namespace DB
class DiskSelector;
using DiskSelectorPtr = std::shared_ptr<const DiskSelector>;
using DisksMap = std::map<String, DiskPtr>;
/// Parse .xml configuration and store information about disks
/// Mostly used for introspection.

View File

@ -12,7 +12,8 @@ struct DiskType
Local,
RAM,
S3,
HDFS
HDFS,
Encrypted
};
static String toString(Type disk_type)
{
@ -26,6 +27,8 @@ struct DiskType
return "s3";
case Type::HDFS:
return "hdfs";
case Type::Encrypted:
return "encrypted";
}
__builtin_unreachable();
}

View File

@ -115,7 +115,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskHDFS::writeFile(const String & path
auto hdfs_path = remote_fs_root_path + file_name;
LOG_DEBUG(log, "{} to file by path: {}. HDFS path: {}", mode == WriteMode::Rewrite ? "Write" : "Append",
backQuote(metadata_path + path), remote_fs_root_path + hdfs_path);
backQuote(metadata_path + path), hdfs_path);
/// Single O_WRONLY in libhdfs adds O_TRUNC
auto hdfs_buffer = std::make_unique<WriteBufferFromHDFS>(hdfs_path,
@ -153,6 +153,14 @@ void DiskHDFS::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper)
});
}
bool DiskHDFS::checkUniqueId(const String & hdfs_uri) const
{
if (!boost::algorithm::starts_with(hdfs_uri, remote_fs_root_path))
return false;
const size_t begin_of_path = hdfs_uri.find('/', hdfs_uri.find("//") + 2);
const String remote_fs_object_path = hdfs_uri.substr(begin_of_path);
return (0 == hdfsExists(hdfs_fs.get(), remote_fs_object_path.c_str()));
}
namespace
{
@ -170,7 +178,8 @@ void registerDiskHDFS(DiskFactory & factory)
auto creator = [](const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr context_) -> DiskPtr
ContextPtr context_,
const DisksMap & /*map*/) -> DiskPtr
{
fs::path disk = fs::path(context_->getPath()) / "disks" / name;
fs::create_directories(disk);

View File

@ -44,6 +44,8 @@ public:
DiskType::Type getType() const override { return DiskType::Type::HDFS; }
bool supportZeroCopyReplication() const override { return true; }
std::unique_ptr<ReadBufferFromFileBase> readFile(
const String & path,
size_t buf_size,
@ -58,6 +60,11 @@ public:
RemoteFSPathKeeperPtr createFSPathKeeper() const override;
/// Check file exists and ClickHouse has an access to it
/// Overrode in remote disk
/// Required for remote disk to ensure that replica has access to data written by other node
bool checkUniqueId(const String & hdfs_uri) const override;
private:
String getRandomName() { return toString(UUIDHelpers::generateV4()); }

View File

@ -1,6 +1,7 @@
#pragma once
#include <Interpreters/Context_fwd.h>
#include <Interpreters/Context.h>
#include <Core/Defines.h>
#include <common/types.h>
#include <Common/CurrentMetrics.h>
@ -211,19 +212,23 @@ public:
/// Return disk type - "local", "s3", etc.
virtual DiskType::Type getType() const = 0;
/// Whether this disk support zero-copy replication.
/// Overrode in remote fs disks.
virtual bool supportZeroCopyReplication() const = 0;
/// Invoked when Global Context is shutdown.
virtual void shutdown() {}
/// Performs action on disk startup.
virtual void startup() {}
/// Return some uniq string for file, overrode for S3
/// Required for distinguish different copies of the same part on S3
/// Return some uniq string for file, overrode for IDiskRemote
/// Required for distinguish different copies of the same part on remote disk
virtual String getUniqueId(const String & path) const { return path; }
/// Check file exists and ClickHouse has an access to it
/// Overrode in DiskS3
/// Required for S3 to ensure that replica has access to data written by other node
/// Overrode in remote FS disks (s3/hdfs)
/// Required for remote disk to ensure that replica has access to data written by other node
virtual bool checkUniqueId(const String & id) const { return exists(id); }
/// Invoked on partitions freeze query.
@ -233,7 +238,7 @@ public:
virtual SyncGuardPtr getDirectorySyncGuard(const String & path) const;
/// Applies new settings for disk in runtime.
virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr) {}
virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr, const String &, const DisksMap &) { }
protected:
friend class DiskDecorator;

View File

@ -344,17 +344,6 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path)
}
void IDiskRemote::removeFileIfExists(const String & path)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
if (fs::exists(fs::path(metadata_path) / path))
{
removeMeta(path, fs_paths_keeper);
removeFromRemoteFS(fs_paths_keeper);
}
}
void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
@ -364,6 +353,18 @@ void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs)
}
void IDiskRemote::removeSharedFileIfExists(const String & path, bool keep_in_remote_fs)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
if (fs::exists(fs::path(metadata_path) / path))
{
removeMeta(path, fs_paths_keeper);
if (!keep_in_remote_fs)
removeFromRemoteFS(fs_paths_keeper);
}
}
void IDiskRemote::removeSharedRecursive(const String & path, bool keep_in_remote_fs)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
@ -488,4 +489,13 @@ bool IDiskRemote::tryReserve(UInt64 bytes)
return false;
}
String IDiskRemote::getUniqueId(const String & path) const
{
Metadata metadata(remote_fs_root_path, metadata_path, path);
String id;
if (!metadata.remote_fs_objects.empty())
id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first;
return id;
}
}

View File

@ -83,12 +83,14 @@ public:
void removeFile(const String & path) override { removeSharedFile(path, false); }
void removeFileIfExists(const String & path) override;
void removeFileIfExists(const String & path) override { removeSharedFileIfExists(path, false); }
void removeRecursive(const String & path) override { removeSharedRecursive(path, false); }
void removeSharedFile(const String & path, bool keep_in_remote_fs) override;
void removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) override;
void removeSharedRecursive(const String & path, bool keep_in_remote_fs) override;
void listFiles(const String & path, std::vector<String> & file_names) override;
@ -117,6 +119,10 @@ public:
ReservationPtr reserve(UInt64 bytes) override;
String getUniqueId(const String & path) const override;
bool checkUniqueId(const String & id) const override = 0;
virtual void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) = 0;
virtual RemoteFSPathKeeperPtr createFSPathKeeper() const = 0;

View File

@ -158,15 +158,6 @@ DiskS3::DiskS3(
{
}
String DiskS3::getUniqueId(const String & path) const
{
Metadata metadata(remote_fs_root_path, metadata_path, path);
String id;
if (!metadata.remote_fs_objects.empty())
id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first;
return id;
}
RemoteFSPathKeeperPtr DiskS3::createFSPathKeeper() const
{
auto settings = current_settings.get();
@ -930,7 +921,7 @@ void DiskS3::onFreeze(const String & path)
revision_file_buf.finalize();
}
void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context)
void DiskS3::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &)
{
auto new_settings = settings_getter(config, "storage_configuration.disks." + name, context);

View File

@ -98,22 +98,21 @@ public:
DiskType::Type getType() const override { return DiskType::Type::S3; }
bool supportZeroCopyReplication() const override { return true; }
void shutdown() override;
void startup() override;
/// Return some uniq string for file
/// Required for distinguish different copies of the same part on S3
String getUniqueId(const String & path) const override;
/// Check file exists and ClickHouse has an access to it
/// Required for S3 to ensure that replica has access to data wroten by other node
/// Overrode in remote disk
/// Required for remote disk to ensure that replica has access to data written by other node
bool checkUniqueId(const String & id) const override;
/// Dumps current revision counter into file 'revision.txt' at given path.
void onFreeze(const String & path) override;
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context) override;
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &) override;
private:
void createFileOperationObject(const String & operation_name, UInt64 revision, const ObjectMetadata & metadata);

View File

@ -167,7 +167,8 @@ void registerDiskS3(DiskFactory & factory)
auto creator = [](const String & name,
const Poco::Util::AbstractConfiguration & config,
const String & config_prefix,
ContextPtr context) -> DiskPtr {
ContextPtr context,
const DisksMap & /*map*/) -> DiskPtr {
S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint")));
if (uri.key.back() != '/')
throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS);

View File

@ -16,6 +16,10 @@ void registerDiskMemory(DiskFactory & factory);
void registerDiskS3(DiskFactory & factory);
#endif
#if USE_SSL
void registerDiskEncrypted(DiskFactory & factory);
#endif
#if USE_HDFS
void registerDiskHDFS(DiskFactory & factory);
#endif
@ -32,6 +36,10 @@ void registerDisks()
registerDiskS3(factory);
#endif
#if USE_SSL
registerDiskEncrypted(factory);
#endif
#if USE_HDFS
registerDiskHDFS(factory);
#endif

View File

@ -10,6 +10,7 @@ PEERDIR(
SRCS(
DiskCacheWrapper.cpp
DiskDecorator.cpp
DiskEncrypted.cpp
DiskFactory.cpp
DiskLocal.cpp
DiskMemory.cpp

View File

@ -33,6 +33,7 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
extern const int FORMAT_IS_NOT_SUITABLE_FOR_INPUT;
extern const int FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT;
extern const int UNSUPPORTED_METHOD;
}
const FormatFactory::Creators & FormatFactory::getCreators(const String & name) const
@ -207,6 +208,9 @@ BlockOutputStreamPtr FormatFactory::getOutputStreamParallelIfPossible(
WriteCallback callback,
const std::optional<FormatSettings> & _format_settings) const
{
if (context->getMySQLProtocolContext() && name != "MySQLWire")
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats");
const auto & output_getter = getCreators(name).output_processor_creator;
const Settings & settings = context->getSettingsRef();
@ -309,7 +313,10 @@ OutputFormatPtr FormatFactory::getOutputFormatParallelIfPossible(
{
const auto & output_getter = getCreators(name).output_processor_creator;
if (!output_getter)
throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output (with processors)", name);
if (context->getMySQLProtocolContext() && name != "MySQLWire")
throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "MySQL protocol does not support custom output formats");
auto format_settings = _format_settings ? *_format_settings : getFormatSettings(context);
@ -344,7 +351,7 @@ OutputFormatPtr FormatFactory::getOutputFormat(
{
const auto & output_getter = getCreators(name).output_processor_creator;
if (!output_getter)
throw Exception("Format " + name + " is not suitable for output (with processors)", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
throw Exception(ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT, "Format {} is not suitable for output (with processors)", name);
if (context->hasQueryContext() && context->getSettingsRef().log_queries)
context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Format, name);

View File

@ -124,3 +124,6 @@ endif()
# Signed integer overflow on user-provided data inside boost::geometry - ignore.
set_source_files_properties("pointInPolygon.cpp" PROPERTIES COMPILE_FLAGS -fno-sanitize=signed-integer-overflow)
# target_link_libraries(clickhouse_functions PRIVATE ${S2_LIBRARY})
target_include_directories(clickhouse_functions SYSTEM PUBLIC ${S2_GEOMETRY_INCLUDE_DIR})

View File

@ -49,47 +49,48 @@ Columns convertConstTupleToConstantElements(const ColumnConst & column)
return res;
}
ColumnWithTypeAndName columnGetNested(const ColumnWithTypeAndName & col)
{
if (col.type->isNullable())
{
const DataTypePtr & nested_type = static_cast<const DataTypeNullable &>(*col.type).getNestedType();
if (!col.column)
{
return ColumnWithTypeAndName{nullptr, nested_type, col.name};
}
else if (const auto * nullable = checkAndGetColumn<ColumnNullable>(*col.column))
{
const auto & nested_col = nullable->getNestedColumnPtr();
return ColumnWithTypeAndName{nested_col, nested_type, col.name};
}
else if (const auto * const_column = checkAndGetColumn<ColumnConst>(*col.column))
{
const auto * nullable_column = checkAndGetColumn<ColumnNullable>(const_column->getDataColumn());
ColumnPtr nullable_res;
if (nullable_column)
{
const auto & nested_col = nullable_column->getNestedColumnPtr();
nullable_res = ColumnConst::create(nested_col, col.column->size());
}
else
{
nullable_res = makeNullable(col.column);
}
return ColumnWithTypeAndName{ nullable_res, nested_type, col.name };
}
else
throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN);
}
return col;
}
ColumnsWithTypeAndName createBlockWithNestedColumns(const ColumnsWithTypeAndName & columns)
{
ColumnsWithTypeAndName res;
for (const auto & col : columns)
{
if (col.type->isNullable())
{
const DataTypePtr & nested_type = static_cast<const DataTypeNullable &>(*col.type).getNestedType();
if (!col.column)
{
res.emplace_back(ColumnWithTypeAndName{nullptr, nested_type, col.name});
}
else if (const auto * nullable = checkAndGetColumn<ColumnNullable>(*col.column))
{
const auto & nested_col = nullable->getNestedColumnPtr();
res.emplace_back(ColumnWithTypeAndName{nested_col, nested_type, col.name});
}
else if (const auto * const_column = checkAndGetColumn<ColumnConst>(*col.column))
{
const auto * nullable_column = checkAndGetColumn<ColumnNullable>(const_column->getDataColumn());
ColumnPtr nullable_res;
if (nullable_column)
{
const auto & nested_col = nullable_column->getNestedColumnPtr();
nullable_res = ColumnConst::create(nested_col, col.column->size());
}
else
{
nullable_res = makeNullable(col.column);
}
res.emplace_back(ColumnWithTypeAndName{ nullable_res, nested_type, col.name });
}
else
throw Exception("Illegal column for DataTypeNullable", ErrorCodes::ILLEGAL_COLUMN);
}
else
res.emplace_back(col);
}
res.emplace_back(columnGetNested(col));
return res;
}

View File

@ -81,6 +81,8 @@ inline std::enable_if_t<IsDecimalNumber<T>, Field> toField(const T & x, UInt32 s
Columns convertConstTupleToConstantElements(const ColumnConst & column);
/// Returns nested column with corrected type if nullable
ColumnWithTypeAndName columnGetNested(const ColumnWithTypeAndName & col);
/// Returns the copy of a given columns in which each column is replaced with its respective nested
/// column if it is nullable.

View File

@ -580,36 +580,44 @@ template <typename Name> struct ConvertImpl<DataTypeDateTime, DataTypeDateTime64
template <typename DataType>
struct FormatImpl
{
static void execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType *, const DateLUTImpl *)
template <typename ReturnType = void>
static ReturnType execute(const typename DataType::FieldType x, WriteBuffer & wb, const DataType *, const DateLUTImpl *)
{
writeText(x, wb);
return ReturnType(true);
}
};
template <>
struct FormatImpl<DataTypeDate>
{
static void execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl *)
template <typename ReturnType = void>
static ReturnType execute(const DataTypeDate::FieldType x, WriteBuffer & wb, const DataTypeDate *, const DateLUTImpl *)
{
writeDateText(DayNum(x), wb);
return ReturnType(true);
}
};
template <>
struct FormatImpl<DataTypeDateTime>
{
static void execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime *, const DateLUTImpl * time_zone)
template <typename ReturnType = void>
static ReturnType execute(const DataTypeDateTime::FieldType x, WriteBuffer & wb, const DataTypeDateTime *, const DateLUTImpl * time_zone)
{
writeDateTimeText(x, wb, *time_zone);
return ReturnType(true);
}
};
template <>
struct FormatImpl<DataTypeDateTime64>
{
static void execute(const DataTypeDateTime64::FieldType x, WriteBuffer & wb, const DataTypeDateTime64 * type, const DateLUTImpl * time_zone)
template <typename ReturnType = void>
static ReturnType execute(const DataTypeDateTime64::FieldType x, WriteBuffer & wb, const DataTypeDateTime64 * type, const DateLUTImpl * time_zone)
{
writeDateTimeText(DateTime64(x), type->getScale(), wb, *time_zone);
return ReturnType(true);
}
};
@ -617,18 +625,34 @@ struct FormatImpl<DataTypeDateTime64>
template <typename FieldType>
struct FormatImpl<DataTypeEnum<FieldType>>
{
static void execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum<FieldType> * type, const DateLUTImpl *)
template <typename ReturnType = void>
static ReturnType execute(const FieldType x, WriteBuffer & wb, const DataTypeEnum<FieldType> * type, const DateLUTImpl *)
{
writeString(type->getNameForValue(x), wb);
static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
if constexpr (throw_exception)
{
writeString(type->getNameForValue(x), wb);
}
else
{
StringRef res;
bool is_ok = type->getNameForValue(x, res);
if (is_ok)
writeString(res, wb);
return ReturnType(is_ok);
}
}
};
template <typename FieldType>
struct FormatImpl<DataTypeDecimal<FieldType>>
{
static void execute(const FieldType x, WriteBuffer & wb, const DataTypeDecimal<FieldType> * type, const DateLUTImpl *)
template <typename ReturnType = void>
static ReturnType execute(const FieldType x, WriteBuffer & wb, const DataTypeDecimal<FieldType> * type, const DateLUTImpl *)
{
writeText(x, type->getScale(), wb);
return ReturnType(true);
}
};
@ -643,6 +667,16 @@ struct ConvertImpl<DataTypeEnum<FieldType>, DataTypeNumber<FieldType>, Name, Con
}
};
static ColumnUInt8::MutablePtr copyNullMap(ColumnPtr col)
{
ColumnUInt8::MutablePtr null_map = nullptr;
if (const auto * col_null = checkAndGetColumn<ColumnNullable>(col.get()))
{
null_map = ColumnUInt8::create();
null_map->insertRangeFrom(col_null->getNullMapColumn(), 0, col_null->size());
}
return null_map;
}
template <typename FromDataType, typename Name>
struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType, DataTypeString>, DataTypeString>, Name, ConvertDefaultBehaviorTag>
@ -652,13 +686,18 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/)
{
const auto & col_with_type_and_name = arguments[0];
ColumnUInt8::MutablePtr null_map = copyNullMap(arguments[0].column);
const auto & col_with_type_and_name = columnGetNested(arguments[0]);
const auto & type = static_cast<const FromDataType &>(*col_with_type_and_name.type);
const DateLUTImpl * time_zone = nullptr;
/// For argument of DateTime type, second argument with time zone could be specified.
if constexpr (std::is_same_v<FromDataType, DataTypeDateTime> || std::is_same_v<FromDataType, DataTypeDateTime64>)
time_zone = &extractTimeZoneFromFunctionArguments(arguments, 1, 0);
{
auto non_null_args = createBlockWithNestedColumns(arguments);
time_zone = &extractTimeZoneFromFunctionArguments(non_null_args, 1, 0);
}
if (const auto col_from = checkAndGetColumn<ColVecType>(col_with_type_and_name.column.get()))
{
@ -684,14 +723,30 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
WriteBufferFromVector<ColumnString::Chars> write_buffer(data_to);
for (size_t i = 0; i < size; ++i)
if (null_map)
{
FormatImpl<FromDataType>::execute(vec_from[i], write_buffer, &type, time_zone);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
for (size_t i = 0; i < size; ++i)
{
bool is_ok = FormatImpl<FromDataType>::template execute<bool>(vec_from[i], write_buffer, &type, time_zone);
null_map->getData()[i] |= !is_ok;
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
}
}
else
{
for (size_t i = 0; i < size; ++i)
{
FormatImpl<FromDataType>::template execute<void>(vec_from[i], write_buffer, &type, time_zone);
writeChar(0, write_buffer);
offsets_to[i] = write_buffer.count();
}
}
write_buffer.finalize();
if (null_map)
return ColumnNullable::create(std::move(col_to), std::move(null_map));
return col_to;
}
else
@ -705,9 +760,11 @@ struct ConvertImpl<FromDataType, std::enable_if_t<!std::is_same_v<FromDataType,
/// Generic conversion of any type to String.
struct ConvertImplGenericToString
{
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments)
static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type)
{
const auto & col_with_type_and_name = arguments[0];
ColumnUInt8::MutablePtr null_map = copyNullMap(arguments[0].column);
const auto & col_with_type_and_name = columnGetNested(arguments[0]);
const IDataType & type = *col_with_type_and_name.type;
const IColumn & col_from = *col_with_type_and_name.column;
@ -733,6 +790,9 @@ struct ConvertImplGenericToString
}
write_buffer.finalize();
if (result_type->isNullable() && null_map)
return ColumnNullable::create(std::move(col_to), std::move(null_map));
return col_to;
}
};
@ -1420,7 +1480,11 @@ public:
/// Function actually uses default implementation for nulls,
/// but we need to know if return type is Nullable or not,
/// so we use checked_return_type only to intercept the first call to getReturnTypeImpl(...).
bool useDefaultImplementationForNulls() const override { return checked_return_type; }
bool useDefaultImplementationForNulls() const override
{
bool to_nullable_string = to_nullable && std::is_same_v<ToDataType, DataTypeString>;
return checked_return_type && !to_nullable_string;
}
bool useDefaultImplementationForConstants() const override { return true; }
ColumnNumbers getArgumentsThatAreAlwaysConstant() const override
@ -1485,7 +1549,10 @@ private:
throw Exception{"Function " + getName() + " expects at least 1 argument",
ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION};
const IDataType * from_type = arguments[0].type.get();
if (result_type->onlyNull())
return result_type->createColumnConstWithDefaultValue(input_rows_count);
const DataTypePtr from_type = removeNullable(arguments[0].type);
ColumnPtr result_column;
auto call = [&](const auto & types, const auto & tag) -> bool
@ -1581,7 +1648,7 @@ private:
/// Generic conversion of any type to String.
if (std::is_same_v<ToDataType, DataTypeString>)
{
return ConvertImplGenericToString::execute(arguments);
return ConvertImplGenericToString::execute(arguments, result_type);
}
else
throw Exception("Illegal type " + arguments[0].type->getName() + " of argument of function " + getName(),

View File

@ -58,10 +58,10 @@ struct CountEqualAction
namespace Impl
{
template <
class ConcreteAction,
typename ConcreteAction,
bool RightArgIsConstant = false,
class IntegralInitial = UInt64,
class IntegralResult = UInt64>
typename IntegralInitial = UInt64,
typename IntegralResult = UInt64>
struct Main
{
private:
@ -94,13 +94,13 @@ private:
}
/// LowCardinality
static bool compare(const IColumn & left, const Result& right, size_t i, size_t)
static bool compare(const IColumn & left, const Result & right, size_t i, size_t)
{
return left.getUInt(i) == right;
}
/// Generic
static bool compare(const IColumn& left, const IColumn& right, size_t i, size_t j)
static bool compare(const IColumn & left, const IColumn & right, size_t i, size_t j)
{
return 0 == left.compareAt(i, RightArgIsConstant ? 0 : j, right, 1);
}
@ -109,7 +109,7 @@ private:
static constexpr bool hasNull(const NullMap * const null_map, size_t i) noexcept { return (*null_map)[i]; }
template <size_t Case, class Data, class Target>
template <size_t Case, typename Data, typename Target>
static void process(
const Data & data, const ArrOffsets & offsets, const Target & target, ResultArr & result,
[[maybe_unused]] const NullMap * const null_map_data,
@ -148,7 +148,7 @@ private:
continue;
}
else if (!compare(data, target, current_offset + j, i))
continue;
continue;
ConcreteAction::apply(current, j);
@ -162,7 +162,7 @@ private:
}
public:
template <class Data, class Target>
template <typename Data, typename Target>
static void vector(
const Data & data,
const ArrOffsets & offsets,
@ -183,7 +183,7 @@ public:
};
/// When the 2nd function argument is a NULL value.
template <class ConcreteAction>
template <typename ConcreteAction>
struct Null
{
using ResultType = typename ConcreteAction::ResultType;
@ -227,7 +227,7 @@ struct Null
}
};
template <class ConcreteAction>
template <typename ConcreteAction>
struct String
{
private:
@ -350,7 +350,7 @@ public:
};
}
template <class ConcreteAction, class Name>
template <typename ConcreteAction, typename Name>
class FunctionArrayIndex : public IFunction
{
public:
@ -565,7 +565,7 @@ private:
* Integral s = {s1, s2, ...}
* (s1, s1, s2, ...), (s2, s1, s2, ...), (s3, s1, s2, ...)
*/
template <class ...Integral>
template <typename... Integral>
static inline ColumnPtr executeIntegral(const ColumnsWithTypeAndName & arguments)
{
const ColumnArray * const left = checkAndGetColumn<ColumnArray>(arguments[0].column.get());
@ -590,14 +590,14 @@ private:
return nullptr;
}
template <class ...Integral>
template <typename... Integral>
static inline bool executeIntegral(ExecutionData& data)
{
return (executeIntegralExpanded<Integral, Integral...>(data) || ...);
}
/// Invoke executeIntegralImpl with such parameters: (A, other1), (A, other2), ...
template <class A, class ...Other>
template <typename A, typename... Other>
static inline bool executeIntegralExpanded(ExecutionData& data)
{
return (executeIntegralImpl<A, Other>(data) || ...);
@ -608,7 +608,7 @@ private:
* second argument, namely, the @e value, so it's possible to invoke the <tt>has(Array(Int8), UInt64)</tt> e.g.
* so we have to check all possible variants for #Initial and #Resulting types.
*/
template <class Initial, class Resulting>
template <typename Initial, typename Resulting>
static bool executeIntegralImpl(ExecutionData& data)
{
const ColumnVector<Initial> * col_nested = checkAndGetColumn<ColumnVector<Initial>>(&data.left);
@ -647,7 +647,7 @@ private:
}
/**
* Catches arguments of type LC(T) (left) and U (right).
* Catches arguments of type LowCardinality(T) (left) and U (right).
*
* The perftests
* https://clickhouse-test-reports.s3.yandex.net/12550/2d27fa0fa8c198a82bf1fe3625050ccf56695976/integration_tests_(release).html
@ -726,7 +726,7 @@ private:
return col_result;
}
else if (col_lc->nestedIsNullable()) // LC(Nullable(T)) and U
else if (col_lc->nestedIsNullable()) // LowCardinality(Nullable(T)) and U
{
const ColumnPtr left_casted = col_lc->convertToFullColumnIfLowCardinality(); // Nullable(T)
const ColumnNullable& left_nullable = *checkAndGetColumn<ColumnNullable>(left_casted.get());
@ -746,16 +746,17 @@ private:
? right_nullable->getNestedColumn()
: *right_casted.get();
ExecutionData data = {
ExecutionData data =
{
left_ptr, right_ptr,
col_array->getOffsets(),
nullptr,
{null_map_left_casted, null_map_right_casted}};
if (dispatchConvertedLCColumns(data))
if (dispatchConvertedLowCardinalityColumns(data))
return data.result_column;
}
else // LC(T) and U, T not Nullable
else // LowCardinality(T) and U, T not Nullable
{
if (col_arg.isNullable())
return nullptr;
@ -764,24 +765,25 @@ private:
arg_lc && arg_lc->isNullable())
return nullptr;
// LC(T) and U (possibly LC(V))
// LowCardinality(T) and U (possibly LowCardinality(V))
const ColumnPtr left_casted = col_lc->convertToFullColumnIfLowCardinality();
const ColumnPtr right_casted = col_arg.convertToFullColumnIfLowCardinality();
ExecutionData data = {
ExecutionData data =
{
*left_casted.get(), *right_casted.get(), col_array->getOffsets(),
nullptr, {null_map_data, null_map_item}
};
if (dispatchConvertedLCColumns(data))
if (dispatchConvertedLowCardinalityColumns(data))
return data.result_column;
}
return nullptr;
}
static bool dispatchConvertedLCColumns(ExecutionData& data)
static bool dispatchConvertedLowCardinalityColumns(ExecutionData & data)
{
if (data.left.isNumeric() && data.right.isNumeric()) // ColumnArrays
return executeIntegral<INTEGRAL_TPL_PACK>(data);

View File

@ -7,6 +7,7 @@
#include <Columns/ColumnArray.h>
#include <Columns/ColumnVector.h>
#include <Interpreters/castColumn.h>
#include <Interpreters/Context.h>
#include <numeric>
@ -31,8 +32,10 @@ class FunctionRange : public IFunction
{
public:
static constexpr auto name = "range";
static constexpr size_t max_elements = 100'000'000;
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionRange>(); }
const size_t max_elements;
static FunctionPtr create(ContextPtr context_) { return std::make_shared<FunctionRange>(std::move(context_)); }
explicit FunctionRange(ContextPtr context) : max_elements(context->getSettingsRef().function_range_max_elements_in_block) {}
private:
String getName() const override { return name; }

View File

@ -6,4 +6,5 @@
#cmakedefine01 USE_SIMDJSON
#cmakedefine01 USE_RAPIDJSON
#cmakedefine01 USE_H3
#cmakedefine01 USE_S2_GEOMETRY
#cmakedefine01 USE_FASTOPS

View File

@ -46,20 +46,23 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isFloat64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be Float64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be Float64",
arg->getName(), 1, getName());
arg = arguments[1].get();
if (!WhichDataType(arg).isFloat64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be Float64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be Float64",
arg->getName(), 2, getName());
arg = arguments[2].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(3) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt8",
arg->getName(), 3, getName());
return std::make_shared<DataTypeUInt64>();
}

111
src/Functions/geoToS2.cpp Normal file
View File

@ -0,0 +1,111 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypesNumber.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <Common/NaNUtils.h>
#include <common/range.h>
#include "s2_fwd.h"
class S2CellId;
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
namespace
{
/**
* Accepts points of the form (longitude, latitude)
* Returns s2 identifier
*/
class FunctionGeoToS2 : public IFunction
{
public:
static constexpr auto name = "geoToS2";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionGeoToS2>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 2; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t i = 0; i < getNumberOfArguments(); ++i)
{
const auto * arg = arguments[i].get();
if (!WhichDataType(arg).isFloat64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be Float64",
arg->getName(), i, getName());
}
return std::make_shared<DataTypeUInt64>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_lon = arguments[0].column.get();
const auto * col_lat = arguments[1].column.get();
auto dst = ColumnVector<UInt64>::create();
auto & dst_data = dst->getData();
dst_data.resize(input_rows_count);
for (const auto row : collections::range(0, input_rows_count))
{
const Float64 lon = col_lon->getFloat64(row);
const Float64 lat = col_lat->getFloat64(row);
if (isNaN(lon) || isNaN(lat))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Arguments must not be NaN");
if (!(isFinite(lon) && isFinite(lat)))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Arguments must not be infinite");
/// S2 acceptes point as (latitude, longitude)
S2LatLng lat_lng = S2LatLng::FromDegrees(lat, lon);
S2CellId id(lat_lng);
dst_data[row] = id.id();
}
return dst;
}
};
}
void registerFunctionGeoToS2(FunctionFactory & factory)
{
factory.registerFunction<FunctionGeoToS2>();
}
}
#endif

View File

@ -44,8 +44,9 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt8",
arg->getName(), 1, getName());
return std::make_shared<DataTypeFloat64>();
}
@ -62,8 +63,10 @@ public:
{
const int resolution = col_hindex->getUInt(row);
if (resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
throw Exception(
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
resolution, getName(), MAX_H3_RES);
// Numerical constant is 180 degrees / pi / Earth radius, Earth radius is from h3 sources
Float64 res = 8.99320592271288084e-6 * getHexagonEdgeLengthAvgM(resolution);

View File

@ -49,8 +49,9 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt8",
arg->getName(), 1, getName());
return std::make_shared<DataTypeFloat64>();
}
@ -67,8 +68,10 @@ public:
{
const UInt64 resolution = col_hindex->getUInt(row);
if (resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
throw Exception(
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
resolution, getName(), MAX_H3_RES);
Float64 res = getHexagonEdgeLengthAvgM(resolution);

View File

@ -41,8 +41,9 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
return std::make_shared<DataTypeUInt8>();
}

View File

@ -41,8 +41,9 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
return std::make_shared<DataTypeUInt8>();
}

View File

@ -44,8 +44,9 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt8",
arg->getName(), 1, getName());
return std::make_shared<DataTypeFloat64>();
}
@ -62,8 +63,10 @@ public:
{
const UInt64 resolution = col_hindex->getUInt(row);
if (resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
throw Exception(
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ",
resolution, getName(), MAX_H3_RES);
Float64 res = getHexagonAreaAvgM2(resolution);

View File

@ -41,14 +41,16 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
arg = arguments[1].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 2, getName());
return std::make_shared<DataTypeUInt8>();
}

View File

@ -41,8 +41,9 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
return std::make_shared<DataTypeUInt8>();
}

View File

@ -50,14 +50,16 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
arg = arguments[1].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt8",
arg->getName(), 2, getName());
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
}
@ -81,14 +83,17 @@ public:
const UInt8 child_resolution = col_resolution->getUInt(row);
if (child_resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(child_resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
throw Exception(
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}",
toString(child_resolution), getName(), toString(MAX_H3_RES));
const size_t vec_size = cellToChildrenSize(parent_hindex, child_resolution);
if (vec_size > MAX_ARRAY_SIZE)
throw Exception("The result of function" + getName()
+ " (array of " + toString(vec_size) + " elements) will be too large with resolution argument = "
+ toString(child_resolution), ErrorCodes::TOO_LARGE_ARRAY_SIZE);
throw Exception(
ErrorCodes::TOO_LARGE_ARRAY_SIZE,
"The result of function {} (array of {} elements) will be too large with resolution argument = {}",
getName(), toString(vec_size), toString(child_resolution));
hindex_vec.resize(vec_size);
cellToChildren(parent_hindex, child_resolution, hindex_vec.data());

View File

@ -44,14 +44,16 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
arg = arguments[1].get();
if (!WhichDataType(arg).isUInt8())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be UInt8",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt8",
arg->getName(), 2, getName());
return std::make_shared<DataTypeUInt64>();
}
@ -71,8 +73,10 @@ public:
const UInt8 resolution = col_resolution->getUInt(row);
if (resolution > MAX_H3_RES)
throw Exception("The argument 'resolution' (" + toString(resolution) + ") of function " + getName()
+ " is out of bounds because the maximum resolution in H3 library is " + toString(MAX_H3_RES), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
throw Exception(
ErrorCodes::ARGUMENT_OUT_OF_BOUND,
"The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is {}",
toString(resolution), getName(), toString(MAX_H3_RES));
UInt64 res = cellToParent(hindex, resolution);

View File

@ -42,8 +42,9 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
return std::make_shared<DataTypeString>();
}
@ -67,16 +68,14 @@ public:
const UInt64 hindex = col_hindex->getUInt(i);
if (!isValidCell(hindex))
{
throw Exception("Invalid H3 index: " + std::to_string(hindex), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
}
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Invalid H3 index: {}", hindex);
h3ToString(hindex, pos, H3_INDEX_STRING_LENGTH);
// move to end of the index
while (*pos != '\0')
{
pos++;
}
vec_offsets[i] = ++pos - begin;
}
vec_res.resize(pos - begin);

View File

@ -47,14 +47,16 @@ public:
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(1) + " of function " + getName() + ". Must be UInt64",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
arg = arguments[1].get();
if (!isInteger(arg))
throw Exception(
"Illegal type " + arg->getName() + " of argument " + std::to_string(2) + " of function " + getName() + ". Must be integer",
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be integer",
arg->getName(), 2, getName());
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
}

96
src/Functions/h3toGeo.cpp Normal file
View File

@ -0,0 +1,96 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_H3
#include <array>
#include <math.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnTuple.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <Functions/FunctionFactory.h>
#include <Functions/IFunction.h>
#include <Common/typeid_cast.h>
#include <h3api.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
namespace
{
/// Implements the function h3ToGeo which takes a single argument (h3Index)
/// and returns the longitude and latitude that correspond to the provided h3 index
class FunctionH3ToGeo : public IFunction
{
public:
static constexpr auto name = "h3ToGeo";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionH3ToGeo>(); }
std::string getName() const override { return name; }
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), 1, getName());
return std::make_shared<DataTypeTuple>(
DataTypes{std::make_shared<DataTypeFloat64>(), std::make_shared<DataTypeFloat64>()},
Strings{"longitude", "latitude"});
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_index = arguments[0].column.get();
auto latitude = ColumnFloat64::create(input_rows_count);
auto longitude = ColumnFloat64::create(input_rows_count);
ColumnFloat64::Container & lon_data = longitude->getData();
ColumnFloat64::Container & lat_data = latitude->getData();
for (size_t row = 0; row < input_rows_count; ++row)
{
H3Index h3index = col_index->getUInt(row);
LatLng coord{};
cellToLatLng(h3index,&coord);
lon_data[row] = radsToDegs(coord.lng);
lat_data[row] = radsToDegs(coord.lat);
}
MutableColumns columns;
columns.emplace_back(std::move(longitude));
columns.emplace_back(std::move(latitude));
return ColumnTuple::create(std::move(columns));
}
};
}
void registerFunctionH3ToGeo(FunctionFactory & factory)
{
factory.registerFunction<FunctionH3ToGeo>();
}
}
#endif

View File

@ -28,6 +28,7 @@ void registerFunctionSvg(FunctionFactory & factory);
#if USE_H3
void registerFunctionGeoToH3(FunctionFactory &);
void registerFunctionH3ToGeo(FunctionFactory &);
void registerFunctionH3EdgeAngle(FunctionFactory &);
void registerFunctionH3EdgeLengthM(FunctionFactory &);
void registerFunctionH3GetResolution(FunctionFactory &);
@ -42,6 +43,19 @@ void registerFunctionH3ToString(FunctionFactory &);
void registerFunctionH3HexAreaM2(FunctionFactory &);
#endif
#if USE_S2_GEOMETRY
void registerFunctionGeoToS2(FunctionFactory &);
void registerFunctionS2ToGeo(FunctionFactory &);
void registerFunctionS2GetNeighbors(FunctionFactory &);
void registerFunctionS2CellsIntersect(FunctionFactory &);
void registerFunctionS2CapContains(FunctionFactory &);
void registerFunctionS2CapUnion(FunctionFactory &);
void registerFunctionS2RectAdd(FunctionFactory &);
void registerFunctionS2RectContains(FunctionFactory &);
void registerFunctionS2RectUnion(FunctionFactory &);
void registerFunctionS2RectIntersection(FunctionFactory &);
#endif
void registerFunctionsGeo(FunctionFactory & factory)
{
@ -66,6 +80,7 @@ void registerFunctionsGeo(FunctionFactory & factory)
#if USE_H3
registerFunctionGeoToH3(factory);
registerFunctionH3ToGeo(factory);
registerFunctionH3EdgeAngle(factory);
registerFunctionH3EdgeLengthM(factory);
registerFunctionH3GetResolution(factory);
@ -79,6 +94,19 @@ void registerFunctionsGeo(FunctionFactory & factory)
registerFunctionH3ToString(factory);
registerFunctionH3HexAreaM2(factory);
#endif
#if USE_S2_GEOMETRY
registerFunctionGeoToS2(factory);
registerFunctionS2ToGeo(factory);
registerFunctionS2GetNeighbors(factory);
registerFunctionS2CellsIntersect(factory);
registerFunctionS2CapContains(factory);
registerFunctionS2CapUnion(factory);
registerFunctionS2RectAdd(factory);
registerFunctionS2RectContains(factory);
registerFunctionS2RectUnion(factory);
registerFunctionS2RectIntersection(factory);
#endif
}
}

View File

@ -0,0 +1,132 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <Common/NaNUtils.h>
#include <common/range.h>
#include "s2_fwd.h"
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
}
namespace
{
/**
* The cap represents a portion of the sphere that has been cut off by a plane.
* It is defined by a point on a sphere and a radius in degrees.
* Imagine that we draw a line through the center of the sphere and our point.
* An infinite number of planes pass through this line, but any plane will intersect the cap in two points.
* Thus the angle is defined by one of this points and the entire line.
* So, the radius of Pi/2 defines a hemisphere and the radius of Pi defines a whole sphere.
*
* This function returns whether a cap contains a point.
*/
class FunctionS2CapContains : public IFunction
{
public:
static constexpr auto name = "s2CapContains";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionS2CapContains>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 3; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t index = 0; index < getNumberOfArguments(); ++index)
{
const auto * arg = arguments[index].get();
/// Radius
if (index == 1)
{
if (!WhichDataType(arg).isFloat64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be Float64",
arg->getName(), 2, getName());
}
else if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), index + 1, getName());
}
return std::make_shared<DataTypeUInt8>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_center = arguments[0].column.get();
const auto * col_degrees = arguments[1].column.get();
const auto * col_point = arguments[2].column.get();
auto dst = ColumnUInt8::create();
auto & dst_data = dst->getData();
dst_data.reserve(input_rows_count);
for (const auto row : collections::range(0, input_rows_count))
{
const auto center = S2CellId(col_center->getUInt(row));
const Float64 degrees = col_degrees->getFloat64(row);
const auto point = S2CellId(col_point->getUInt(row));
if (isNaN(degrees))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan");
if (std::isinf(degrees))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be infinite");
if (!center.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Center is not valid");
if (!point.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid");
S1Angle angle = S1Angle::Degrees(degrees);
S2Cap cap(center.ToPoint(), angle);
dst_data.emplace_back(cap.Contains(point.ToPoint()));
}
return dst;
}
};
}
void registerFunctionS2CapContains(FunctionFactory & factory)
{
factory.registerFunction<FunctionS2CapContains>();
}
}
#endif

View File

@ -0,0 +1,141 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <Common/NaNUtils.h>
#include <common/range.h>
#include "s2_fwd.h"
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
}
namespace
{
/**
* The cap represents a portion of the sphere that has been cut off by a plane.
* See comment for s2CapContains function.
* This function returns the smallest cap that contains both of input caps.
* It is represented by identifier of the center and a radius.
*/
class FunctionS2CapUnion : public IFunction
{
public:
static constexpr auto name = "s2CapUnion";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionS2CapUnion>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 4; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t index = 0; index < getNumberOfArguments(); ++index)
{
const auto * arg = arguments[index].get();
if (index == 1 || index == 3)
{
if (!WhichDataType(arg).isFloat64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be Float64",
arg->getName(), index + 1, getName());
}
else if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), index + 1, getName()
);
}
DataTypePtr center = std::make_shared<DataTypeUInt64>();
DataTypePtr radius = std::make_shared<DataTypeFloat64>();
return std::make_shared<DataTypeTuple>(DataTypes{center, radius});
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_center1 = arguments[0].column.get();
const auto * col_radius1 = arguments[1].column.get();
const auto * col_center2 = arguments[2].column.get();
const auto * col_radius2 = arguments[3].column.get();
auto col_res_center = ColumnUInt64::create();
auto col_res_radius = ColumnFloat64::create();
auto & vec_res_center = col_res_center->getData();
vec_res_center.reserve(input_rows_count);
auto & vec_res_radius = col_res_radius->getData();
vec_res_radius.reserve(input_rows_count);
for (const auto row : collections::range(0, input_rows_count))
{
const UInt64 first_center = col_center1->getUInt(row);
const Float64 first_radius = col_radius1->getFloat64(row);
const UInt64 second_center = col_center2->getUInt(row);
const Float64 second_radius = col_radius2->getFloat64(row);
if (isNaN(first_radius) || isNaN(second_radius))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan");
if (std::isinf(first_radius) || std::isinf(second_radius))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be infinite");
auto first_center_cell = S2CellId(first_center);
auto second_center_cell = S2CellId(second_center);
if (!first_center_cell.is_valid() || !second_center_cell.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Center of the cap is not valid");
S2Cap cap1(first_center_cell.ToPoint(), S1Angle::Degrees(first_radius));
S2Cap cap2(second_center_cell.ToPoint(), S1Angle::Degrees(second_radius));
S2Cap cap_union = cap1.Union(cap2);
vec_res_center.emplace_back(S2CellId(cap_union.center()).id());
vec_res_radius.emplace_back(cap_union.GetRadius().degrees());
}
return ColumnTuple::create(Columns{std::move(col_res_center), std::move(col_res_radius)});
}
};
}
void registerFunctionS2CapUnion(FunctionFactory & factory)
{
factory.registerFunction<FunctionS2CapUnion>();
}
}
#endif

View File

@ -0,0 +1,104 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <common/range.h>
#include "s2_fwd.h"
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
}
namespace
{
/**
* Each cell in s2 library is a quadrilateral bounded by four geodesics.
*/
class FunctionS2CellsIntersect : public IFunction
{
public:
static constexpr auto name = "s2CellsIntersect";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionS2CellsIntersect>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 2; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t i = 0; i < getNumberOfArguments(); ++i)
{
const auto * arg = arguments[i].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), i, getName());
}
return std::make_shared<DataTypeUInt8>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_id_first = arguments[0].column.get();
const auto * col_id_second = arguments[1].column.get();
auto dst = ColumnUInt8::create();
auto & dst_data = dst->getData();
dst_data.reserve(input_rows_count);
for (const auto row : collections::range(0, input_rows_count))
{
const UInt64 id_first = col_id_first->getInt(row);
const UInt64 id_second = col_id_second->getInt(row);
auto first_cell = S2CellId(id_first);
auto second_cell = S2CellId(id_second);
if (!first_cell.is_valid() || !second_cell.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cell is not valid");
dst_data.emplace_back(S2CellId(id_first).intersects(S2CellId(id_second)));
}
return dst;
}
};
}
void registerFunctionS2CellsIntersect(FunctionFactory & factory)
{
factory.registerFunction<FunctionS2CellsIntersect>();
}
}
#endif

View File

@ -0,0 +1,111 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnArray.h>
#include <Columns/ColumnsNumber.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeArray.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <common/range.h>
#include "s2_fwd.h"
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
}
namespace
{
/**
* Each cell in s2 library is a quadrilateral bounded by four geodesics.
* So, each cell has 4 neighbors
*/
class FunctionS2GetNeighbors : public IFunction
{
public:
static constexpr auto name = "s2GetNeighbors";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionS2GetNeighbors>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 1; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
const auto * arg = arguments[0].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be Float64",
arg->getName(), 1, getName());
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUInt64>());
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_id = arguments[0].column.get();
auto dst = ColumnArray::create(ColumnUInt64::create());
auto & dst_data = dst->getData();
auto & dst_offsets = dst->getOffsets();
dst_offsets.resize(input_rows_count);
size_t current_offset = 0;
for (const auto row : collections::range(0, input_rows_count))
{
const UInt64 id = col_id->getUInt(row);
S2CellId cell_id(id);
if (!cell_id.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cell is not valid");
S2CellId neighbors[4];
cell_id.GetEdgeNeighbors(neighbors);
dst_data.reserve(dst_data.size() + 4);
for (auto & neighbor : neighbors)
{
++current_offset;
dst_data.insert(neighbor.id());
}
dst_offsets[row] = current_offset;
}
return dst;
}
};
}
void registerFunctionS2GetNeighbors(FunctionFactory & factory)
{
factory.registerFunction<FunctionS2GetNeighbors>();
}
}
#endif

115
src/Functions/s2RectAdd.cpp Normal file
View File

@ -0,0 +1,115 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <common/range.h>
#include "s2_fwd.h"
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
}
namespace
{
class FunctionS2RectAdd : public IFunction
{
public:
static constexpr auto name = "s2RectAdd";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionS2RectAdd>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 4; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t index = 0; index < getNumberOfArguments(); ++index)
{
const auto * arg = arguments[index].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), index, getName());
}
DataTypePtr element = std::make_shared<DataTypeUInt64>();
return std::make_shared<DataTypeTuple>(DataTypes{element, element});
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_lo = arguments[0].column.get();
const auto * col_hi = arguments[1].column.get();
const auto * col_point = arguments[2].column.get();
auto col_res_first = ColumnUInt64::create();
auto col_res_second = ColumnUInt64::create();
auto & vec_res_first = col_res_first->getData();
vec_res_first.reserve(input_rows_count);
auto & vec_res_second = col_res_second->getData();
vec_res_second.reserve(input_rows_count);
for (const auto row : collections::range(0, input_rows_count))
{
const auto lo = S2CellId(col_lo->getUInt(row));
const auto hi = S2CellId(col_hi->getUInt(row));
const auto point = S2CellId(col_point->getUInt(row));
if (!lo.is_valid() || !hi.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid");
if (!point.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid");
S2LatLngRect rect(lo.ToLatLng(), hi.ToLatLng());
rect.AddPoint(point.ToPoint());
vec_res_first.emplace_back(S2CellId(rect.lo()).id());
vec_res_second.emplace_back(S2CellId(rect.hi()).id());
}
return ColumnTuple::create(Columns{std::move(col_res_first), std::move(col_res_second)});
}
};
}
void registerFunctionS2RectAdd(FunctionFactory & factory)
{
factory.registerFunction<FunctionS2RectAdd>();
}
}
#endif

View File

@ -0,0 +1,105 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <common/range.h>
#include "s2_fwd.h"
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
}
namespace
{
class FunctionS2RectContains : public IFunction
{
public:
static constexpr auto name = "s2RectContains";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionS2RectContains>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 4; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t i = 0; i < getNumberOfArguments(); ++i)
{
const auto * arg = arguments[i].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), i, getName());
}
return std::make_shared<DataTypeUInt8>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_lo = arguments[0].column.get();
const auto * col_hi = arguments[1].column.get();
const auto * col_point = arguments[2].column.get();
auto dst = ColumnVector<UInt8>::create();
auto & dst_data = dst->getData();
dst_data.reserve(input_rows_count);
for (const auto row : collections::range(0, input_rows_count))
{
const auto lo = S2CellId(col_lo->getUInt(row));
const auto hi = S2CellId(col_hi->getUInt(row));
const auto point = S2CellId(col_point->getUInt(row));
if (!lo.is_valid() || !hi.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid");
if (!point.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid");
S2LatLngRect rect(lo.ToLatLng(), hi.ToLatLng());
dst_data.emplace_back(rect.Contains(point.ToLatLng()));
}
return dst;
}
};
}
void registerFunctionS2RectContains(FunctionFactory & factory)
{
factory.registerFunction<FunctionS2RectContains>();
}
}
#endif

View File

@ -0,0 +1,121 @@
#if !defined(ARCADIA_BUILD)
# include "config_functions.h"
#endif
#if USE_S2_GEOMETRY
#include <Columns/ColumnsNumber.h>
#include <Columns/ColumnTuple.h>
#include <DataTypes/DataTypesNumber.h>
#include <DataTypes/DataTypeTuple.h>
#include <Functions/FunctionFactory.h>
#include <Common/typeid_cast.h>
#include <common/range.h>
#include "s2_fwd.h"
class S2CellId;
namespace DB
{
namespace ErrorCodes
{
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int BAD_ARGUMENTS;
}
namespace
{
class FunctionS2RectIntersection : public IFunction
{
public:
static constexpr auto name = "s2RectIntersection";
static FunctionPtr create(ContextPtr)
{
return std::make_shared<FunctionS2RectIntersection>();
}
std::string getName() const override
{
return name;
}
size_t getNumberOfArguments() const override { return 4; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
for (size_t i = 0; i < getNumberOfArguments(); ++i)
{
const auto * arg = arguments[i].get();
if (!WhichDataType(arg).isUInt64())
throw Exception(
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Illegal type {} of argument {} of function {}. Must be UInt64",
arg->getName(), i, getName());
}
DataTypePtr element = std::make_shared<DataTypeUInt64>();
return std::make_shared<DataTypeTuple>(DataTypes{element, element});
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const auto * col_lo1 = arguments[0].column.get();
const auto * col_hi1 = arguments[1].column.get();
const auto * col_lo2 = arguments[2].column.get();
const auto * col_hi2 = arguments[3].column.get();
auto col_res_first = ColumnUInt64::create();
auto col_res_second = ColumnUInt64::create();
auto & vec_res_first = col_res_first->getData();
vec_res_first.reserve(input_rows_count);
auto & vec_res_second = col_res_second->getData();
vec_res_second.reserve(input_rows_count);
for (const auto row : collections::range(0, input_rows_count))
{
const auto lo1 = S2CellId(col_lo1->getUInt(row));
const auto hi1 = S2CellId(col_hi1->getUInt(row));
const auto lo2 = S2CellId(col_lo2->getUInt(row));
const auto hi2 = S2CellId(col_hi2->getUInt(row));
if (!lo1.is_valid() || !hi1.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "First rectangle is not valid");
if (!lo2.is_valid() || !hi2.is_valid())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second rectangle is not valid");
S2LatLngRect rect1(lo1.ToLatLng(), hi1.ToLatLng());
S2LatLngRect rect2(lo2.ToLatLng(), hi2.ToLatLng());
S2LatLngRect rect_intersection = rect1.Intersection(rect2);
vec_res_first.emplace_back(S2CellId(rect_intersection.lo()).id());
vec_res_second.emplace_back(S2CellId(rect_intersection.hi()).id());
}
return ColumnTuple::create(Columns{std::move(col_res_first), std::move(col_res_second)});
}
};
}
void registerFunctionS2RectIntersection(FunctionFactory & factory)
{
factory.registerFunction<FunctionS2RectIntersection>();
}
}
#endif

Some files were not shown because too many files have changed in this diff Show More