mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-22 07:31:57 +00:00
Add tests, docs, implement new syntax sugar for Array(JSON), fix small ugs
This commit is contained in:
avogar
parent
094a1aa970
commit
519494a9d0
@ -5608,3 +5608,10 @@ Default value: `10000000`.
|
||||
Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.
|
||||
|
||||
Default value: `1GiB`.
|
||||
|
||||
## use_json_alias_for_old_object_type
|
||||
|
||||
When enabled, `JSON` data type alias will be used to create an old [Object('json')](../../sql-reference/data-types/ob) type instead of the new [JSON](../../sql-reference/data-types/json.md) type.
|
||||
This setting requires server restart to take effect when changed.
|
||||
|
||||
Default value: `false`.
|
||||
|
||||
@ -19,7 +19,7 @@ To declare a column of `JSON` type, use the following syntax:
|
||||
<column_name> JSON(max_dynamic_paths=N, max_dynamic_types=M, some.path TypeName, SKIP path.to.skip, SKIP REGEXP 'paths_regexp')
|
||||
```
|
||||
Where:
|
||||
- `max_dynamic_paths` is an optional parameter indicating how many paths can be stored separately as subcolumns across single block of data that is stored separately (for example across single data part for MergeTree table). If this limit is exceeded, all other paths will be stored together in a single structure. Default value of `max_dynamic_paths` is `1000`.
|
||||
- `max_dynamic_paths` is an optional parameter indicating how many paths can be stored separately as subcolumns across single block of data that is stored separately (for example across single data part for MergeTree table). If this limit is exceeded, all other paths will be stored together in a single structure. Default value of `max_dynamic_paths` is `1024`.
|
||||
- `max_dynamic_types` is an optional parameter between `1` and `255` indicating how many different data types can be stored inside a single path column with type `Dynamic` across single block of data that is stored separately (for example across single data part for MergeTree table). If this limit is exceeded, all new types will be converted to type `String`. Default value of `max_dynamic_types` is `32`.
|
||||
- `some.path TypeName` is an optional type hint for particular path in the JSON. Such paths will be always stored as subcolumns with specified type.
|
||||
- `SKIP path.to.skip` is an optional hint for particular path that should be skipped during JSON parsing. Such paths will never be stored in the JSON column. If specified path is a nested JSON object, the whole nested object will be skipped.
|
||||
@ -267,18 +267,18 @@ JSON paths that contains an array of objects are parsed as type `Array(JSON)` an
|
||||
```sql
|
||||
CREATE TABLE test (json JSON) ENGINE = Memory;
|
||||
INSERT INTO test VALUES
|
||||
('{"a" : {"b" : [{"c" : 42, "d" : "Hello", "f" : {"g" : 42.42}}, {"c" : 43}, {"e" : [1, 2, 3], "d" : "My", "f" : {"g" : 43.43, "h" : "2020-01-01"}}]}}'),
|
||||
('{"a" : {"b" : [{"c" : 42, "d" : "Hello", "f" : [[{"g" : 42.42}]], "k" : {"j" : 1000}}, {"c" : 43}, {"e" : [1, 2, 3], "d" : "My", "f" : [[{"g" : 43.43, "h" : "2020-01-01"}]], "k" : {"j" : 2000}}]}}'),
|
||||
('{"a" : {"b" : [1, 2, 3]}}'),
|
||||
('{"a" : {"b" : [{"c" : 44, "f" : {"h" : "2020-01-02"}}, {"e" : [4, 5, 6], "d" : "World", "f" : {"g" : 44.44}}]}}');
|
||||
('{"a" : {"b" : [{"c" : 44, "f" : [[{"h" : "2020-01-02"}]]}, {"e" : [4, 5, 6], "d" : "World", "f" : [[{"g" : 44.44}]], "k" : {"j" : 3000}}]}}');
|
||||
SELECT json FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ {"a":{"b":[{"c":"42","d":"Hello","f":{"g":42.42}},{"c":"43"},{"d":"My","e":["1","2","3"],"f":{"g":43.43,"h":"2020-01-01"}}]}} │
|
||||
│ {"a":{"b":["1","2","3"]}} │
|
||||
│ {"a":{"b":[{"c":"44","f":{"h":"2020-01-02"}},{"d":"World","e":["4","5","6"],"f":{"g":44.44}}]}} │
|
||||
└───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```text3
|
||||
┌─json────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐
|
||||
│ {"a":{"b":[{"c":"42","d":"Hello","f":[[{"g":42.42}]],"k":{"j":"1000"}},{"c":"43"},{"d":"My","e":["1","2","3"],"f":[[{"g":43.43,"h":"2020-01-01"}]],"k":{"j":"2000"}}]}} │
|
||||
│ {"a":{"b":["1","2","3"]}} │
|
||||
│ {"a":{"b":[{"c":"44","f":[[{"h":"2020-01-02"}]]},{"d":"World","e":["4","5","6"],"f":[[{"g":44.44}]],"k":{"j":"3000"}}]}} │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
```sql
|
||||
@ -286,55 +286,87 @@ SELECT json.a.b, dynamicType(json.a.b) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json.a.b────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─dynamicType(json.a.b)───────────────────────────────────┐
|
||||
│ ['{"c":"42","d":"Hello","f":{"g":42.42}}','{"c":"43"}','{"d":"My","e":["1","2","3"],"f":{"g":43.43,"h":"2020-01-01"}}'] │ Array(JSON(max_dynamic_types=8, max_dynamic_paths=125)) │
|
||||
│ [1,2,3] │ Array(Nullable(Int64)) │
|
||||
│ ['{"c":"44","f":{"h":"2020-01-02"}}','{"d":"World","e":["4","5","6"],"f":{"g":44.44}}'] │ Array(JSON(max_dynamic_types=8, max_dynamic_paths=125)) │
|
||||
└─────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴─────────────────────────────────────────────────────────┘
|
||||
┌─json.a.b──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┬─dynamicType(json.a.b)────────────────────────────────────┐
|
||||
│ ['{"c":"42","d":"Hello","f":[[{"g":42.42}]],"k":{"j":"1000"}}','{"c":"43"}','{"d":"My","e":["1","2","3"],"f":[[{"g":43.43,"h":"2020-01-01"}]],"k":{"j":"2000"}}'] │ Array(JSON(max_dynamic_types=16, max_dynamic_paths=256)) │
|
||||
│ [1,2,3] │ Array(Nullable(Int64)) │
|
||||
│ ['{"c":"44","f":[[{"h":"2020-01-02"}]]}','{"d":"World","e":["4","5","6"],"f":[[{"g":44.44}]],"k":{"j":"3000"}}'] │ Array(JSON(max_dynamic_types=16, max_dynamic_paths=256)) │
|
||||
└───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┴──────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
As you can notice, the `max_dynamic_types/max_dynamic_paths` parameters of the nested `JSON` type were reduced compared to the default values. It's needed to avoid number of subcolumns to grow uncontrolled on nested arrays of JSON objects.
|
||||
|
||||
Let's try to read subcolumns of this nested `JSON` column:
|
||||
Let's try to read subcolumns from this nested `JSON` column:
|
||||
|
||||
```sql
|
||||
SELECT json.a.b.:`Array(JSON)`.c, json.a.b.:`Array(JSON)`.f.g, json.a.b.:`Array(JSON)`.f.g FROM test;
|
||||
SELECT json.a.b.:`Array(JSON)`.c, json.a.b.:`Array(JSON)`.f, json.a.b.:`Array(JSON)`.d FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json.a.b.:`Array(JSON)`.c─┬─json.a.b.:`Array(JSON)`.f.g─┬─json.a.b.:`Array(JSON)`.f.g─┐
|
||||
│ [42,43,NULL] │ [42.42,NULL,43.43] │ [42.42,NULL,43.43] │
|
||||
│ [] │ [] │ [] │
|
||||
│ [44,NULL] │ [NULL,44.44] │ [NULL,44.44] │
|
||||
└───────────────────────────┴─────────────────────────────┴─────────────────────────────┘
|
||||
┌─json.a.b.:`Array(JSON)`.c─┬─json.a.b.:`Array(JSON)`.f───────────────────────────────────┬─json.a.b.:`Array(JSON)`.d─┐
|
||||
│ [42,43,NULL] │ [[['{"g":42.42}']],NULL,[['{"g":43.43,"h":"2020-01-01"}']]] │ ['Hello',NULL,'My'] │
|
||||
│ [] │ [] │ [] │
|
||||
│ [44,NULL] │ [[['{"h":"2020-01-02"}']],[['{"g":44.44}']]] │ [NULL,'World'] │
|
||||
└───────────────────────────┴─────────────────────────────────────────────────────────────┴───────────────────────────┘
|
||||
```
|
||||
|
||||
We can also read subcolumns of `Dynamic` columns:
|
||||
We can avoid writing `Array(JSON)` subcolumn name using special syntax:
|
||||
|
||||
```sql
|
||||
SELECT json.a.b.:`Array(JSON)`.f.h.:Date FROM test;
|
||||
SELECT json.a.b[].c, json.a.b[].f, json.a.b[].d FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json.a.b.:`Array(JSON)`.f.h.:`Date`─┐
|
||||
│ [NULL,NULL,'2020-01-01'] │
|
||||
│ [] │
|
||||
│ ['2020-01-02',NULL] │
|
||||
└─────────────────────────────────────┘
|
||||
┌─json.a.b.:`Array(JSON)`.c─┬─json.a.b.:`Array(JSON)`.f───────────────────────────────────┬─json.a.b.:`Array(JSON)`.d─┐
|
||||
│ [42,43,NULL] │ [[['{"g":42.42}']],NULL,[['{"g":43.43,"h":"2020-01-01"}']]] │ ['Hello',NULL,'My'] │
|
||||
│ [] │ [] │ [] │
|
||||
│ [44,NULL] │ [[['{"h":"2020-01-02"}']],[['{"g":44.44}']]] │ [NULL,'World'] │
|
||||
└───────────────────────────┴─────────────────────────────────────────────────────────────┴───────────────────────────┘
|
||||
```
|
||||
|
||||
The number of `[]` after path indicates the array level. `json.path[][]` will be transformed to `json.path.:Array(Array(JSON))`
|
||||
|
||||
Let's check the paths and types inside our `Array(JSON)`:
|
||||
|
||||
```sql
|
||||
SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(json.a.b[]))) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─arrayJoin(JSONAllPathsWithTypes(arrayJoin(json.a.b.:`Array(JSON)`)))──┐
|
||||
│ ('c','Int64') │
|
||||
│ ('d','String') │
|
||||
│ ('f','Array(Array(JSON(max_dynamic_types=8, max_dynamic_paths=64)))') │
|
||||
│ ('k.j','Int64') │
|
||||
│ ('e','Array(Nullable(Int64))') │
|
||||
└───────────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
Let's read subcolumns from `Array(JSON)` column:
|
||||
|
||||
```sql
|
||||
SELECT json.a.b[].c.:Int64, json.a.b[].f[][].g.:Float64, json.a.b[].f[][].h.:Date FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json.a.b.:`Array(JSON)`.c.:`Int64`─┬─json.a.b.:`Array(JSON)`.f.:`Array(Array(JSON))`.g.:`Float64`─┬─json.a.b.:`Array(JSON)`.f.:`Array(Array(JSON))`.h.:`Date`─┐
|
||||
│ [42,43,NULL] │ [[[42.42]],[],[[43.43]]] │ [[[NULL]],[],[['2020-01-01']]] │
|
||||
│ [] │ [] │ [] │
|
||||
│ [44,NULL] │ [[[NULL]],[[44.44]]] │ [[['2020-01-02']],[[NULL]]] │
|
||||
└────────────────────────────────────┴──────────────────────────────────────────────────────────────┴───────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
We can also read sub-object subcolumns from nested `JSON` column:
|
||||
|
||||
```sql
|
||||
SELECT json.a.b.:`Array(JSON)`.^f FROM test
|
||||
SELECT json.a.b[].^k FROM test
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json.a.b.:`Array(JSON)`.^`f`────────────────────────┐
|
||||
│ ['{"g":42.42}','{}','{"g":43.43,"h":"2020-01-01"}'] │
|
||||
│ [] │
|
||||
│ ['{"h":"2020-01-02"}','{"g":44.44}'] │
|
||||
└─────────────────────────────────────────────────────┘
|
||||
┌─json.a.b.:`Array(JSON)`.^`k`─────────┐
|
||||
│ ['{"j":"1000"}','{}','{"j":"2000"}'] │
|
||||
│ [] │
|
||||
│ ['{}','{"j":"3000"}'] │
|
||||
└──────────────────────────────────────┘
|
||||
```
|
||||
|
||||
## Reading JSON type from the data
|
||||
@ -386,7 +418,7 @@ SELECT json FROM format(TSV, 'json JSON(a.b.c UInt32, SKIP a.b.d, SKIP REGEXP \'
|
||||
|
||||
## Reaching the limit of dynamic paths inside JSON
|
||||
|
||||
`JSON` data type can store only limited number of paths as separate subcolumns inside. By default, this limit is 1000, but you can change it in type declaration using parameter `max_dynamic_paths`.
|
||||
`JSON` data type can store only limited number of paths as separate subcolumns inside. By default, this limit is 1024, but you can change it in type declaration using parameter `max_dynamic_paths`.
|
||||
When the limit is reached, all new paths inserted to `JSON` column will be stored in a single shared data structure. It's still possible to read such paths as subcolumns, but it will require reading the whole
|
||||
shared data structure to extract the values of this path. This limit is needed to avoid the enormous number of different subcolumns that can make the table unusable.
|
||||
|
||||
@ -470,7 +502,9 @@ SELECT count(), dynamicType(d), _part FROM test GROUP BY _part, dynamicType(d) O
|
||||
|
||||
As we can see, ClickHouse kept the most frequent paths `a`, `b` and `c` and moved paths `e` and `d` to shared data structure.
|
||||
|
||||
## Introspection functions
|
||||
|
||||
There are several functions that can help to inspect the content of the JSON column: [JSONAllPaths](../functions/json-functions.md#jsonallpaths), [JSONAllPathsWithTypes](../functions/json-functions.md#jsonallpathswithtypes), [JSONDynamicPaths](../functions/json-functions.md#jsondynamicpaths), [JSONDynamicPathsWithTypes](../functions/json-functions.md#jsondynamicpathswithtypes), [JSONSharedDataPaths](../functions/json-functions.md#jsonshareddatapaths), [JSONSharedDataPathsWithTypes](../functions/json-functions.md#jsonshareddatapathswithtypes).
|
||||
|
||||
## Tips for better usage of the JSON type
|
||||
|
||||
|
||||
@ -13,7 +13,7 @@ keywords: [object, data type]
|
||||
|
||||
Stores JavaScript Object Notation (JSON) documents in a single column.
|
||||
|
||||
`JSON` is an alias for `Object('json')`.
|
||||
`JSON` can be used as an alias to `Object('json')` when setting [use_json_alias_for_old_object_type](../../operations/settings/settings.md#usejsonaliasforoldobjecttype) is enabled.
|
||||
|
||||
## Example
|
||||
|
||||
|
||||
@ -1,86 +0,0 @@
|
||||
---
|
||||
slug: /en/sql-reference/data-types/json
|
||||
sidebar_position: 26
|
||||
sidebar_label: JSON
|
||||
---
|
||||
|
||||
# JSON
|
||||
|
||||
:::note
|
||||
This feature is experimental and is not production-ready. If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json.md) instead.
|
||||
:::
|
||||
|
||||
Stores JavaScript Object Notation (JSON) documents in a single column.
|
||||
|
||||
`JSON` is an alias for `Object('json')`.
|
||||
|
||||
:::note
|
||||
The JSON data type is an obsolete feature. Do not use it.
|
||||
If you want to use it, set `allow_experimental_object_type = 1`.
|
||||
:::
|
||||
|
||||
## Example
|
||||
|
||||
**Example 1**
|
||||
|
||||
Creating a table with a `JSON` column and inserting data into it:
|
||||
|
||||
```sql
|
||||
CREATE TABLE json
|
||||
(
|
||||
o JSON
|
||||
)
|
||||
ENGINE = Memory
|
||||
```
|
||||
|
||||
```sql
|
||||
INSERT INTO json VALUES ('{"a": 1, "b": { "c": 2, "d": [1, 2, 3] }}')
|
||||
```
|
||||
|
||||
```sql
|
||||
SELECT o.a, o.b.c, o.b.d[3] FROM json
|
||||
```
|
||||
|
||||
```text
|
||||
┌─o.a─┬─o.b.c─┬─arrayElement(o.b.d, 3)─┐
|
||||
│ 1 │ 2 │ 3 │
|
||||
└─────┴───────┴────────────────────────┘
|
||||
```
|
||||
|
||||
**Example 2**
|
||||
|
||||
To be able to create an ordered `MergeTree` family table the sorting key has to be extracted into its column. For example, to insert a file of compressed HTTP access logs in JSON format:
|
||||
|
||||
```sql
|
||||
CREATE TABLE logs
|
||||
(
|
||||
timestamp DateTime,
|
||||
message JSON
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY timestamp
|
||||
```
|
||||
|
||||
```sql
|
||||
INSERT INTO logs
|
||||
SELECT parseDateTimeBestEffort(JSONExtractString(json, 'timestamp')), json
|
||||
FROM file('access.json.gz', JSONAsString)
|
||||
```
|
||||
|
||||
## Displaying JSON columns
|
||||
|
||||
When displaying a `JSON` column ClickHouse only shows the field values by default (because internally, it is represented as a tuple). You can display the field names as well by setting `output_format_json_named_tuples_as_objects = 1`:
|
||||
|
||||
```sql
|
||||
SET output_format_json_named_tuples_as_objects = 1
|
||||
|
||||
SELECT * FROM json FORMAT JSONEachRow
|
||||
```
|
||||
|
||||
```text
|
||||
{"o":{"a":1,"b":{"c":2,"d":[1,2,3]}}}
|
||||
```
|
||||
|
||||
## Related Content
|
||||
|
||||
- [Getting Data Into ClickHouse - Part 2 - A JSON detour](https://clickhouse.com/blog/getting-data-into-clickhouse-part-2-json)
|
||||
@ -1155,3 +1155,239 @@ SELECT jsonMergePatch('{"a":1}', '{"name": "joey"}', '{"name": "tom"}', '{"name"
|
||||
│ {"a":1,"name":"zoey"} │
|
||||
└───────────────────────┘
|
||||
```
|
||||
|
||||
### JSONAllPaths
|
||||
|
||||
Returns the list of all paths stored in each row in [JSON](../data-types/json.md) column.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
JSONAllPaths(json)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `json` — [JSON](../data-types/json.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- An array of paths. [Array(String)](../data-types/array.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONAllPaths(json) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json─────────────────────────────────┬─JSONAllPaths(json)─┐
|
||||
│ {"a":"42"} │ ['a'] │
|
||||
│ {"b":"Hello"} │ ['b'] │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ ['a','c'] │
|
||||
└──────────────────────────────────────┴────────────────────┘
|
||||
```
|
||||
|
||||
### JSONAllPathsWithTypes
|
||||
|
||||
Returns the map of all paths and their data types stored in each row in [JSON](../data-types/json.md) column.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
JSONAllPathsWithTypes(json)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `json` — [JSON](../data-types/json.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- An array of paths. [Map(String, String)](../data-types/array.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONAllPathsWithTypes(json) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json─────────────────────────────────┬─JSONAllPathsWithTypes(json)───────────────┐
|
||||
│ {"a":"42"} │ {'a':'Int64'} │
|
||||
│ {"b":"Hello"} │ {'b':'String'} │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ {'a':'Array(Nullable(Int64))','c':'Date'} │
|
||||
└──────────────────────────────────────┴───────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### JSONDynamicPaths
|
||||
|
||||
Returns the list of dynamic paths that are stored as separate subcolumns in [JSON](../data-types/json.md) column.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
JSONDynamicPaths(json)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `json` — [JSON](../data-types/json.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- An array of paths. [Array(String)](../data-types/array.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONDynamicPaths(json) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json─────────────────────────────────┬─JSONDynamicPaths(json)─┐
|
||||
| {"a":"42"} │ ['a'] │
|
||||
│ {"b":"Hello"} │ [] │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ ['a'] │
|
||||
└──────────────────────────────────────┴────────────────────────┘
|
||||
```
|
||||
|
||||
### JSONDynamicPathsWithTypes
|
||||
|
||||
Returns the map of dynamic paths that are stored as separate subcolumns and their types in each row in [JSON](../data-types/json.md) column.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
JSONAllPathsWithTypes(json)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `json` — [JSON](../data-types/json.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- An array of paths. [Map(String, String)](../data-types/array.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONDynamicPathsWithTypes(json) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json─────────────────────────────────┬─JSONDynamicPathsWithTypes(json)─┐
|
||||
│ {"a":"42"} │ {'a':'Int64'} │
|
||||
│ {"b":"Hello"} │ {} │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ {'a':'Array(Nullable(Int64))'} │
|
||||
└──────────────────────────────────────┴─────────────────────────────────┘
|
||||
```
|
||||
|
||||
### JSONSharedDataPaths
|
||||
|
||||
Returns the list of paths that are stored in shared data structure in [JSON](../data-types/json.md) column.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
JSONSharedDataPaths(json)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `json` — [JSON](../data-types/json.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- An array of paths. [Array(String)](../data-types/array.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONSharedDataPaths(json) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json─────────────────────────────────┬─JSONSharedDataPaths(json)─┐
|
||||
│ {"a":"42"} │ [] │
|
||||
│ {"b":"Hello"} │ ['b'] │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ ['c'] │
|
||||
└──────────────────────────────────────┴───────────────────────────┘
|
||||
```
|
||||
|
||||
### JSONSharedDataPathsWithTypes
|
||||
|
||||
Returns the map of paths that are stored in shared data structure and their types in each row in [JSON](../data-types/json.md) column.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
JSONSharedDataPathsWithTypes(json)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `json` — [JSON](../data-types/json.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- An array of paths. [Map(String, String)](../data-types/array.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONSharedDataPathsWithTypes(json) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json─────────────────────────────────┬─JSONSharedDataPathsWithTypes(json)─┐
|
||||
│ {"a":"42"} │ {} │
|
||||
│ {"b":"Hello"} │ {'b':'String'} │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ {'c':'Date'} │
|
||||
└──────────────────────────────────────┴────────────────────────────────────┘
|
||||
```
|
||||
|
||||
### JSONEmpty
|
||||
|
||||
Checks whether the input [JSON](../data-types/json.md) object is empty.
|
||||
|
||||
``` sql
|
||||
JSONEmpty(json)
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `json` — [JSON](../data-types/json.md).
|
||||
|
||||
**Returned value**
|
||||
|
||||
- Returns `1` for an empty JSON object or `0` for a non-empty JSON object. [UInt8](../data-types/int-uint.md).
|
||||
|
||||
**Example**
|
||||
|
||||
``` sql
|
||||
CREATE TABLE test (json JSON) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {}}, {"json" : {"a" : [1, 2, 3], "b" : "2020-01-01"}}, {"json" : {}},
|
||||
SELECT json, JSONEmpty(json) FROM test;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─json─────────────────────────────────┬─JSONEmpty(json)─┐
|
||||
│ {} │ 1 │
|
||||
│ {"a":["1","2","3"],"b":"2020-01-01"} │ 0 │
|
||||
│ {} │ 1 │
|
||||
└──────────────────────────────────────┴─────────────────┘
|
||||
```
|
||||
|
||||
@ -424,7 +424,7 @@ void ColumnObject::insertRangeFrom(const IColumn & src, size_t start, size_t len
|
||||
void ColumnObject::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
|
||||
#endif
|
||||
{
|
||||
/// TODO: try to parallelize doInsertRangeFrom over typed/dynamic paths.
|
||||
/// TODO: try to parallelize doInsertRangeFrom over typed/dynamic paths if it makes sense.
|
||||
const auto & src_object_column = assert_cast<const ColumnObject &>(src);
|
||||
|
||||
/// First, insert typed paths, they must be the same for both columns.
|
||||
|
||||
@ -101,14 +101,15 @@ public:
|
||||
ColumnPtr replicate(const Offsets & replicate_offsets) const override;
|
||||
MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
|
||||
|
||||
void getPermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation &) const override;
|
||||
void updatePermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation &, EqualRanges &) const override {}
|
||||
|
||||
/// Values of ColumnObject are not comparable.
|
||||
#if !defined(ABORT_ON_LOGICAL_ERROR)
|
||||
int compareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
|
||||
#else
|
||||
int doCompareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
|
||||
#endif
|
||||
void getPermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation &) const override;
|
||||
void updatePermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation &, EqualRanges &) const override {}
|
||||
void getExtremes(Field & min, Field & max) const override
|
||||
{
|
||||
min = Object();
|
||||
|
||||
@ -14,9 +14,7 @@
|
||||
#include <Parsers/ASTNameTypePair.h>
|
||||
#include <Formats/JSONExtractTree.h>
|
||||
#include <Interpreters/Context.h>
|
||||
|
||||
#include <Core/Settings.h>
|
||||
|
||||
#if USE_SIMDJSON
|
||||
#include <Common/JSONParsers/SimdJSONParser.h>
|
||||
#endif
|
||||
@ -480,7 +478,7 @@ static DataTypePtr createJSON(const ASTPtr & arguments)
|
||||
void registerDataTypeJSON(DataTypeFactory & factory)
|
||||
{
|
||||
if (!Context::getGlobalContextInstance()->getSettingsRef().use_json_alias_for_old_object_type)
|
||||
factory.registerDataType("JSON", createJSON);
|
||||
factory.registerDataType("JSON", createJSON, DataTypeFactory::CaseInsensitive);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -24,7 +24,7 @@ public:
|
||||
};
|
||||
|
||||
/// Don't change these constants, it can break backward compatibility.
|
||||
static constexpr size_t DEFAULT_MAX_SEPARATELY_STORED_PATHS = 1000;
|
||||
static constexpr size_t DEFAULT_MAX_SEPARATELY_STORED_PATHS = 1024;
|
||||
static constexpr size_t NESTED_OBJECT_MAX_DYNAMIC_PATHS_REDUCE_FACTOR = 4;
|
||||
static constexpr size_t NESTED_OBJECT_MAX_DYNAMIC_TYPES_REDUCE_FACTOR = 2;
|
||||
|
||||
@ -44,10 +44,7 @@ public:
|
||||
|
||||
MutableColumnPtr createColumn() const override;
|
||||
|
||||
Field getDefault() const override
|
||||
{
|
||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getDefault() is not implemented for data type {}", getName());
|
||||
}
|
||||
Field getDefault() const override { return Object(); }
|
||||
|
||||
bool isParametric() const override { return true; }
|
||||
bool canBeInsideNullable() const override { return false; }
|
||||
|
||||
@ -423,6 +423,9 @@ void SerializationObject::deserializeBinaryBulkWithMultipleStreams(
|
||||
DeserializeBinaryBulkStatePtr & state,
|
||||
SubstreamsCache * cache) const
|
||||
{
|
||||
if (!state)
|
||||
return;
|
||||
|
||||
auto * object_state = checkAndGetState<DeserializeBinaryBulkStateObject>(state);
|
||||
auto * structure_state = checkAndGetState<DeserializeBinaryBulkStateObjectStructure>(object_state->structure_state);
|
||||
auto mutable_column = column->assumeMutable();
|
||||
|
||||
@ -147,6 +147,18 @@ void SerializationObjectDynamicPath::deserializeBinaryBulkWithMultipleStreams(
|
||||
settings.path.pop_back();
|
||||
}
|
||||
/// Otherwise, read the whole shared data column and extract requested path from it.
|
||||
/// TODO: We can read several subcolumns of the same path located in the shared data
|
||||
/// and right now we extract the whole path column from shared data every time
|
||||
/// and then extract the requested subcolumns. We can optimize it and use substreams
|
||||
/// cache here to avoid extracting the same path from shared data several times.
|
||||
///
|
||||
/// TODO: We can change the serialization of shared data to optimize reading paths from it.
|
||||
/// Right now we cannot know if shared data contains our path in current range or not,
|
||||
/// but we can change the serialization and write the list of all paths stored in shared
|
||||
/// data before each granule, and then replace the column that stores paths with column
|
||||
/// with indexes in this list. It can also reduce the storage, because we will store
|
||||
/// each path only once and can replace UInt64 string offset column with indexes column
|
||||
/// that can have smaller type depending on the number of paths in the list.
|
||||
else
|
||||
{
|
||||
settings.path.push_back(Substream::ObjectSharedData);
|
||||
|
||||
@ -11,6 +11,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int NOT_IMPLEMENTED;
|
||||
extern const int LOGICAL_ERROR;
|
||||
}
|
||||
|
||||
struct SerializationVariantElement::DeserializeBinaryBulkStateVariantElement : public ISerialization::DeserializeBinaryBulkState
|
||||
@ -188,27 +189,21 @@ void SerializationVariantElement::deserializeBinaryBulkWithMultipleStreams(
|
||||
assert_cast<ColumnLowCardinality &>(*variant_element_state->variant->assumeMutable()).nestedRemoveNullable();
|
||||
}
|
||||
|
||||
/// If nothing to deserialize, just insert defaults.
|
||||
if (variant_limit == 0)
|
||||
{
|
||||
mutable_column->insertManyDefaults(num_new_discriminators);
|
||||
return;
|
||||
}
|
||||
|
||||
addVariantToPath(settings.path);
|
||||
nested_serialization->deserializeBinaryBulkWithMultipleStreams(variant_element_state->variant, *variant_limit, settings, variant_element_state->variant_element_state, cache);
|
||||
removeVariantFromPath(settings.path);
|
||||
|
||||
/// If nothing was deserialized when variant_limit > 0
|
||||
/// it means that we don't have a stream for such sub-column.
|
||||
/// It may happen during ALTER MODIFY column with Variant extension.
|
||||
/// In this case we should just insert default values.
|
||||
if (variant_element_state->variant->empty())
|
||||
/// If there was nothing to deserialize or nothing was actually deserialized when variant_limit > 0, just insert defaults.
|
||||
/// The second case means that we don't have a stream for such sub-column. It may happen during ALTER MODIFY column with Variant extension.
|
||||
if (variant_limit == 0 || variant_element_state->variant->empty())
|
||||
{
|
||||
mutable_column->insertManyDefaults(num_new_discriminators);
|
||||
return;
|
||||
}
|
||||
|
||||
if (variant_element_state->variant->size() < *variant_limit)
|
||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of deserialized variant column less than the limit: {} < {}", variant_element_state->variant->size(), *variant_limit);
|
||||
|
||||
size_t variant_offset = variant_element_state->variant->size() - *variant_limit;
|
||||
|
||||
/// If we have only our discriminator in range, insert the whole range to result column.
|
||||
|
||||
@ -1510,7 +1510,7 @@ private:
|
||||
}
|
||||
case ElementType::OBJECT:
|
||||
{
|
||||
return getObjectType();
|
||||
return std::make_shared<DataTypeObject>(DataTypeObject::SchemaFormat::JSON, max_dynamic_paths_for_object, max_dynamic_types_for_object);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1548,12 +1548,6 @@ private:
|
||||
return float64_type;
|
||||
}
|
||||
|
||||
const DataTypePtr & getObjectType() const
|
||||
{
|
||||
static const DataTypePtr object_type = std::make_shared<DataTypeObject>(DataTypeObject::SchemaFormat::JSON, max_dynamic_paths_for_object, max_dynamic_types_for_object);
|
||||
return object_type;
|
||||
}
|
||||
|
||||
const DataTypePtr & getNullType() const
|
||||
{
|
||||
static const DataTypePtr null_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeNothing>());
|
||||
|
||||
@ -99,7 +99,7 @@ REGISTER_FUNCTION(JSONEmpty)
|
||||
{
|
||||
factory.registerFunction<FunctionJSONEmpty>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Checks whether thee input JSON object is empty.
|
||||
Checks whether the input JSON object is empty.
|
||||
)",
|
||||
.syntax = {"JSONEmpty(json)"},
|
||||
.arguments = {{"json", "JSON column"}},
|
||||
|
||||
@ -76,6 +76,7 @@ struct JSONSharedDataPathsWithTypesImpl
|
||||
};
|
||||
|
||||
/// Implements functions that extracts paths and types from JSON object column.
|
||||
/// Used for introspection of the content of the JSON object column.
|
||||
template <typename Impl>
|
||||
class FunctionJSONPaths : public IFunction
|
||||
{
|
||||
@ -95,9 +96,12 @@ public:
|
||||
|
||||
DataTypePtr getReturnTypeImpl(const DataTypes & data_types) const override
|
||||
{
|
||||
if (data_types.size() != 1 || data_types[0]->getTypeId() != TypeIndex::Object)
|
||||
if (data_types.size() != 1 )
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires single argument with type JSON", getName());
|
||||
|
||||
if (data_types[0]->getTypeId() != TypeIndex::Object)
|
||||
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} requires argument with type JSON, got: {}", getName(),data_types[0]->getName());
|
||||
|
||||
if constexpr (Impl::with_types)
|
||||
return std::make_shared<DataTypeMap>(std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>());
|
||||
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
|
||||
@ -142,14 +146,23 @@ private:
|
||||
/// We want the resulting arrays of paths to be sorted for consistency.
|
||||
std::sort(dynamic_paths.begin(), dynamic_paths.end());
|
||||
|
||||
for (const auto & path : dynamic_paths)
|
||||
data.insertData(path.data(), path.size());
|
||||
offsets.push_back(data.size());
|
||||
return res->replicate(IColumn::Offsets(1, column_object.size()));
|
||||
size_t size = column_object.size();
|
||||
for (size_t i = 0; i != size; ++i)
|
||||
{
|
||||
for (const auto & path : dynamic_paths)
|
||||
{
|
||||
/// Don't include path if it contains NULL, because we consider
|
||||
/// it to be equivalent to the absense of this path in this row.
|
||||
if (!dynamic_path_columns.at(path)->isNullAt(i))
|
||||
data.insertData(path.data(), path.size());
|
||||
}
|
||||
offsets.push_back(data.size());
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
/// Collect all paths: typed, dynamic and paths from shared data.
|
||||
std::vector<StringRef> sorted_dynamic_and_typed_paths;
|
||||
std::vector<String> sorted_dynamic_and_typed_paths;
|
||||
const auto & typed_path_columns = column_object.getTypedPaths();
|
||||
const auto & dynamic_path_columns = column_object.getDynamicPaths();
|
||||
for (const auto & [path, _] : typed_path_columns)
|
||||
@ -174,7 +187,9 @@ private:
|
||||
while (sorted_paths_index != sorted_dynamic_and_typed_paths.size() && sorted_dynamic_and_typed_paths[sorted_paths_index] < shared_data_path)
|
||||
{
|
||||
auto path = sorted_dynamic_and_typed_paths[sorted_paths_index];
|
||||
data.insertData(path.data, path.size);
|
||||
/// If it's dynamic path include it only if it's not NULL.
|
||||
if (auto it = dynamic_path_columns.find(path); it == dynamic_path_columns.end() || !it->second->isNullAt(i))
|
||||
data.insertData(path.data(), path.size());
|
||||
++sorted_paths_index;
|
||||
}
|
||||
|
||||
@ -184,7 +199,8 @@ private:
|
||||
for (; sorted_paths_index != sorted_dynamic_and_typed_paths.size(); ++sorted_paths_index)
|
||||
{
|
||||
auto path = sorted_dynamic_and_typed_paths[sorted_paths_index];
|
||||
data.insertData(path.data, path.size);
|
||||
if (auto it = dynamic_path_columns.find(path); it == dynamic_path_columns.end() || !it->second->isNullAt(i))
|
||||
data.insertData(path.data(), path.size());
|
||||
}
|
||||
|
||||
offsets.push_back(data.size());
|
||||
@ -215,12 +231,16 @@ private:
|
||||
{
|
||||
for (auto & path : sorted_dynamic_paths)
|
||||
{
|
||||
auto type = getDynamicValueType(dynamic_path_columns.at(path), i);
|
||||
paths_column->insertData(path.data(), path.size());
|
||||
types_column->insertData(type.data(), type.size());
|
||||
auto column = dynamic_path_columns.at(path);
|
||||
if (!column->isNullAt(i))
|
||||
{
|
||||
auto type = getDynamicValueType(column, i);
|
||||
paths_column->insertData(path.data(), path.size());
|
||||
types_column->insertData(type.data(), type.size());
|
||||
}
|
||||
}
|
||||
|
||||
offsets.push_back(types_column->size());
|
||||
offsets.push_back(paths_column->size());
|
||||
}
|
||||
|
||||
return ColumnMap::create(ColumnPtr(std::move(paths_column)), ColumnPtr(std::move(types_column)), ColumnPtr(std::move(offsets_column)));
|
||||
@ -237,9 +257,11 @@ private:
|
||||
size_t end = shared_data_offsets[ssize_t(i)];
|
||||
for (size_t j = start; j != end; ++j)
|
||||
{
|
||||
paths_column->insertFrom(*shared_data_paths, j);
|
||||
auto type_name = getDynamicValueTypeFromSharedData(shared_data_values->getDataAt(j));
|
||||
types_column->insertData(type_name.data(), type_name.size());
|
||||
if (auto type_name = getDynamicValueTypeFromSharedData(shared_data_values->getDataAt(j)))
|
||||
{
|
||||
paths_column->insertFrom(*shared_data_paths, j);
|
||||
types_column->insertData(type_name->data(), type_name->size());
|
||||
}
|
||||
}
|
||||
|
||||
offsets.push_back(paths_column->size());
|
||||
@ -273,28 +295,44 @@ private:
|
||||
{
|
||||
auto shared_data_path = shared_data_paths->getDataAt(j);
|
||||
auto type_name = getDynamicValueTypeFromSharedData(shared_data_values->getDataAt(j));
|
||||
/// Skip NULL values.
|
||||
if (!type_name)
|
||||
continue;
|
||||
|
||||
while (sorted_paths_index != sorted_typed_and_dynamic_paths_with_types.size() && sorted_typed_and_dynamic_paths_with_types[sorted_paths_index].first < shared_data_path)
|
||||
{
|
||||
auto & [path, type] = sorted_typed_and_dynamic_paths_with_types[sorted_paths_index];
|
||||
paths_column->insertData(path.data(), path.size());
|
||||
/// Update type for path from dynamic paths.
|
||||
if (auto it = dynamic_path_columns.find(path); it != dynamic_path_columns.end())
|
||||
{
|
||||
/// Skip NULL values.
|
||||
if (it->second->isNullAt(i))
|
||||
{
|
||||
++sorted_paths_index;
|
||||
continue;
|
||||
}
|
||||
type = getDynamicValueType(it->second, i);
|
||||
}
|
||||
paths_column->insertData(path.data(), path.size());
|
||||
types_column->insertData(type.data(), type.size());
|
||||
++sorted_paths_index;
|
||||
}
|
||||
|
||||
paths_column->insertData(shared_data_path.data, shared_data_path.size);
|
||||
types_column->insertData(type_name.data(), type_name.size());
|
||||
types_column->insertData(type_name->data(), type_name->size());
|
||||
}
|
||||
|
||||
for (; sorted_paths_index != sorted_typed_and_dynamic_paths_with_types.size(); ++sorted_paths_index)
|
||||
{
|
||||
auto & [path, type] = sorted_typed_and_dynamic_paths_with_types[sorted_paths_index];
|
||||
paths_column->insertData(path.data(), path.size());
|
||||
if (auto it = dynamic_path_columns.find(path); it != dynamic_path_columns.end())
|
||||
{
|
||||
/// Skip NULL values.
|
||||
if (it->second->isNullAt(i))
|
||||
continue;
|
||||
type = getDynamicValueType(it->second, i);
|
||||
}
|
||||
paths_column->insertData(path.data(), path.size());
|
||||
types_column->insertData(type.data(), type.size());
|
||||
}
|
||||
|
||||
@ -310,17 +348,18 @@ private:
|
||||
const auto & variant_info = dynamic_column->getVariantInfo();
|
||||
const auto & variant_column = dynamic_column->getVariantColumn();
|
||||
auto global_discr = variant_column.globalDiscriminatorAt(i);
|
||||
if (global_discr == ColumnVariant::NULL_DISCRIMINATOR)
|
||||
return "None";
|
||||
|
||||
/// We don't output path with NULL values. It should be checked before calling getDynamicValueType.
|
||||
chassert(global_discr != ColumnVariant::NULL_DISCRIMINATOR);
|
||||
return variant_info.variant_names[global_discr];
|
||||
}
|
||||
|
||||
String getDynamicValueTypeFromSharedData(StringRef value) const
|
||||
std::optional<String> getDynamicValueTypeFromSharedData(StringRef value) const
|
||||
{
|
||||
ReadBufferFromMemory buf(value.data, value.size);
|
||||
auto type = decodeDataType(buf);
|
||||
return isNothing(type) ? "None" : type->getName();
|
||||
if (isNothing(type))
|
||||
return std::nullopt;
|
||||
return type->getName();
|
||||
}
|
||||
};
|
||||
|
||||
@ -328,12 +367,143 @@ private:
|
||||
|
||||
REGISTER_FUNCTION(JSONPaths)
|
||||
{
|
||||
factory.registerFunction<FunctionJSONPaths<JSONAllPathsImpl>>();
|
||||
factory.registerFunction<FunctionJSONPaths<JSONAllPathsWithTypesImpl>>();
|
||||
factory.registerFunction<FunctionJSONPaths<JSONDynamicPathsImpl>>();
|
||||
factory.registerFunction<FunctionJSONPaths<JSONDynamicPathsWithTypesImpl>>();
|
||||
factory.registerFunction<FunctionJSONPaths<JSONSharedDataPathsImpl>>();
|
||||
factory.registerFunction<FunctionJSONPaths<JSONSharedDataPathsWithTypesImpl>>();
|
||||
factory.registerFunction<FunctionJSONPaths<JSONAllPathsImpl>>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Returns the list of all paths stored in each row in JSON column.
|
||||
)",
|
||||
.syntax = {"JSONAllPaths(json)"},
|
||||
.arguments = {{"json", "JSON column"}},
|
||||
.examples = {{{
|
||||
"Example",
|
||||
R"(
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONAllPaths(json) FROM test;
|
||||
)",
|
||||
R"(
|
||||
┌─json─────────────────────────────────┬─JSONAllPaths(json)─┐
|
||||
│ {"a":"42"} │ ['a'] │
|
||||
│ {"b":"Hello"} │ ['b'] │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ ['a','c'] │
|
||||
└──────────────────────────────────────┴────────────────────┘
|
||||
)"}}},
|
||||
.categories{"JSON"},
|
||||
});
|
||||
|
||||
factory.registerFunction<FunctionJSONPaths<JSONAllPathsWithTypesImpl>>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Returns the list of all paths and their data types stored in each row in JSON column.
|
||||
)",
|
||||
.syntax = {"JSONAllPathsWithTypes(json)"},
|
||||
.arguments = {{"json", "JSON column"}},
|
||||
.examples = {{{
|
||||
"Example",
|
||||
R"(
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONAllPathsWithTypes(json) FROM test;
|
||||
)",
|
||||
R"(
|
||||
┌─json─────────────────────────────────┬─JSONAllPathsWithTypes(json)───────────────┐
|
||||
│ {"a":"42"} │ {'a':'Int64'} │
|
||||
│ {"b":"Hello"} │ {'b':'String'} │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ {'a':'Array(Nullable(Int64))','c':'Date'} │
|
||||
└──────────────────────────────────────┴───────────────────────────────────────────┘
|
||||
)"}}},
|
||||
.categories{"JSON"},
|
||||
});
|
||||
|
||||
factory.registerFunction<FunctionJSONPaths<JSONDynamicPathsImpl>>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Returns the list of dynamic paths that are stored as separate subcolumns in JSON column.
|
||||
)",
|
||||
.syntax = {"JSONDynamicPaths(json)"},
|
||||
.arguments = {{"json", "JSON column"}},
|
||||
.examples = {{{
|
||||
"Example",
|
||||
R"(
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONDynamicPaths(json) FROM test;
|
||||
)",
|
||||
R"(
|
||||
┌─json─────────────────────────────────┬─JSONDynamicPaths(json)─┐
|
||||
│ {"a":"42"} │ ['a'] │
|
||||
│ {"b":"Hello"} │ [] │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ ['a'] │
|
||||
└──────────────────────────────────────┴────────────────────────┘
|
||||
)"}}},
|
||||
.categories{"JSON"},
|
||||
});
|
||||
|
||||
factory.registerFunction<FunctionJSONPaths<JSONDynamicPathsWithTypesImpl>>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Returns the list of dynamic paths that are stored as separate subcolumns and their types in each row in JSON column.
|
||||
)",
|
||||
.syntax = {"JSONDynamicPathsWithTypes(json)"},
|
||||
.arguments = {{"json", "JSON column"}},
|
||||
.examples = {{{
|
||||
"Example",
|
||||
R"(
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONDynamicPathsWithTypes(json) FROM test;
|
||||
)",
|
||||
R"(
|
||||
┌─json─────────────────────────────────┬─JSONDynamicPathsWithTypes(json)─┐
|
||||
│ {"a":"42"} │ {'a':'Int64'} │
|
||||
│ {"b":"Hello"} │ {} │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ {'a':'Array(Nullable(Int64))'} │
|
||||
└──────────────────────────────────────┴─────────────────────────────────┘
|
||||
)"}}},
|
||||
.categories{"JSON"},
|
||||
});
|
||||
|
||||
factory.registerFunction<FunctionJSONPaths<JSONSharedDataPathsImpl>>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Returns the list of paths that are stored in shared data structure in JSON column.
|
||||
)",
|
||||
.syntax = {"JSONDynamicPaths(json)"},
|
||||
.arguments = {{"json", "JSON column"}},
|
||||
.examples = {{{
|
||||
"Example",
|
||||
R"(
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONSharedDataPaths(json) FROM test;
|
||||
)",
|
||||
R"(
|
||||
┌─json─────────────────────────────────┬─JSONSharedDataPaths(json)─┐
|
||||
│ {"a":"42"} │ [] │
|
||||
│ {"b":"Hello"} │ ['b'] │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ ['c'] │
|
||||
└──────────────────────────────────────┴───────────────────────────┘
|
||||
)"}}},
|
||||
.categories{"JSON"},
|
||||
});
|
||||
|
||||
factory.registerFunction<FunctionJSONPaths<JSONSharedDataPathsWithTypesImpl>>(FunctionDocumentation{
|
||||
.description = R"(
|
||||
Returns the list of paths that are stored in shared data structure and their types in each row in JSON column.
|
||||
)",
|
||||
.syntax = {"JSONDynamicPathsWithTypes(json)"},
|
||||
.arguments = {{"json", "JSON column"}},
|
||||
.examples = {{{
|
||||
"Example",
|
||||
R"(
|
||||
CREATE TABLE test (json JSON(max_dynamic_paths=1)) ENGINE = Memory;
|
||||
INSERT INTO test FORMAT JSONEachRow {"json" : {"a" : 42}}, {"json" : {"b" : "Hello"}}, {"json" : {"a" : [1, 2, 3], "c" : "2020-01-01"}}
|
||||
SELECT json, JSONDynamicPathsWithTypes(json) FROM test;
|
||||
)",
|
||||
R"(
|
||||
┌─json─────────────────────────────────┬─JSONDynamicPathsWithTypes(json)─┐
|
||||
│ {"a":"42"} │ {'a':'Int64'} │
|
||||
│ {"b":"Hello"} │ {} │
|
||||
│ {"a":["1","2","3"],"c":"2020-01-01"} │ {'a':'Array(Nullable(Int64))'} │
|
||||
└──────────────────────────────────────┴─────────────────────────────────┘
|
||||
)"}}},
|
||||
.categories{"JSON"},
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -282,6 +282,45 @@ bool ParserTableAsStringLiteralIdentifier::parseImpl(Pos & pos, ASTPtr & node, E
|
||||
return true;
|
||||
}
|
||||
|
||||
namespace
|
||||
{
|
||||
|
||||
/// Parser of syntax sugar for reading JSON subcolumns of type Array(JSON):
|
||||
/// json.a.b[][].c -> json.a.b.:Array(Array(JSON)).c
|
||||
class ParserArrayOfJSONIdentifierAddition : public IParserBase
|
||||
{
|
||||
public:
|
||||
String getLastArrayOfJSONSubcolumnIdentifier() const
|
||||
{
|
||||
String subcolumn = ":`";
|
||||
for (size_t i = 0; i != last_array_level; ++i)
|
||||
subcolumn += "Array(";
|
||||
subcolumn += "JSON";
|
||||
for (size_t i = 0; i != last_array_level; ++i)
|
||||
subcolumn += ")";
|
||||
return subcolumn + "`";
|
||||
}
|
||||
|
||||
protected:
|
||||
const char * getName() const override { return "ParserArrayOfJSONIdentifierDelimiter"; }
|
||||
|
||||
bool parseImpl(Pos & pos, ASTPtr & /*node*/, Expected & expected) override
|
||||
{
|
||||
last_array_level = 0;
|
||||
ParserTokenSequence brackets_parser(std::vector<TokenType>{TokenType::OpeningSquareBracket, TokenType::ClosingSquareBracket});
|
||||
if (!brackets_parser.check(pos, expected))
|
||||
return false;
|
||||
++last_array_level;
|
||||
while (brackets_parser.check(pos, expected))
|
||||
++last_array_level;
|
||||
return true;
|
||||
}
|
||||
|
||||
private:
|
||||
size_t last_array_level;
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
|
||||
{
|
||||
@ -290,6 +329,7 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
|
||||
delimiter_parsers.emplace_back(std::make_unique<ParserTokenSequence>(std::vector<TokenType>{TokenType::Dot, TokenType::Colon}), SpecialDelimiter::JSON_PATH_DYNAMIC_TYPE);
|
||||
delimiter_parsers.emplace_back(std::make_unique<ParserTokenSequence>(std::vector<TokenType>{TokenType::Dot, TokenType::Caret}), SpecialDelimiter::JSON_PATH_PREFIX);
|
||||
delimiter_parsers.emplace_back(std::make_unique<ParserToken>(TokenType::Dot), SpecialDelimiter::NONE);
|
||||
ParserArrayOfJSONIdentifierAddition array_of_json_identifier_addition;
|
||||
|
||||
std::vector<String> parts;
|
||||
SpecialDelimiter last_special_delimiter = SpecialDelimiter::NONE;
|
||||
@ -308,16 +348,23 @@ bool ParserCompoundIdentifier::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
|
||||
break;
|
||||
}
|
||||
|
||||
is_first = false;
|
||||
|
||||
if (last_special_delimiter != SpecialDelimiter::NONE)
|
||||
{
|
||||
parts.push_back(static_cast<char>(last_special_delimiter) + backQuote(getIdentifierName(element)));
|
||||
}
|
||||
else
|
||||
{
|
||||
parts.push_back(getIdentifierName(element));
|
||||
/// Check if we have Array of JSON subcolumn additioon after identifier
|
||||
/// and replace it with corresponding type subcolumn.
|
||||
if (!is_first && array_of_json_identifier_addition.check(pos, expected))
|
||||
parts.push_back(array_of_json_identifier_addition.getLastArrayOfJSONSubcolumnIdentifier());
|
||||
}
|
||||
|
||||
if (parts.back().empty())
|
||||
params.push_back(element->as<ASTIdentifier>()->getParam());
|
||||
|
||||
is_first = false;
|
||||
begin = pos;
|
||||
bool parsed_delimiter = false;
|
||||
for (const auto & [parser, special_delimiter] : delimiter_parsers)
|
||||
|
||||
@ -56,6 +56,7 @@ protected:
|
||||
* There is also special delimiters `.:` and `.^` for JSON type subcolumns. In case of special delimiter
|
||||
* the next identifier part after it will include special delimiter and be back quoted always: json.a.b.:UInt32 -> ['json', 'a', 'b', ':`UInt32`'].
|
||||
* It's needed to distinguish identifiers json.a.b.:UInt32 and json.a.b.`:UInt32`.
|
||||
* There is also a special syntax sugar for reading JSON subcolumns of type Array(JSON): json.a.b[][].c -> json.a.b.:Array(Array(JSON)).c
|
||||
*/
|
||||
class ParserCompoundIdentifier : public IParserBase
|
||||
{
|
||||
|
||||
@ -34,6 +34,7 @@ With 2023-11-14 05:50:12.123
|
||||
With hallo
|
||||
With [\'foo\',\'bar\']
|
||||
With {"foo":"bar"}
|
||||
With {"foo":"bar"}
|
||||
With (42,\'foo\')
|
||||
With {42:\'foo\'}
|
||||
With 122.233.64.201
|
||||
|
||||
@ -2,6 +2,7 @@
|
||||
-- no-fasttest: json type needs rapidjson library, geo types need s2 geometry
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
SET allow_experimental_json_type = 1;
|
||||
SET allow_suspicious_low_cardinality_types=1;
|
||||
|
||||
SELECT '-- Const string + non-const arbitrary type';
|
||||
@ -40,6 +41,7 @@ SELECT concat('With ', materialize('2023-11-14 05:50:12.123' :: DateTime64(3, 'E
|
||||
SELECT concat('With ', materialize('hallo' :: Enum('hallo' = 1)));
|
||||
SELECT concat('With ', materialize(['foo', 'bar'] :: Array(String)));
|
||||
SELECT concat('With ', materialize('{"foo": "bar"}' :: JSON));
|
||||
SELECT concat('With ', materialize('{"foo": "bar"}' :: Object('json')));
|
||||
SELECT concat('With ', materialize((42, 'foo') :: Tuple(Int32, String)));
|
||||
SELECT concat('With ', materialize(map(42, 'foo') :: Map(Int32, String)));
|
||||
SELECT concat('With ', materialize('122.233.64.201' :: IPv4));
|
||||
|
||||
14
tests/queries/0_stateless/01825_new_type_json_10.reference
Normal file
14
tests/queries/0_stateless/01825_new_type_json_10.reference
Normal file
@ -0,0 +1,14 @@
|
||||
('a.b','Int64')
|
||||
('a.c','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
('a.c','Array(Nullable(String))')
|
||||
('e','Array(Nullable(Int64))')
|
||||
('f','Int64')
|
||||
('d','Int64')
|
||||
{"o":{"a":{"b":"1","c":[{"d":"10","e":["31"]},{"d":"20","e":["63","127"]}]}}}
|
||||
{"o":{"a":{"b":"2","c":[]}}}
|
||||
{"o":{"a":{"b":"3","c":[{"e":["32"],"f":"20"},{"e":["64","128"],"f":"30"}]}}}
|
||||
{"o":{"a":{"b":"4","c":[]}}}
|
||||
1 [10,20] [[31],[63,127]] [NULL,NULL]
|
||||
2 [] [] []
|
||||
3 [NULL,NULL] [[32],[64,128]] [20,30]
|
||||
4 [] [] []
|
||||
16
tests/queries/0_stateless/01825_new_type_json_10.sql
Normal file
16
tests/queries/0_stateless/01825_new_type_json_10.sql
Normal file
@ -0,0 +1,16 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
SET allow_experimental_json_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_10;
|
||||
CREATE TABLE t_json_10 (o JSON) ENGINE = Memory;
|
||||
|
||||
INSERT INTO t_json_10 FORMAT JSONAsObject {"a": {"b": 1, "c": [{"d": 10, "e": [31]}, {"d": 20, "e": [63, 127]}]}} {"a": {"b": 2, "c": []}}
|
||||
INSERT INTO t_json_10 FORMAT JSONAsObject {"a": {"b": 3, "c": [{"f": 20, "e": [32]}, {"f": 30, "e": [64, 128]}]}} {"a": {"b": 4, "c": []}}
|
||||
|
||||
SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(o)) FROM t_json_10;
|
||||
SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(o.a.c.:`Array(JSON)`))) FROM t_json_10;
|
||||
SELECT o FROM t_json_10 ORDER BY o.a.b FORMAT JSONEachRow;
|
||||
SELECT o.a.b, o.a.c.:`Array(JSON)`.d, o.a.c.:`Array(JSON)`.e, o.a.c.:`Array(JSON)`.f FROM t_json_10 ORDER BY o.a.b;
|
||||
|
||||
DROP TABLE t_json_10;
|
||||
13
tests/queries/0_stateless/01825_new_type_json_11.reference
Normal file
13
tests/queries/0_stateless/01825_new_type_json_11.reference
Normal file
@ -0,0 +1,13 @@
|
||||
('id','Int64')
|
||||
('key_1','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
('key_2','Int64')
|
||||
('key_3','Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')
|
||||
('key_4','Array(JSON(max_dynamic_types=4, max_dynamic_paths=16))')
|
||||
('key_7','Int64')
|
||||
('key_5','Int64')
|
||||
{"obj":{"id":"1","key_1":[{"key_2":"100","key_3":[{"key_4":[{"key_5":"-2"}],"key_7":"257"}]},{"key_2":"65536"}]}}
|
||||
{"obj":{"id":"2","key_1":[{"key_2":"101","key_3":[{"key_4":[{"key_5":"-2"}]}]},{"key_2":"102","key_3":[{"key_7":"257"}]},{"key_2":"65536"}]}}
|
||||
{"obj.key_1.:`Array(JSON)`.key_3":[[{"key_4":[{"key_5":"-2"}],"key_7":"257"}],null]}
|
||||
{"obj.key_1.:`Array(JSON)`.key_3":[[{"key_4":[{"key_5":"-2"}]}],[{"key_7":"257"}],null]}
|
||||
[[[-2]],[]] [[257],[]]
|
||||
[[[-2]],[[]],[]] [[NULL],[257],[]]
|
||||
64
tests/queries/0_stateless/01825_new_type_json_11.sh
Executable file
64
tests/queries/0_stateless/01825_new_type_json_11.sh
Executable file
@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_11"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_11 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_json_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_11 FORMAT JSONAsObject"
|
||||
{
|
||||
"id": 1,
|
||||
"key_1":[
|
||||
{
|
||||
"key_2":100,
|
||||
"key_3": [
|
||||
{
|
||||
"key_7":257,
|
||||
"key_4":[{"key_5":-2}]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key_2":65536
|
||||
}
|
||||
]
|
||||
}
|
||||
{
|
||||
"id": 2,
|
||||
"key_1":[
|
||||
{
|
||||
"key_2":101,
|
||||
"key_3": [
|
||||
{
|
||||
"key_4":[{"key_5":-2}]
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key_2":102,
|
||||
"key_3": [
|
||||
{
|
||||
"key_7":257
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key_2":65536
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(obj)) FROM t_json_11;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(obj.key_1[]))) FROM t_json_11;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(obj.key_1[].key_3[])))) FROM t_json_11;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(arrayJoin(obj.key_1[].key_3[].key_4[]))))) FROM t_json_11;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_11 ORDER BY obj.id FORMAT JSONEachRow"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT obj.key_1[].key_3 FROM t_json_11 ORDER BY obj.id FORMAT JSONEachRow"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT obj.key_1[].key_3[].key_4[].key_5, obj.key_1[].key_3[].key_7 FROM t_json_11 ORDER BY obj.id"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE t_json_11;"
|
||||
14
tests/queries/0_stateless/01825_new_type_json_12.reference
Normal file
14
tests/queries/0_stateless/01825_new_type_json_12.reference
Normal file
@ -0,0 +1,14 @@
|
||||
('id','Int64')
|
||||
('key_0','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
('key_1','Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')
|
||||
('key_3','Array(JSON(max_dynamic_types=4, max_dynamic_paths=16))')
|
||||
('key_4','Int64')
|
||||
('key_5','Float64')
|
||||
('key_6','Float64')
|
||||
('key_7','Int64')
|
||||
('key_6','String')
|
||||
('key_5','Int64')
|
||||
('key_7','Float64')
|
||||
('key_4','String')
|
||||
{"obj":{"id":"1","key_0":[{"key_1":[{"key_3":[{"key_4":"1048576","key_5":0.0001048576,"key_6":25.5,"key_7":"1025"},{"key_6":"","key_7":"2"}]}]},{},{"key_1":[{"key_3":[{"key_5":"-1","key_6":"aqbjfiruu","key_7":-922337203685477600},{"key_4":"","key_6":"","key_7":"65537"}]},{"key_3":[{"key_4":"ghdqyeiom","key_5":"1048575","key_7":21474836.48}]}]}]}}
|
||||
[[[1048576,NULL]],[],[[NULL,''],['ghdqyeiom']]] [[[0.0001048576,NULL]],[],[[-1,NULL],[1048575]]] [[[25.5,'']],[],[['aqbjfiruu',''],[NULL]]] [[[1025,2]],[],[[-922337203685477600,65537],[21474836.48]]]
|
||||
54
tests/queries/0_stateless/01825_new_type_json_12.sh
Executable file
54
tests/queries/0_stateless/01825_new_type_json_12.sh
Executable file
@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_12"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_12 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_json_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_12 FORMAT JSONAsObject"
|
||||
{
|
||||
"id": 1,
|
||||
"key_0":[
|
||||
{
|
||||
"key_1":[
|
||||
{
|
||||
"key_3":[
|
||||
{"key_7":1025,"key_6":25.5,"key_4":1048576,"key_5":0.0001048576},
|
||||
{"key_7":2,"key_6":"","key_4":null}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
{},
|
||||
{
|
||||
"key_1":[
|
||||
{
|
||||
"key_3":[
|
||||
{"key_7":-922337203685477580.8,"key_6":"aqbjfiruu","key_5":-1},
|
||||
{"key_7":65537,"key_6":"","key_4":""}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key_3":[
|
||||
{"key_7":21474836.48,"key_4":"ghdqyeiom","key_5":1048575}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(obj)) FROM t_json_12;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(obj.key_0[]))) FROM t_json_12;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(obj.key_0[].key_1[])))) FROM t_json_12;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(arrayJoin(obj.key_0[].key_1[].key_3[]))))) FROM t_json_12;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_12 ORDER BY obj.id FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1
|
||||
$CLICKHOUSE_CLIENT -q "SELECT obj.key_0[].key_1[].key_3[].key_4, obj.key_0[].key_1[].key_3[].key_5, \
|
||||
obj.key_0[].key_1[].key_3[].key_6, obj.key_0[].key_1[].key_3[].key_7 FROM t_json_12 ORDER BY obj.id"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE t_json_12;"
|
||||
@ -0,0 +1,4 @@
|
||||
('id','Int64')
|
||||
('key_1','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
{"obj":{"id":"1","key_1":[{"key_2":[{"key_3":[{"key_8":"65537"},{"key_4":[{"key_5":-0.02},{"key_7":"1023"},{"key_6":"9223372036854775807","key_7":"1"}]},{"key_4":[{"key_7":"65537"}]}]}]}]}}
|
||||
\N \N \N \N
|
||||
50
tests/queries/0_stateless/01825_new_type_json_13.sh
Executable file
50
tests/queries/0_stateless/01825_new_type_json_13.sh
Executable file
@ -0,0 +1,50 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_13"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_13 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_json_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_13 FORMAT JSONAsObject"
|
||||
{
|
||||
"id": 1,
|
||||
"key_1":[
|
||||
{
|
||||
"key_2":[
|
||||
{
|
||||
"key_3":[
|
||||
{"key_8":65537},
|
||||
{
|
||||
"key_4":[
|
||||
{"key_5":-0.02},
|
||||
{"key_7":1023},
|
||||
{"key_7":1,"key_6":9223372036854775807}
|
||||
]
|
||||
},
|
||||
{
|
||||
"key_4":[{"key_7":65537,"key_6":null}]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(obj)) FROM t_json_13;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(obj.key1[]))) FROM t_json_13;"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT obj FROM t_json_13 ORDER BY obj.id FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1
|
||||
$CLICKHOUSE_CLIENT -q "SELECT \
|
||||
obj.key_1.key_2.key_3.key_8, \
|
||||
obj.key_1.key_2.key_3.key_4.key_5, \
|
||||
obj.key_1.key_2.key_3.key_4.key_6, \
|
||||
obj.key_1.key_2.key_3.key_4.key_7 \
|
||||
FROM t_json_13 ORDER BY obj.id"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE t_json_13;"
|
||||
@ -0,0 +1,2 @@
|
||||
1 {"k1":"1"} {'k1':'Int64'}
|
||||
1 {"k1":["1","2"]} {'k1':'Array(Nullable(Int64))'}
|
||||
16
tests/queries/0_stateless/01825_new_type_json_18.sql
Normal file
16
tests/queries/0_stateless/01825_new_type_json_18.sql
Normal file
@ -0,0 +1,16 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
SET allow_experimental_json_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_2;
|
||||
|
||||
CREATE TABLE t_json_2(id UInt64, data JSON)
|
||||
ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data" : {"k1": 1}};
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_2 ORDER BY id;
|
||||
|
||||
TRUNCATE TABLE t_json_2;
|
||||
|
||||
INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data" : {"k1": [1, 2]}};
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_2 ORDER BY id;
|
||||
24
tests/queries/0_stateless/01825_new_type_json_2.reference
Normal file
24
tests/queries/0_stateless/01825_new_type_json_2.reference
Normal file
@ -0,0 +1,24 @@
|
||||
1 {"k1":"1","k2":"2"} {'k1':'Int64','k2':'Int64'}
|
||||
2 {"k2":"3","k3":"4"} {'k2':'Int64','k3':'Int64'}
|
||||
1 1 2 \N
|
||||
2 \N 3 4
|
||||
1 {"k1":"1","k2":"2"} {'k1':'Int64','k2':'Int64'}
|
||||
2 {"k2":"3","k3":"4"} {'k2':'Int64','k3':'Int64'}
|
||||
3 {"k3":"10"} {'k3':'Int64'}
|
||||
4 {"k2":"5","k3":"str"} {'k2':'Int64','k3':'String'}
|
||||
1 1 2 \N
|
||||
2 \N 3 4
|
||||
3 \N \N 10
|
||||
4 \N 5 str
|
||||
============
|
||||
1 {"k1":[1,2,3.3]} {'k1':'Array(Nullable(Float64))'}
|
||||
1 [1,2,3.3]
|
||||
1 {"k1":[1,2,3.3]} {'k1':'Array(Nullable(Float64))'}
|
||||
2 {"k1":["a","4","b"]} {'k1':'Array(Nullable(String))'}
|
||||
1 [1,2,3.3]
|
||||
2 ['a','4','b']
|
||||
============
|
||||
1 {"k1":[{"k2":"11"},{"k3":"22"}]} {'k1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'}
|
||||
2 {"k1":[{"k3":"33"},{"k4":"44"},{"k3":"55","k4":"66"}]} {'k1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'}
|
||||
1 \N \N \N
|
||||
2 \N \N \N
|
||||
41
tests/queries/0_stateless/01825_new_type_json_2.sql
Normal file
41
tests/queries/0_stateless/01825_new_type_json_2.sql
Normal file
@ -0,0 +1,41 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
SET allow_experimental_json_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_2;
|
||||
|
||||
CREATE TABLE t_json_2(id UInt64, data JSON)
|
||||
ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_2/t_json_2', 'r1') ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data": {"k1": 1, "k2" : 2}} {"id": 2, "data": {"k2": 3, "k3" : 4}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_2 ORDER BY id;
|
||||
SELECT id, data.k1, data.k2, data.k3 FROM t_json_2 ORDER BY id;
|
||||
|
||||
INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 3, "data": {"k3" : 10}} {"id": 4, "data": {"k2": 5, "k3" : "str"}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_2 ORDER BY id;
|
||||
SELECT id, data.k1, data.k2, data.k3 FROM t_json_2 ORDER BY id;
|
||||
|
||||
SELECT '============';
|
||||
TRUNCATE TABLE t_json_2;
|
||||
|
||||
INSERT INTO TABLE t_json_2 FORMAT JSONEachRow {"id": 1, "data": {"k1" : [1, 2, 3.3]}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_2 ORDER BY id;
|
||||
SELECT id, data.k1 FROM t_json_2 ORDEr BY id;
|
||||
|
||||
INSERT INTO TABLE t_json_2 FORMAT JSONEachRow {"id": 2, "data": {"k1" : ["a", 4, "b"]}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_2 ORDER BY id;
|
||||
SELECT id, data.k1 FROM t_json_2 ORDER BY id;
|
||||
|
||||
SELECT '============';
|
||||
TRUNCATE TABLE t_json_2;
|
||||
|
||||
INSERT INTO TABLE t_json_2 FORMAT JSONEachRow {"id": 1, "data": {"k1" : [{"k2" : 11}, {"k3" : 22}]}} {"id": 2, "data": {"k1" : [{"k3" : 33}, {"k4" : 44}, {"k3" : 55, "k4" : 66}]}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_2 ORDER BY id;
|
||||
SELECT id, data.k1.k2, data.k1.k3, data.k1.k4 FROM t_json_2 ORDER BY id;
|
||||
|
||||
DROP TABLE t_json_2;
|
||||
59
tests/queries/0_stateless/01825_new_type_json_3.reference.j2
Normal file
59
tests/queries/0_stateless/01825_new_type_json_3.reference.j2
Normal file
@ -0,0 +1,59 @@
|
||||
1 {} {}
|
||||
2 {"k1":"v1","k2":"2"} {'k1':'String','k2':'Int64'}
|
||||
1 \N \N
|
||||
2 v1 2
|
||||
========
|
||||
1 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
2 {"k1":[{"k2":"v1","k3":"v3"},{"k2":"v4"}]} {'k1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'}
|
||||
1 \N \N
|
||||
2 \N \N
|
||||
1 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
2 {"k1":[{"k2":"v1","k3":"v3"},{"k2":"v4"}]} {'k1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'}
|
||||
3 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
4 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
1 \N \N
|
||||
2 \N \N
|
||||
3 \N \N
|
||||
4 \N \N
|
||||
all_2_2_0 data JSON
|
||||
all_3_3_0 data JSON
|
||||
data JSON
|
||||
1 \N \N
|
||||
2 \N \N
|
||||
3 \N \N
|
||||
4 \N \N
|
||||
========
|
||||
1 {"k1":{"k2":"1","k3":"foo"}} {'k1.k2':'Int64','k1.k3':'String'}
|
||||
2 {"k4":["1","2","3"]} {'k4':'Array(Nullable(Int64))'}
|
||||
3 {"k1":{"k2":"10"},"k4":[]} {'k1.k2':'Int64','k4':'Array(Nullable(String))'}
|
||||
1 1 foo \N
|
||||
2 \N \N [1,2,3]
|
||||
3 10 \N []
|
||||
1 {} {}
|
||||
2 {"k1":"v1","k2":"2"} {'k1':'String','k2':'Int64'}
|
||||
1 \N \N
|
||||
2 v1 2
|
||||
========
|
||||
1 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
2 {"k1":[{"k2":"v1","k3":"v3"},{"k2":"v4"}]} {'k1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'}
|
||||
1 \N \N
|
||||
2 \N \N
|
||||
1 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
2 {"k1":[{"k2":"v1","k3":"v3"},{"k2":"v4"}]} {'k1':'Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))'}
|
||||
3 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
4 {"k1":[]} {'k1':'Array(Nullable(String))'}
|
||||
1 \N \N
|
||||
2 \N \N
|
||||
3 \N \N
|
||||
4 \N \N
|
||||
1 \N \N
|
||||
2 \N \N
|
||||
3 \N \N
|
||||
4 \N \N
|
||||
========
|
||||
1 {"k1":{"k2":"1","k3":"foo"}} {'k1.k2':'Int64','k1.k3':'String'}
|
||||
2 {"k4":["1","2","3"]} {'k4':'Array(Nullable(Int64))'}
|
||||
3 {"k1":{"k2":"10"},"k4":[]} {'k1.k2':'Int64','k4':'Array(Nullable(String))'}
|
||||
1 1 foo \N
|
||||
2 \N \N [1,2,3]
|
||||
3 10 \N []
|
||||
64
tests/queries/0_stateless/01825_new_type_json_3.sql.j2
Normal file
64
tests/queries/0_stateless/01825_new_type_json_3.sql.j2
Normal file
@ -0,0 +1,64 @@
|
||||
|
||||
-- Tags: no-fasttest
|
||||
|
||||
{% for engine in ["ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_3/t_json_3', 'r1') ORDER BY tuple()", "Memory"] -%}
|
||||
|
||||
SET insert_keeper_fault_injection_probability=0; -- disable fault injection; part ids are non-deterministic in case of insert retries
|
||||
|
||||
SET allow_experimental_json_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_3;
|
||||
|
||||
CREATE TABLE t_json_3(id UInt64, data JSON)
|
||||
ENGINE = {{ engine }};
|
||||
|
||||
{% if 'MergeTree' in engine %}
|
||||
SYSTEM STOP MERGES t_json_3;
|
||||
{% endif %}
|
||||
|
||||
INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 1, "data": {"k1": null}}, {"id": 2, "data": {"k1": "v1", "k2" : 2}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_3 ORDER BY id;
|
||||
SELECT id, data.k1, data.k2 FROM t_json_3 ORDER BY id;
|
||||
|
||||
SELECT '========';
|
||||
TRUNCATE TABLE t_json_3;
|
||||
|
||||
INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 1, "data": {"k1" : []}} {"id": 2, "data": {"k1" : [{"k2" : "v1", "k3" : "v3"}, {"k2" : "v4"}]}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_3 ORDER BY id;
|
||||
SELECT id, data.k1.k2, data.k1.k3 FROM t_json_3 ORDER BY id;
|
||||
|
||||
INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 3, "data": {"k1" : []}} {"id": 4, "data": {"k1" : []}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_3 ORDER BY id;
|
||||
SELECT id, data.k1.k2, data.k1.k3 FROM t_json_3 ORDER BY id;
|
||||
|
||||
{% if 'MergeTree' in engine %}
|
||||
SELECT name, column, type
|
||||
FROM system.parts_columns
|
||||
WHERE table = 't_json_3' AND database = currentDatabase() AND active AND column = 'data'
|
||||
ORDER BY name;
|
||||
|
||||
SYSTEM START MERGES t_json_3;
|
||||
OPTIMIZE TABLE t_json_3 FINAL;
|
||||
|
||||
SELECT column, type
|
||||
FROM system.parts_columns
|
||||
WHERE table = 't_json_3' AND database = currentDatabase() AND active AND column = 'data'
|
||||
ORDER BY name;
|
||||
{% endif %}
|
||||
|
||||
SELECT id, data.k1.k2, data.k1.k3 FROM t_json_3 ORDER BY id;
|
||||
|
||||
SELECT '========';
|
||||
TRUNCATE TABLE t_json_3;
|
||||
|
||||
INSERT INTO t_json_3 FORMAT JSONEachRow {"id": 1, "data": {"k1" : {"k2" : 1, "k3" : "foo"}}} {"id": 2, "data": {"k1" : null, "k4" : [1, 2, 3]}}, {"id" : 3, "data": {"k1" : {"k2" : 10}, "k4" : []}};
|
||||
|
||||
SELECT id, data, JSONAllPathsWithTypes(data) FROM t_json_3 ORDER BY id;
|
||||
SELECT id, data.k1.k2, data.k1.k3, data.k4 FROM t_json_3 ORDER BY id;
|
||||
|
||||
DROP TABLE t_json_3;
|
||||
|
||||
{% endfor -%}
|
||||
10
tests/queries/0_stateless/01825_new_type_json_6.reference
Normal file
10
tests/queries/0_stateless/01825_new_type_json_6.reference
Normal file
@ -0,0 +1,10 @@
|
||||
('key','String')
|
||||
('out','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
('outputs','Array(Nullable(String))')
|
||||
('type','Int64')
|
||||
('value','Int64')
|
||||
('outputs','Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')
|
||||
('index','Int64')
|
||||
('n','Int64')
|
||||
v1 [0,0] [1,2] [[],[1960131]] [[],[0]]
|
||||
v2 [1,1] [4,3] [[1881212],[]] [[1],[]]
|
||||
59
tests/queries/0_stateless/01825_new_type_json_6.sh
Executable file
59
tests/queries/0_stateless/01825_new_type_json_6.sh
Executable file
@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_6;"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_6 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_json_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_6 FORMAT JSONAsObject"
|
||||
{
|
||||
"key": "v1",
|
||||
"out": [
|
||||
{
|
||||
"type": 0,
|
||||
"value": 1,
|
||||
"outputs": []
|
||||
},
|
||||
{
|
||||
"type": 0,
|
||||
"value": 2,
|
||||
"outputs": [
|
||||
{
|
||||
"index": 1960131,
|
||||
"n": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
{
|
||||
"key": "v2",
|
||||
"out": [
|
||||
{
|
||||
"type": 1,
|
||||
"value": 4,
|
||||
"outputs": [
|
||||
{
|
||||
"index": 1881212,
|
||||
"n": 1
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": 1,
|
||||
"value": 3
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(data)) FROM t_json_6;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(data.out[]))) FROM t_json_6;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(data.out[].outputs[])))) FROM t_json_6;"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT data.key, data.out[].type, data.out[].value, data.out[].outputs[].index, data.out[].outputs[].n FROM t_json_6 ORDER BY data.key"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE t_json_6;"
|
||||
@ -0,0 +1,8 @@
|
||||
{"id":"1","s":{}}
|
||||
{"id":"2","s":{"k1":"100"}}
|
||||
{"id":"1"}
|
||||
{"id":"2"}
|
||||
{"id":"1","s":{}}
|
||||
{"id":"2","s":{"k1":"100"}}
|
||||
{"id":"1"}
|
||||
{"id":"2"}
|
||||
@ -0,0 +1,23 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
{% for storage in ["MergeTree", "ReplicatedMergeTree('/clickhouse/tables/{database}/test_01825_add_column/', 'r1')"] -%}
|
||||
|
||||
DROP TABLE IF EXISTS t_json_add_column;
|
||||
SET allow_experimental_json_type = 1;
|
||||
|
||||
CREATE TABLE t_json_add_column (id UInt64) ENGINE = {{ storage }} ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_add_column VALUES (1);
|
||||
ALTER TABLE t_json_add_column ADD COLUMN s JSON;
|
||||
|
||||
INSERT INTO t_json_add_column VALUES(2, '{"k1": 100}');
|
||||
|
||||
SELECT * FROM t_json_add_column ORDER BY id FORMAT JSONEachRow;
|
||||
|
||||
ALTER TABLE t_json_add_column DROP COLUMN s;
|
||||
|
||||
SELECT * FROM t_json_add_column ORDER BY id FORMAT JSONEachRow;
|
||||
|
||||
DROP TABLE t_json_add_column;
|
||||
|
||||
{% endfor -%}
|
||||
33
tests/queries/0_stateless/01825_new_type_json_btc.reference
Normal file
33
tests/queries/0_stateless/01825_new_type_json_btc.reference
Normal file
@ -0,0 +1,33 @@
|
||||
100
|
||||
('double_spend','Bool')
|
||||
('fee','Int64')
|
||||
('hash','String')
|
||||
('inputs','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
('lock_time','Int64')
|
||||
('out','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
('relayed_by','String')
|
||||
('size','Int64')
|
||||
('time','Int64')
|
||||
('tx_index','Int64')
|
||||
('ver','Int64')
|
||||
('vin_sz','Int64')
|
||||
('vout_sz','Int64')
|
||||
('weight','Int64')
|
||||
('rbf','Bool')
|
||||
('index','Int64')
|
||||
('prev_out.addr','String')
|
||||
('prev_out.n','Int64')
|
||||
('prev_out.script','String')
|
||||
('prev_out.spending_outpoints','Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')
|
||||
('prev_out.spent','Bool')
|
||||
('prev_out.tx_index','Int64')
|
||||
('prev_out.type','Int64')
|
||||
('prev_out.value','Int64')
|
||||
('script','String')
|
||||
('sequence','Int64')
|
||||
('witness','String')
|
||||
('n','Int64')
|
||||
('tx_index','Int64')
|
||||
8174.56 2680
|
||||
2.32 1
|
||||
[[],['{"n":"0","tx_index":"359661801933760"}']]
|
||||
31
tests/queries/0_stateless/01825_new_type_json_btc.sh
Executable file
31
tests/queries/0_stateless/01825_new_type_json_btc.sh
Executable file
@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
mkdir -p ${CLICKHOUSE_USER_FILES_UNIQUE}/
|
||||
rm -rf "${CLICKHOUSE_USER_FILES_UNIQUE:?}"/*
|
||||
cp $CUR_DIR/data_json/btc_transactions.json ${CLICKHOUSE_USER_FILES_UNIQUE}/
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS btc"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE btc (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_json_type 1
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO btc SELECT * FROM file('${CLICKHOUSE_TEST_UNIQUE_NAME}/btc_transactions.json', 'JSONAsObject')"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM btc WHERE NOT ignore(*)"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT distinct arrayJoin(JSONAllPathsWithTypes(data)) from btc"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT distinct arrayJoin(JSONAllPathsWithTypes(arrayJoin(data.inputs.:\`Array(JSON)\`))) from btc"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT distinct arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(data.inputs.:\`Array(JSON)\`.prev_out.spending_outpoints.:\`Array(JSON)\`)))) from btc"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT avg(data.fee.:Int64), median(data.fee.:Int64) FROM btc"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT avg(length(data.inputs.:\`Array(JSON)\`.prev_out.spending_outpoints.:\`Array(JSON)\`) AS outpoints_length), median(outpoints_length) FROM btc"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT data.out.:\`Array(JSON)\`.spending_outpoints.:\`Array(JSON)\` AS outpoints FROM btc WHERE arrayExists(x -> notEmpty(x), outpoints)"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS btc"
|
||||
|
||||
rm ${CLICKHOUSE_USER_FILES_UNIQUE}/btc_transactions.json
|
||||
@ -0,0 +1 @@
|
||||
PushEvent some-repo {"actor":{"avatar_url":"https:\\/\\/avatars.githubusercontent.com\\/u\\/123213213?","display_login":"github-actions","gravatar_id":"","id":"123123123","login":"github-actions[bot]","url":"https:\\/\\/api.github.com\\/users\\/github-actions[bot]"},"created_at":"2022-01-04 07:00:00.000000000","repo":{"id":"1001001010101","name":"some-repo","url":"https:\\/\\/api.github.com\\/repos\\/some-repo"},"type":"PushEvent"}
|
||||
18
tests/queries/0_stateless/01825_new_type_json_ephemeral.sql
Normal file
18
tests/queries/0_stateless/01825_new_type_json_ephemeral.sql
Normal file
@ -0,0 +1,18 @@
|
||||
|
||||
SET allow_experimental_json_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_github_json;
|
||||
|
||||
CREATE table t_github_json
|
||||
(
|
||||
event_type LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'type'),
|
||||
repo_name LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'repo', 'name'),
|
||||
message JSON DEFAULT empty(message_raw) ? '{}' : message_raw,
|
||||
message_raw String EPHEMERAL
|
||||
) ENGINE = MergeTree ORDER BY (event_type, repo_name);
|
||||
|
||||
INSERT INTO t_github_json (message_raw) FORMAT JSONEachRow {"message_raw": "{\"type\":\"PushEvent\", \"created_at\": \"2022-01-04 07:00:00\", \"actor\":{\"avatar_url\":\"https://avatars.githubusercontent.com/u/123213213?\",\"display_login\":\"github-actions\",\"gravatar_id\":\"\",\"id\":123123123,\"login\":\"github-actions[bot]\",\"url\":\"https://api.github.com/users/github-actions[bot]\"},\"repo\":{\"id\":1001001010101,\"name\":\"some-repo\",\"url\":\"https://api.github.com/repos/some-repo\"}}"}
|
||||
|
||||
SELECT * FROM t_github_json ORDER BY event_type, repo_name;
|
||||
|
||||
DROP TABLE t_github_json;
|
||||
@ -0,0 +1 @@
|
||||
1
|
||||
27
tests/queries/0_stateless/01825_new_type_json_ghdata_insert_select.sh
Executable file
27
tests/queries/0_stateless/01825_new_type_json_ghdata_insert_select.sh
Executable file
@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2_string"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2_from_string"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2 (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_json_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2_string (data String) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2_from_string (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_json_type 1
|
||||
|
||||
cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata_2 FORMAT JSONAsObject"
|
||||
cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata_2_string FORMAT JSONAsString"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata_2_from_string SELECT data FROM ghdata_2_string"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT \
|
||||
(SELECT groupUniqArrayMap(JSONAllPathsWithTypes(data)), sum(cityHash64(toString(data))) FROM ghdata_2_from_string) = \
|
||||
(SELECT groupUniqArrayMap(JSONAllPathsWithTypes(data)), sum(cityHash64(toString(data))) FROM ghdata_2)"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2_string"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2_from_string"
|
||||
@ -0,0 +1,65 @@
|
||||
1000
|
||||
('_id.$oid','String')
|
||||
('date.$date','String')
|
||||
('teams','Array(JSON(max_dynamic_types=16, max_dynamic_paths=256))')
|
||||
('abbreviation','String')
|
||||
('city','String')
|
||||
('home','Bool')
|
||||
('name','String')
|
||||
('players','Array(JSON(max_dynamic_types=8, max_dynamic_paths=64))')
|
||||
('results.ast','Int64')
|
||||
('results.blk','Int64')
|
||||
('results.drb','Int64')
|
||||
('results.fg','Int64')
|
||||
('results.fg3','Int64')
|
||||
('results.fg3_pct','DateTime64(9)')
|
||||
('results.fg3a','Int64')
|
||||
('results.fg_pct','DateTime64(9)')
|
||||
('results.fga','Int64')
|
||||
('results.ft','Int64')
|
||||
('results.ft_pct','DateTime64(9)')
|
||||
('results.fta','Int64')
|
||||
('results.mp','Int64')
|
||||
('results.orb','Int64')
|
||||
('results.pf','Int64')
|
||||
('results.pts','Int64')
|
||||
('results.stl','Int64')
|
||||
('results.tov','Int64')
|
||||
('results.trb','Int64')
|
||||
('score','Int64')
|
||||
('won','Int64')
|
||||
('results.fg3_pct','String')
|
||||
Boston Celtics 70
|
||||
Los Angeles Lakers 64
|
||||
Milwaukee Bucks 61
|
||||
Philadelphia 76ers 57
|
||||
Atlanta Hawks 55
|
||||
('ast','Int64')
|
||||
('blk','Int64')
|
||||
('drb','Int64')
|
||||
('fg','Int64')
|
||||
('fg3','Int64')
|
||||
('fg3_pct','String')
|
||||
('fg3a','Int64')
|
||||
('fg_pct','DateTime64(9)')
|
||||
('fga','Int64')
|
||||
('ft','Int64')
|
||||
('ft_pct','DateTime64(9)')
|
||||
('fta','Int64')
|
||||
('mp','String')
|
||||
('orb','Int64')
|
||||
('pf','Int64')
|
||||
('player','String')
|
||||
('pts','Int64')
|
||||
('stl','Int64')
|
||||
('tov','Int64')
|
||||
('trb','Int64')
|
||||
('fg3_pct','DateTime64(9)')
|
||||
('fg_pct','String')
|
||||
('ft_pct','String')
|
||||
Larry Bird 10
|
||||
Clyde Drexler 4
|
||||
Alvin Robertson 3
|
||||
Magic Johnson 3
|
||||
Charles Barkley 2
|
||||
1
|
||||
54
tests/queries/0_stateless/01825_new_type_json_nbagames.sh
Executable file
54
tests/queries/0_stateless/01825_new_type_json_nbagames.sh
Executable file
@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames_string"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames_from_string"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_json_type 1
|
||||
|
||||
cat $CUR_DIR/data_json/nbagames_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO nbagames FORMAT JSONAsObject"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM nbagames WHERE NOT ignore(*)"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(data)) from nbagames"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(data.teams[]))) from nbagames"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q \
|
||||
"SELECT teams.name.:String AS name, sum(teams.won.:Int64) AS wins FROM nbagames \
|
||||
ARRAY JOIN data.teams[] AS teams GROUP BY name \
|
||||
ORDER BY wins DESC LIMIT 5;"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT DISTINCT arrayJoin(JSONAllPathsWithTypes(arrayJoin(arrayJoin(data.teams[].players[])))) from nbagames"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q \
|
||||
"SELECT player, sum(triple_double) AS triple_doubles FROM \
|
||||
( \
|
||||
SELECT \
|
||||
arrayJoin(arrayJoin(data.teams[].players[])) as players, \
|
||||
players.player.:String as player, \
|
||||
((players.pts.:Int64 >= 10) + \
|
||||
(players.ast.:Int64 >= 10) + \
|
||||
(players.blk.:Int64 >= 10) + \
|
||||
(players.stl.:Int64 >= 10) + \
|
||||
(players.trb.:Int64 >= 10)) >= 3 AS triple_double \
|
||||
from nbagames \
|
||||
) \
|
||||
GROUP BY player ORDER BY triple_doubles DESC, player LIMIT 5"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames_string (data String) ENGINE = MergeTree ORDER BY tuple()"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames_from_string (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_json_type 1
|
||||
|
||||
cat $CUR_DIR/data_json/nbagames_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO nbagames_string FORMAT JSONAsString"
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO nbagames_from_string SELECT data FROM nbagames_string"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT \
|
||||
(SELECT groupUniqArrayMap(JSONAllPathsWithTypes(data)), sum(cityHash64(toString(data))) FROM nbagames_from_string) = \
|
||||
(SELECT groupUniqArrayMap(JSONAllPathsWithTypes(data)), sum(cityHash64(toString(data))) FROM nbagames)"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames_string"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames_from_string"
|
||||
@ -0,0 +1 @@
|
||||
{'k1':['Int64'],'k2':['String']} 500000
|
||||
@ -0,0 +1,10 @@
|
||||
-- Tags: long
|
||||
DROP TABLE IF EXISTS t_json_parallel;
|
||||
|
||||
SET allow_experimental_json_type = 1, max_insert_threads = 20, max_threads = 20, min_insert_block_size_rows = 65536;
|
||||
CREATE TABLE t_json_parallel (data JSON) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_parallel SELECT materialize('{"k1":1, "k2": "some"}') FROM numbers_mt(500000);
|
||||
SELECT groupUniqArrayMap(JSONAllPathsWithTypes(data)), count() FROM t_json_parallel;
|
||||
|
||||
DROP TABLE t_json_parallel;
|
||||
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_11"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_11 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_11 (obj Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_11 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_12"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_12 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_12 (obj Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_12 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_13"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_13 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_13 (obj Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_13 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -4,7 +4,7 @@ SET allow_experimental_object_type = 1;
|
||||
SET output_format_json_named_tuples_as_objects = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_14;
|
||||
CREATE TABLE t_json_14 (id UInt32, o JSON) ENGINE = Memory;
|
||||
CREATE TABLE t_json_14 (id UInt32, o Object('json')) ENGINE = Memory;
|
||||
|
||||
INSERT INTO t_json_14 VALUES (1, '{"key_10":65536,"key_11":"anve","key_0":{"key_1":{"key_2":1025,"key_3":1},"key_4":1,"key_5":256}}');
|
||||
INSERT INTO t_json_14 VALUES (2, '{"key_0":[{"key_12":"buwvq","key_11":0.0000000255}]}'); -- { serverError INCOMPATIBLE_COLUMNS }
|
||||
|
||||
@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_15"
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_15 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_15 (obj Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_15 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_16"
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_16 (obj JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_16 (obj Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_16 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -4,7 +4,7 @@ DROP TABLE IF EXISTS t_json_17;
|
||||
SET allow_experimental_object_type = 1;
|
||||
SET output_format_json_named_tuples_as_objects = 1;
|
||||
|
||||
CREATE TABLE t_json_17(obj JSON)
|
||||
CREATE TABLE t_json_17(obj Object('json'))
|
||||
ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
DROP FUNCTION IF EXISTS hasValidSizes17;
|
||||
|
||||
@ -8,7 +8,7 @@ SET allow_experimental_object_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_3;
|
||||
|
||||
CREATE TABLE t_json_3(id UInt64, data JSON)
|
||||
CREATE TABLE t_json_3(id UInt64, data Object('json'))
|
||||
ENGINE = {{ engine }};
|
||||
|
||||
{% if 'MergeTree' in engine %}
|
||||
|
||||
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_4"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_4(id UInt64, data JSON) \
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_4(id UInt64, data Object('json')) \
|
||||
ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
echo '{"id": 1, "data": {"k1": "v1"}}, {"id": 2, "data": {"k1": [1, 2]}}' \
|
||||
|
||||
@ -9,7 +9,7 @@ DROP TABLE IF EXISTS t_json_5;
|
||||
DROP TABLE IF EXISTS t_json_str_5;
|
||||
|
||||
CREATE TABLE t_json_str_5 (data String) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE t_json_5 (data JSON) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE t_json_5 (data Object('json')) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_str_5 FORMAT JSONAsString {"k1": 1, "k2": {"k4": [22, 33]}}, {"k1": 2, "k2": {"k3": "qqq", "k4": [44]}}
|
||||
;
|
||||
|
||||
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_6;"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_6 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_6 (data Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_6 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_7;"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_7 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_7 (data Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_7 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_8"
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_8 (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_8 (data Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat <<EOF | $CLICKHOUSE_CLIENT -q "INSERT INTO t_json_8 FORMAT JSONAsObject"
|
||||
{
|
||||
|
||||
@ -4,7 +4,7 @@ DROP TABLE IF EXISTS t_json;
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
|
||||
CREATE TABLE t_json(id UInt64, obj JSON) ENGINE = MergeTree ORDER BY id;
|
||||
CREATE TABLE t_json(id UInt64, obj Object('json')) ENGINE = MergeTree ORDER BY id;
|
||||
|
||||
INSERT INTO t_json format JSONEachRow {"id": 1, "obj": {"foo": 1, "k1": 2}};
|
||||
INSERT INTO t_json format JSONEachRow {"id": 2, "obj": {"foo": 1, "k2": 2}};
|
||||
|
||||
@ -9,7 +9,7 @@ SET allow_experimental_object_type = 1;
|
||||
CREATE TABLE t_json_add_column (id UInt64) ENGINE = {{ storage }} ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_add_column VALUES (1);
|
||||
ALTER TABLE t_json_add_column ADD COLUMN s JSON;
|
||||
ALTER TABLE t_json_add_column ADD COLUMN s Object('json');
|
||||
|
||||
INSERT INTO t_json_add_column VALUES(2, '{"k1": 100}');
|
||||
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
DROP TABLE IF EXISTS t_json_bools;
|
||||
SET allow_experimental_object_type = 1;
|
||||
|
||||
CREATE TABLE t_json_bools (data JSON) ENGINE = Memory;
|
||||
CREATE TABLE t_json_bools (data Object('json')) ENGINE = Memory;
|
||||
INSERT INTO t_json_bools VALUES ('{"k1": true, "k2": false}');
|
||||
SELECT data, toTypeName(data) FROM t_json_bools;
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ cp $CUR_DIR/data_json/btc_transactions.json ${CLICKHOUSE_USER_FILES_UNIQUE}/
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS btc"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE btc (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE btc (data Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO btc SELECT * FROM file('${CLICKHOUSE_TEST_UNIQUE_NAME}/btc_transactions.json', 'JSONAsObject')"
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@ SET allow_experimental_object_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_desc;
|
||||
|
||||
CREATE TABLE t_json_desc (data JSON) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE t_json_desc (data Object('json')) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_desc FORMAT JSONAsObject {"k1": 10}
|
||||
;
|
||||
|
||||
@ -5,7 +5,7 @@ SET allow_experimental_object_type = 1;
|
||||
DROP TABLE IF EXISTS t_json_local;
|
||||
DROP TABLE IF EXISTS t_json_dist;
|
||||
|
||||
CREATE TABLE t_json_local(data JSON) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE t_json_local(data Object('json')) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE t_json_dist AS t_json_local ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), t_json_local);
|
||||
|
||||
INSERT INTO t_json_local FORMAT JSONAsObject {"k1": 2, "k2": {"k3": "qqq", "k4": [44, 55]}}
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
DROP TABLE IF EXISTS t_json_empty_str;
|
||||
SET allow_experimental_object_type = 1;
|
||||
|
||||
CREATE TABLE t_json_empty_str(id UInt32, o JSON) ENGINE = Memory;
|
||||
CREATE TABLE t_json_empty_str(id UInt32, o Object('json')) ENGINE = Memory;
|
||||
|
||||
INSERT INTO t_json_empty_str VALUES (1, ''), (2, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');
|
||||
|
||||
|
||||
@ -7,7 +7,7 @@ CREATE table t_github_json
|
||||
(
|
||||
event_type LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'type'),
|
||||
repo_name LowCardinality(String) DEFAULT JSONExtractString(message_raw, 'repo', 'name'),
|
||||
message JSON DEFAULT message_raw,
|
||||
message Object('json') DEFAULT message_raw,
|
||||
message_raw String EPHEMERAL
|
||||
) ENGINE = MergeTree ORDER BY (event_type, repo_name);
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ SET allow_experimental_object_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_field;
|
||||
|
||||
CREATE TABLE t_json_field (id UInt32, data JSON)
|
||||
CREATE TABLE t_json_field (id UInt32, data Object('json'))
|
||||
ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_field VALUES (1, (10, 'a')::Tuple(a UInt32, s String));
|
||||
|
||||
@ -7,7 +7,7 @@ DROP TABLE IF EXISTS t_map;
|
||||
SET allow_experimental_object_type = 1;
|
||||
SET optimize_trivial_insert_select = 1;
|
||||
|
||||
CREATE TABLE t_json(id UInt64, obj JSON) ENGINE = MergeTree ORDER BY id;
|
||||
CREATE TABLE t_json(id UInt64, obj Object('json')) ENGINE = MergeTree ORDER BY id;
|
||||
CREATE TABLE t_map(id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id;
|
||||
|
||||
INSERT INTO t_map
|
||||
|
||||
@ -9,9 +9,9 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2_string"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ghdata_2_from_string"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2 (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2 (data Object('json')) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2_string (data String) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2_from_string (data JSON) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE ghdata_2_from_string (data Object('json')) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi'" --allow_experimental_object_type 1
|
||||
|
||||
cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata_2 FORMAT JSONAsObject"
|
||||
cat $CUR_DIR/data_json/ghdata_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO ghdata_2_string FORMAT JSONAsString"
|
||||
|
||||
@ -3,7 +3,7 @@
|
||||
SET allow_experimental_object_type = 1;
|
||||
DROP TABLE IF EXISTS t_json_array;
|
||||
|
||||
CREATE TABLE t_json_array (id UInt32, arr Array(JSON)) ENGINE = MergeTree ORDER BY id;
|
||||
CREATE TABLE t_json_array (id UInt32, arr Array(Object('json'))) ENGINE = MergeTree ORDER BY id;
|
||||
|
||||
INSERT INTO t_json_array FORMAT JSONEachRow {"id": 1, "arr": [{"k1": 1, "k2": {"k3": 2, "k4": 3}}, {"k1": 2, "k2": {"k5": "foo"}}]}
|
||||
INSERT INTO t_json_array FORMAT JSONEachRow {"id": 2, "arr": [{"k1": 3, "k2": {"k3": 4, "k4": 5}}]}
|
||||
@ -28,8 +28,8 @@ SELECT toTypeName(arrayJoin(arrayJoin(arr.k1))) AS arr FROM t_json_array LIMIT 1
|
||||
|
||||
DROP TABLE t_json_array;
|
||||
|
||||
SELECT * FROM values('arr Array(JSON)', '[\'{"x" : 1}\']') FORMAT JSONEachRow;
|
||||
SELECT * FROM values('arr Map(String, JSON)', '{\'x\' : \'{"y" : 1}\', \'t\' : \'{"y" : 2}\'}') FORMAT JSONEachRow;
|
||||
SELECT * FROM values('arr Tuple(Int32, JSON)', '(1, \'{"y" : 1}\')', '(2, \'{"y" : 2}\')') FORMAT JSONEachRow;
|
||||
SELECT * FROM values('arr Array(Object(''json''))', '[\'{"x" : 1}\']') FORMAT JSONEachRow;
|
||||
SELECT * FROM values('arr Map(String, Object(''json''))', '{\'x\' : \'{"y" : 1}\', \'t\' : \'{"y" : 2}\'}') FORMAT JSONEachRow;
|
||||
SELECT * FROM values('arr Tuple(Int32, Object(''json''))', '(1, \'{"y" : 1}\')', '(2, \'{"y" : 2}\')') FORMAT JSONEachRow;
|
||||
SELECT * FROM format(JSONEachRow, '{"arr" : [{"x" : "aaa", "y" : [1,2,3]}]}') FORMAT JSONEachRow;
|
||||
SELECT * FROM values('arr Array(JSON)', '[\'{"x" : 1}\']') FORMAT JSONEachRow;
|
||||
SELECT * FROM values('arr Array(Object(''json''))', '[\'{"x" : 1}\']') FORMAT JSONEachRow;
|
||||
|
||||
@ -12,7 +12,7 @@ ${CLICKHOUSE_CLIENT} -q "
|
||||
CREATE TABLE t_json_nested
|
||||
(
|
||||
id UInt32,
|
||||
data Tuple(String, Map(String, Array(JSON)), JSON)
|
||||
data Tuple(String, Map(String, Array(Object('json'))), Object('json'))
|
||||
)
|
||||
ENGINE = MergeTree ORDER BY id" --allow_experimental_object_type 1
|
||||
|
||||
|
||||
@ -5,7 +5,7 @@ SET allow_experimental_object_type = 1;
|
||||
DROP TABLE IF EXISTS type_json_src;
|
||||
DROP TABLE IF EXISTS type_json_dst;
|
||||
|
||||
CREATE TABLE type_json_src (id UInt32, data JSON) ENGINE = MergeTree ORDER BY id;
|
||||
CREATE TABLE type_json_src (id UInt32, data Object('json')) ENGINE = MergeTree ORDER BY id;
|
||||
CREATE TABLE type_json_dst AS type_json_src;
|
||||
|
||||
INSERT INTO type_json_src VALUES (1, '{"k1": 1, "k2": "foo"}');
|
||||
@ -35,7 +35,7 @@ SELECT id, data FROM type_json_dst ORDER BY id;
|
||||
DROP TABLE type_json_src;
|
||||
DROP TABLE type_json_dst;
|
||||
|
||||
CREATE TABLE type_json_dst (data JSON) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE type_json_dst (data Object('json')) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE type_json_src (data String) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
SYSTEM STOP MERGES type_json_src;
|
||||
|
||||
@ -4,7 +4,7 @@ DROP TABLE IF EXISTS t_json;
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
|
||||
CREATE TABLE t_json(id UInt64, obj JSON)
|
||||
CREATE TABLE t_json(id UInt64, obj Object('json'))
|
||||
ENGINE = MergeTree ORDER BY id
|
||||
SETTINGS min_bytes_for_wide_part = 0;
|
||||
|
||||
|
||||
@ -14,9 +14,9 @@ for i in {0..5}; do
|
||||
done
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_json_files"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_files (file String, data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_files (file String, data Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_json_files SELECT _file, data FROM file('${CLICKHOUSE_DATABASE}_*.json', 'JSONAsObject', 'data JSON')" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_json_files SELECT _file, data FROM file('${CLICKHOUSE_DATABASE}_*.json', 'JSONAsObject', 'data Object(''json'')')" --allow_experimental_object_type 1
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT data FROM t_json_files ORDER BY file FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT toTypeName(data) FROM t_json_files LIMIT 1"
|
||||
@ -24,7 +24,7 @@ ${CLICKHOUSE_CLIENT} -q "SELECT toTypeName(data) FROM t_json_files LIMIT 1"
|
||||
${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE IF EXISTS t_json_files"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_json_files \
|
||||
SELECT _file, data FROM file('${CLICKHOUSE_DATABASE}_*.json', 'JSONAsObject', 'data JSON') \
|
||||
SELECT _file, data FROM file('${CLICKHOUSE_DATABASE}_*.json', 'JSONAsObject', 'data Object(''json'')') \
|
||||
ORDER BY _file LIMIT 3" --max_threads 1 --min_insert_block_size_rows 1 --max_insert_block_size 1 --max_block_size 1 --allow_experimental_object_type 1
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT data FROM t_json_files ORDER BY file, data FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1
|
||||
@ -33,7 +33,7 @@ ${CLICKHOUSE_CLIENT} -q "SELECT toTypeName(data) FROM t_json_files LIMIT 1"
|
||||
${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE IF EXISTS t_json_files"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_json_files \
|
||||
SELECT _file, data FROM file('${CLICKHOUSE_DATABASE}_*.json', 'JSONAsObject', 'data JSON') \
|
||||
SELECT _file, data FROM file('${CLICKHOUSE_DATABASE}_*.json', 'JSONAsObject', 'data Object(''json'')') \
|
||||
WHERE _file IN ('${CLICKHOUSE_DATABASE}_1.json', '${CLICKHOUSE_DATABASE}_3.json')" --allow_experimental_object_type 1
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT data FROM t_json_files ORDER BY file FORMAT JSONEachRow" --output_format_json_named_tuples_as_objects 1
|
||||
|
||||
@ -6,7 +6,7 @@ SET allow_experimental_object_type = 1;
|
||||
SET output_format_json_named_tuples_as_objects = 1;
|
||||
SET mutations_sync = 2;
|
||||
|
||||
CREATE TABLE t_json_mutations(id UInt32, s String, obj JSON) ENGINE = MergeTree ORDER BY id;
|
||||
CREATE TABLE t_json_mutations(id UInt32, s String, obj Object('json')) ENGINE = MergeTree ORDER BY id;
|
||||
|
||||
INSERT INTO t_json_mutations VALUES (1, 'q', '{"k1": 1, "k2": 2, "k3": [{"k4": "aaa"}, {"k4": "bbb"}]}');
|
||||
INSERT INTO t_json_mutations VALUES (2, 'w', '{"k1": 3, "k2": 4, "k3": [{"k4": "ccc"}]}');
|
||||
|
||||
@ -9,7 +9,7 @@ ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames_string"
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS nbagames_from_string"
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE nbagames (data Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
cat $CUR_DIR/data_json/nbagames_sample.json | ${CLICKHOUSE_CLIENT} -q "INSERT INTO nbagames FORMAT JSONAsObject"
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
SELECT dummy FROM system.one ORDER BY materialize('{"k":"v"}'::JSON);
|
||||
SELECT dummy FROM system.one ORDER BY materialize('{"k":"v"}'::JSON), dummy;
|
||||
SELECT materialize('{"k":"v"}'::JSON) SETTINGS extremes = 1;
|
||||
SELECT dummy FROM system.one ORDER BY materialize('{"k":"v"}'::Object('json'));
|
||||
SELECT dummy FROM system.one ORDER BY materialize('{"k":"v"}'::Object('json')), dummy;
|
||||
SELECT materialize('{"k":"v"}'::Object('json')) SETTINGS extremes = 1;
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
DROP TABLE IF EXISTS t_json_parallel;
|
||||
|
||||
SET allow_experimental_object_type = 1, max_insert_threads = 20, max_threads = 20, min_insert_block_size_rows = 65536;
|
||||
CREATE TABLE t_json_parallel (data JSON) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE t_json_parallel (data Object('json')) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_parallel SELECT materialize('{"k1":1, "k2": "some"}') FROM numbers_mt(500000);
|
||||
SELECT any(toTypeName(data)), count() FROM t_json_parallel;
|
||||
|
||||
@ -5,7 +5,7 @@ DROP TABLE IF EXISTS t_json_partitions;
|
||||
SET allow_experimental_object_type = 1;
|
||||
SET output_format_json_named_tuples_as_objects = 1;
|
||||
|
||||
CREATE TABLE t_json_partitions (id UInt32, obj JSON)
|
||||
CREATE TABLE t_json_partitions (id UInt32, obj Object('json'))
|
||||
ENGINE MergeTree ORDER BY id PARTITION BY id;
|
||||
|
||||
INSERT INTO t_json_partitions FORMAT JSONEachRow {"id": 1, "obj": {"k1": "v1"}} {"id": 2, "obj": {"k2": "v2"}};
|
||||
|
||||
@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_race"
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_race (data JSON) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
$CLICKHOUSE_CLIENT -q "CREATE TABLE t_json_race (data Object('json')) ENGINE = MergeTree ORDER BY tuple()" --allow_experimental_object_type 1
|
||||
|
||||
function test_case()
|
||||
{
|
||||
|
||||
@ -5,7 +5,7 @@ DROP TABLE IF EXISTS t_json_sparse;
|
||||
SET allow_experimental_object_type = 1;
|
||||
SET optimize_trivial_insert_select = 1;
|
||||
|
||||
CREATE TABLE t_json_sparse (data JSON)
|
||||
CREATE TABLE t_json_sparse (data Object('json'))
|
||||
ENGINE = MergeTree ORDER BY tuple()
|
||||
SETTINGS ratio_of_defaults_for_sparse_serialization = 0.1,
|
||||
min_bytes_for_wide_part = 0, index_granularity = 8192, index_granularity_bytes = '10Mi';
|
||||
|
||||
@ -5,7 +5,7 @@ SET output_format_json_named_tuples_as_objects = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_wide_parts;
|
||||
|
||||
CREATE TABLE t_json_wide_parts (data JSON)
|
||||
CREATE TABLE t_json_wide_parts (data Object('json'))
|
||||
ENGINE MergeTree ORDER BY tuple()
|
||||
SETTINGS min_bytes_for_wide_part = 0;
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@ DROP TABLE IF EXISTS t_subcolumns_sizes;
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
|
||||
CREATE TABLE t_subcolumns_sizes (id UInt64, arr Array(UInt64), n Nullable(String), d JSON)
|
||||
CREATE TABLE t_subcolumns_sizes (id UInt64, arr Array(UInt64), n Nullable(String), d Object('json'))
|
||||
ENGINE = MergeTree ORDER BY id
|
||||
SETTINGS min_bytes_for_wide_part = 0;
|
||||
|
||||
|
||||
@ -12,7 +12,7 @@ INSERT INTO t_flatten_tuple VALUES (([(1, 'a'), (2, 'b')], 3, ('c', 4)));
|
||||
SELECT flattenTuple(t) AS ft, toTypeName(ft) FROM t_flatten_tuple;
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
CREATE TABLE t_flatten_object(data JSON) ENGINE = Memory;
|
||||
CREATE TABLE t_flatten_object(data Object('json')) ENGINE = Memory;
|
||||
|
||||
INSERT INTO t_flatten_object VALUES ('{"id": 1, "obj": {"k1": 1, "k2": {"k3": 2, "k4": [{"k5": 3}, {"k5": 4}]}}, "s": "foo"}');
|
||||
INSERT INTO t_flatten_object VALUES ('{"id": 2, "obj": {"k2": {"k3": "str", "k4": [{"k6": 55}]}, "some": 42}, "s": "bar"}');
|
||||
|
||||
@ -28,7 +28,7 @@ SELECT `t`.`1`.`1`, `t`.`1`.`2`, `t`.`2` FROM t_tuple_numeric;
|
||||
DROP TABLE t_tuple_numeric;
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
CREATE TABLE t_tuple_numeric (t JSON) ENGINE = Memory;
|
||||
CREATE TABLE t_tuple_numeric (t Object('json')) ENGINE = Memory;
|
||||
INSERT INTO t_tuple_numeric FORMAT JSONEachRow {"t":{"1":{"2":2,"3":3},"4":4}}
|
||||
|
||||
SELECT toTypeName(t) FROM t_tuple_numeric LIMIT 1;
|
||||
|
||||
@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_json_async_insert"
|
||||
$CLICKHOUSE_CLIENT --allow_experimental_object_type=1 -q "CREATE TABLE t_json_async_insert (data JSON) ENGINE = MergeTree ORDER BY tuple()"
|
||||
$CLICKHOUSE_CLIENT --allow_experimental_object_type=1 -q "CREATE TABLE t_json_async_insert (data Object(''json'')) ENGINE = MergeTree ORDER BY tuple()"
|
||||
|
||||
$CLICKHOUSE_CLIENT --async_insert=1 --wait_for_async_insert=1 -q 'INSERT INTO t_json_async_insert FORMAT JSONAsObject {"aaa"}' 2>&1 | grep -o -m1 "Cannot parse object"
|
||||
$CLICKHOUSE_CLIENT -q "SELECT count() FROM t_json_async_insert"
|
||||
|
||||
@ -11,7 +11,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_json_empty_parts;"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT 'Collapsing';"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_empty_parts (id UInt64, s Int8, data JSON) ENGINE = CollapsingMergeTree(s) ORDER BY id SETTINGS old_parts_lifetime=5;" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_empty_parts (id UInt64, s Int8, data Object('json')) ENGINE = CollapsingMergeTree(s) ORDER BY id SETTINGS old_parts_lifetime=5;" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_json_empty_parts VALUES (1, 1, '{\"k1\": \"aaa\"}') (1, -1, '{\"k2\": \"bbb\"}');"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_json_empty_parts;"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active;"
|
||||
@ -19,7 +19,7 @@ ${CLICKHOUSE_CLIENT} -q "DESC TABLE t_json_empty_parts SETTINGS describe_extend_
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_json_empty_parts;"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT 'DELETE all';"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_empty_parts (id UInt64, data JSON) ENGINE = MergeTree ORDER BY id SETTINGS old_parts_lifetime=5;" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_empty_parts (id UInt64, data Object('json')) ENGINE = MergeTree ORDER BY id SETTINGS old_parts_lifetime=5;" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_json_empty_parts VALUES (1, '{\"k1\": \"aaa\"}') (1, '{\"k2\": \"bbb\"}');"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_json_empty_parts;"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.parts WHERE table = 't_json_empty_parts' AND database = currentDatabase() AND active;"
|
||||
@ -32,7 +32,7 @@ ${CLICKHOUSE_CLIENT} -q "DESC TABLE t_json_empty_parts SETTINGS describe_extend_
|
||||
|
||||
${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_json_empty_parts;"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT 'TTL';"
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_empty_parts (id UInt64, d Date, data JSON) ENGINE = MergeTree ORDER BY id TTL d WHERE id % 2 = 1 SETTINGS old_parts_lifetime=5;" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_json_empty_parts (id UInt64, d Date, data Object('json')) ENGINE = MergeTree ORDER BY id TTL d WHERE id % 2 = 1 SETTINGS old_parts_lifetime=5;" --allow_experimental_object_type 1
|
||||
${CLICKHOUSE_CLIENT} -q "INSERT INTO t_json_empty_parts VALUES (1, '2000-01-01', '{\"k1\": \"aaa\"}') (2, '2000-01-01', '{\"k2\": \"bbb\"}');"
|
||||
${CLICKHOUSE_CLIENT} -q "OPTIMIZE TABLE t_json_empty_parts FINAL;"
|
||||
${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_json_empty_parts;"
|
||||
|
||||
@ -21,7 +21,7 @@ echo '
|
||||
}
|
||||
}' > 02482_object_data.jsonl
|
||||
|
||||
$CLICKHOUSE_LOCAL --allow_experimental_object_type=1 -q "select * from file(02482_object_data.jsonl, auto, 'obj JSON')"
|
||||
$CLICKHOUSE_LOCAL --allow_experimental_object_type=1 -q "select * from file(02482_object_data.jsonl, auto, 'obj Object('json')')"
|
||||
|
||||
rm 02482_object_data.jsonl
|
||||
|
||||
|
||||
@ -1,9 +1,9 @@
|
||||
-- Tags: no-fasttest
|
||||
|
||||
set allow_experimental_object_type=0;
|
||||
select CAST('{"x" : 1}', 'JSON'); -- {serverError ILLEGAL_COLUMN}
|
||||
select CAST('{"x" : 1}', 'Object(''json'')'); -- {serverError ILLEGAL_COLUMN}
|
||||
desc file(nonexist.json, JSONAsObject); -- {serverError ILLEGAL_COLUMN}
|
||||
desc file(nonexist.json, JSONEachRow, 'x JSON'); -- {serverError ILLEGAL_COLUMN}
|
||||
desc file(nonexist.json, JSONEachRow, 'x Object(''json'')'); -- {serverError ILLEGAL_COLUMN}
|
||||
|
||||
set allow_suspicious_low_cardinality_types=0;
|
||||
select CAST(1000000, 'LowCardinality(UInt64)'); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY}
|
||||
|
||||
@ -2,7 +2,7 @@ SET allow_experimental_object_type = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_attach_partition;
|
||||
|
||||
CREATE TABLE t_json_attach_partition(b UInt64, c JSON) ENGINE = MergeTree ORDER BY tuple();
|
||||
CREATE TABLE t_json_attach_partition(b UInt64, c Object('json')) ENGINE = MergeTree ORDER BY tuple();
|
||||
|
||||
INSERT INTO t_json_attach_partition FORMAT JSONEachRow {"b": 1, "c" : {"k1": 1}};
|
||||
ALTER TABLE t_json_attach_partition DETACH PARTITION tuple();
|
||||
|
||||
@ -3,7 +3,7 @@ SET allow_experimental_object_type = 1;
|
||||
SET allow_experimental_analyzer = 1;
|
||||
|
||||
DROP TABLE IF EXISTS t_json_analyzer;
|
||||
CREATE TABLE t_json_analyzer (a JSON) ENGINE = Memory;
|
||||
CREATE TABLE t_json_analyzer (a Object('json')) ENGINE = Memory;
|
||||
INSERT INTO t_json_analyzer VALUES ('{"id": 2, "obj": {"k2": {"k3": "str", "k4": [{"k6": 55}]}, "some": 42}, "s": "bar"}');
|
||||
|
||||
SELECT any(a) AS data FROM t_json_analyzer FORMAT JSONEachRow;
|
||||
|
||||
@ -1,3 +1,3 @@
|
||||
set allow_experimental_object_type=1;
|
||||
select 42 as num, [42, 42] as arr, [[[42, 42], [42, 42]], [[42, 42]]] as nested_arr, tuple(42, 42)::Tuple(a UInt32, b UInt32) as tuple, tuple(tuple(tuple(42, 42), 42), 42)::Tuple(a Tuple(b Tuple(c UInt32, d UInt32), e UInt32), f UInt32) as nested_tuple, map(42, 42, 24, 24) as map, map(42, map(42, map(42, 42))) as nested_map, [tuple(map(42, 42), [42, 42]), tuple(map(42, 42), [42, 42])]::Array(Tuple(Map(UInt32, UInt32), Array(UInt32))) as nested_types, '{"a" : {"b" : 1, "c" : 2}}'::JSON as json_object format PrettyNDJSON;
|
||||
select 42 as num, [42, 42] as arr, [[[42, 42], [42, 42]], [[42, 42]]] as nested_arr, tuple(42, 42)::Tuple(a UInt32, b UInt32) as tuple, tuple(tuple(tuple(42, 42), 42), 42)::Tuple(a Tuple(b Tuple(c UInt32, d UInt32), e UInt32), f UInt32) as nested_tuple, map(42, 42, 24, 24) as map, map(42, map(42, map(42, 42))) as nested_map, [tuple(map(42, 42), [42, 42]), tuple(map(42, 42), [42, 42])]::Array(Tuple(Map(UInt32, UInt32), Array(UInt32))) as nested_types, '{"a" : {"b" : 1, "c" : 2}}'::Object('json') as json_object format PrettyNDJSON;
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
DROP TABLE IF EXISTS tab;
|
||||
|
||||
SET allow_suspicious_low_cardinality_types=1;
|
||||
SET allow_experimental_object_type=1;
|
||||
SET allow_experimental_json_type=1;
|
||||
|
||||
CREATE TABLE tab
|
||||
(
|
||||
|
||||
@ -33,7 +33,7 @@ send -- "DROP TABLE IF EXISTS tab;\r"
|
||||
expect "Query OK, 0 rows affected"
|
||||
|
||||
send -- "SET allow_suspicious_low_cardinality_types=1;\r"
|
||||
send -- "SET allow_experimental_object_type=1;\r"
|
||||
send -- "SET allow_experimental_json_type=1;\r"
|
||||
|
||||
send -- "
|
||||
CREATE TABLE tab
|
||||
|
||||
@ -49,7 +49,7 @@ CREATE TABLE tab
|
||||
(
|
||||
id UInt64,
|
||||
tup Tuple(UInt64, UInt64) SETTINGS (min_compress_block_size = 81920, max_compress_block_size = 163840),
|
||||
json JSON SETTINGS (min_compress_block_size = 81920, max_compress_block_size = 163840),
|
||||
json Object('json') SETTINGS (min_compress_block_size = 81920, max_compress_block_size = 163840),
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
ORDER BY id
|
||||
|
||||
@ -34,6 +34,7 @@ The answer to all questions is 2023-11-14 05:50:12.123.
|
||||
The answer to all questions is hallo.
|
||||
The answer to all questions is [\'foo\',\'bar\'].
|
||||
The answer to all questions is {"foo":"bar"}.
|
||||
The answer to all questions is {"foo":"bar"}.
|
||||
The answer to all questions is (42,\'foo\').
|
||||
The answer to all questions is {42:\'foo\'}.
|
||||
The answer to all questions is 122.233.64.201.
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
-- no-fasttest: json type needs rapidjson library, geo types need s2 geometry
|
||||
|
||||
SET allow_experimental_object_type = 1;
|
||||
SET allow_experimental_json_type = 1;
|
||||
SET allow_suspicious_low_cardinality_types=1;
|
||||
|
||||
SELECT '-- Const string + non-const arbitrary type';
|
||||
@ -40,6 +41,7 @@ SELECT format('The {0} to all questions is {1}.', 'answer', materialize('2023-11
|
||||
SELECT format('The {0} to all questions is {1}.', 'answer', materialize('2023-11-14 05:50:12.123' :: DateTime64(3, 'Europe/Amsterdam')));
|
||||
SELECT format('The {0} to all questions is {1}.', 'answer', materialize('hallo' :: Enum('hallo' = 1)));
|
||||
SELECT format('The {0} to all questions is {1}.', 'answer', materialize(['foo', 'bar'] :: Array(String)));
|
||||
SELECT format('The {0} to all questions is {1}.', 'answer', materialize('{"foo": "bar"}' :: Object('json')));
|
||||
SELECT format('The {0} to all questions is {1}.', 'answer', materialize('{"foo": "bar"}' :: JSON));
|
||||
SELECT format('The {0} to all questions is {1}.', 'answer', materialize((42, 'foo') :: Tuple(Int32, String)));
|
||||
SELECT format('The {0} to all questions is {1}.', 'answer', materialize(map(42, 'foo') :: Map(Int32, String)));
|
||||
|
||||
@ -7,7 +7,7 @@ Decimal 45 Decimal(10, 0)
|
||||
Decimal(M) 46 Decimal(4, 0)
|
||||
Decimal(M, D) 47.21 Decimal(4, 2)
|
||||
Double 48.11 Float64
|
||||
JSON {"foo":"bar"} Object(\'json\')
|
||||
JSON {"foo":"bar"} JSON
|
||||
Real 49.22 Float32
|
||||
Signed 50 Int64
|
||||
Unsigned 52 UInt64
|
||||
@ -21,7 +21,7 @@ Decimal 45 Decimal(10, 0)
|
||||
Decimal(M) 46 Decimal(4, 0)
|
||||
Decimal(M, D) 47.21 Decimal(4, 2)
|
||||
Double 48.11 Float64
|
||||
JSON {"foo":"bar"} Object(\'json\')
|
||||
JSON {"foo":"bar"} JSON
|
||||
Real 49.22 Float32
|
||||
Signed 50 Int64
|
||||
Unsigned 52 UInt64
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user