mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-20 00:30:49 +00:00
Compare commits
464 Commits
8b7d967546
...
e4b2493855
Author | SHA1 | Date | |
---|---|---|---|
|
e4b2493855 | ||
|
2c29b3d98c | ||
|
61524aabb6 | ||
|
39c95f6c73 | ||
|
dfdc25acc9 | ||
|
3efe136635 | ||
|
efc0cec707 | ||
|
396abf7636 | ||
|
f8fb4fb120 | ||
|
1c6165f6ee | ||
|
71dd3d5cf6 | ||
|
cb503ec2ec | ||
|
c0c83236b6 | ||
|
14823f789b | ||
|
3eb5bc1a0f | ||
|
bb6db8926e | ||
|
19353a74db | ||
|
228ac44a92 | ||
|
d2de15871c | ||
|
b94a7167a8 | ||
|
0fdd04254d | ||
|
b88cd79959 | ||
|
818aac02c6 | ||
|
64e58baba1 | ||
|
7f0b7a9158 | ||
|
006d14445e | ||
|
b5de8e622d | ||
|
a3fe155579 | ||
|
373927d6a5 | ||
|
02fcd90a66 | ||
|
e818b65dc0 | ||
|
a997cfad2b | ||
|
f4b4b3cc35 | ||
|
3315e87e1a | ||
|
e0fc95c894 | ||
|
b940171252 | ||
|
cb24849396 | ||
|
e3b207d217 | ||
|
4be8a0feba | ||
|
4c72fb0e32 | ||
|
8f350a7ec9 | ||
|
7fd2207626 | ||
|
4f73c677ac | ||
|
69f45acfd7 | ||
|
4c78206d0a | ||
|
429e8ada79 | ||
|
06b49d18d9 | ||
|
a17a8febf7 | ||
|
55529ec5a2 | ||
|
143d9f0201 | ||
|
3106653852 | ||
|
82dbb3bb32 | ||
|
2218ebebbf | ||
|
1bcdde3e62 | ||
|
2cef99c311 | ||
|
cd7a1a9288 | ||
|
6597a8ed04 | ||
|
3b901f49e5 | ||
|
958c3effae | ||
|
474499d240 | ||
|
839f06035f | ||
|
4f88ccb6a8 | ||
|
a226567bc2 | ||
|
fcda762a27 | ||
|
9c185374e4 | ||
|
13e82d6439 | ||
|
fdee35cccc | ||
|
b08e727aef | ||
|
9eba103c5e | ||
|
f52cdfb795 | ||
|
a210f98819 | ||
|
e574c49e25 | ||
|
7c5d55c6b2 | ||
|
665f362601 | ||
|
80259659ff | ||
|
574a26c63b | ||
|
3674c97ebb | ||
|
8508b1ba37 | ||
|
190d3f04c9 | ||
|
aba7de5091 | ||
|
d11abd634a | ||
|
8db3dddb3d | ||
|
ffaf97a390 | ||
|
3a7c68a052 | ||
|
452fde78c7 | ||
|
51fa9ebf8a | ||
|
e30ebfa23e | ||
|
b21be2bc54 | ||
|
14736d95c5 | ||
|
e8d50aa97f | ||
|
4b69d8e2ca | ||
|
5ce8604869 | ||
|
813bcd896f | ||
|
3a05282bce | ||
|
fd0c7a1c18 | ||
|
88b22094c8 | ||
|
3cb8160240 | ||
|
4704fb8a3b | ||
|
0369aaea87 | ||
|
3f663f8e09 | ||
|
f768717be8 | ||
|
64106e7b3c | ||
|
79223045c9 | ||
|
983b061b58 | ||
|
3a299f382d | ||
|
f8f72ccb00 | ||
|
1ccd461c97 | ||
|
de308acfad | ||
|
cb92aaf968 | ||
|
5aaff37b36 | ||
|
386d54cedf | ||
|
2c1c1a93c6 | ||
|
9f932fb453 | ||
|
f3654b8fc8 | ||
|
676b6238d0 | ||
|
e876997ebb | ||
|
e7eaa01bb3 | ||
|
52dc9a54a7 | ||
|
b4a6d41b52 | ||
|
a329150eef | ||
|
7b94dc1813 | ||
|
8c7c37de1d | ||
|
6a26c5cf8e | ||
|
0cdec0acf1 | ||
|
19e2197582 | ||
|
04f23332c3 | ||
|
d223c4547f | ||
|
58993d3f3b | ||
|
dc02b168a0 | ||
|
8507d209c0 | ||
|
f5b9d5ad34 | ||
|
4af369fbc4 | ||
|
8cdcc431fe | ||
|
f401eccc64 | ||
|
6863dc7647 | ||
|
1963e971f3 | ||
|
056c7af356 | ||
|
cf9200f1d0 | ||
|
187a717872 | ||
|
5f464a6d74 | ||
|
4412946532 | ||
|
03737ddcab | ||
|
038f56cb5e | ||
|
63577507c9 | ||
|
9eb78773a6 | ||
|
6f63a7b213 | ||
|
56cfa74a14 | ||
|
dbb1d043fe | ||
|
7d5203f8a7 | ||
|
0d1d750437 | ||
|
ad31d86a15 | ||
|
56f3030b17 | ||
|
991279e5c6 | ||
|
31ddfc6f5f | ||
|
ddf2e07fd0 | ||
|
5cc12ca9ee | ||
|
e13247b67e | ||
|
c184aae686 | ||
|
14a6b0422b | ||
|
2650a20628 | ||
|
9a31fc385d | ||
|
ddc506a677 | ||
|
2812953a8a | ||
|
492461271b | ||
|
3c47f3df4b | ||
|
54e75fd4dc | ||
|
ae6e236acf | ||
|
11c7cdabf8 | ||
|
71553022e0 | ||
|
53e1975833 | ||
|
8299b31d47 | ||
|
97c8d2897c | ||
|
2c165096cd | ||
|
11d2963497 | ||
|
f9335a2fd5 | ||
|
8a89d7b2b9 | ||
|
aab0d3dd9e | ||
|
5a34b9f24e | ||
|
a0a4858e00 | ||
|
9c1f4f4545 | ||
|
2e82e06330 | ||
|
4963ab603c | ||
|
7b2810bea2 | ||
|
e2bfce66dd | ||
|
0bb3967d14 | ||
|
401a3d0931 | ||
|
51d770fa7a | ||
|
beffb92411 | ||
|
877002f689 | ||
|
16f93ea1b3 | ||
|
1e3bc6d359 | ||
|
bde54b96f7 | ||
|
4a9b376e2a | ||
|
562c23eac6 | ||
|
8d5babf65f | ||
|
99ede620be | ||
|
92351a67e8 | ||
|
f292767778 | ||
|
7d36f3b764 | ||
|
21bd47f09e | ||
|
fc83c1c7a2 | ||
|
f378047f30 | ||
|
e8cec05d08 | ||
|
2876a4e714 | ||
|
0de3b1dacb | ||
|
8d5d7dd83a | ||
|
35df5ff28e | ||
|
61ebcdc2ed | ||
|
1df897db27 | ||
|
8cdc10cf65 | ||
|
4ece895b41 | ||
|
65019c4b9b | ||
|
190339c4e6 | ||
|
5a86371b02 | ||
|
f5d49f8e10 | ||
|
a903e1a726 | ||
|
ee304c7fc3 | ||
|
03c7f3817b | ||
|
fdbf8e71ab | ||
|
f44eaa808d | ||
|
e388f6f99b | ||
|
92507d9938 | ||
|
6170c15c90 | ||
|
d23145fd19 | ||
|
fb8999a885 | ||
|
03ac70f988 | ||
|
5f5acd3c44 | ||
|
2fa6be55ff | ||
|
a3e233a537 | ||
|
8896d1b78b | ||
|
f495a4f431 | ||
|
955412888c | ||
|
9633563fbd | ||
|
d9a4964cd9 | ||
|
a44b3d0268 | ||
|
83854cf293 | ||
|
e874c6e1de | ||
|
f1377b0b4a | ||
|
f688b903db | ||
|
21f9669836 | ||
|
1a386ae4d5 | ||
|
24f4e87f8b | ||
|
03ccf05d14 | ||
|
79fc8d67ad | ||
|
0bf95655aa | ||
|
15a67f10dc | ||
|
596ba574e3 | ||
|
e968984d17 | ||
|
a22f9fd91f | ||
|
7c766e7458 | ||
|
620640a042 | ||
|
ec469a117d | ||
|
7a879980d8 | ||
|
f9b845486a | ||
|
2adc61c215 | ||
|
7000f214ab | ||
|
afc4d08aad | ||
|
3d04f3d33a | ||
|
edc5d8dd92 | ||
|
d6b2a9d534 | ||
|
cb6d142947 | ||
|
a690539935 | ||
|
dc97bd6b92 | ||
|
b2795f06dc | ||
|
7879915493 | ||
|
313b6b533f | ||
|
9f7f6c6f93 | ||
|
553c309477 | ||
|
ae582120ae | ||
|
b29e00b838 | ||
|
ae8d90f6b8 | ||
|
41a4a97ca3 | ||
|
50b3d3172c | ||
|
c298b20ba9 | ||
|
0b29aef1a0 | ||
|
60c6eb2610 | ||
|
9133505952 | ||
|
2741bf00e4 | ||
|
4eca00a666 | ||
|
a0ab22e031 | ||
|
6f806124a3 | ||
|
a65d175a81 | ||
|
c6804122cb | ||
|
189cbe25fe | ||
|
aa4688a982 | ||
|
7aaa0289e1 | ||
|
d6df83d561 | ||
|
1011f8ef9c | ||
|
cb7ef910e2 | ||
|
1c6976d7a5 | ||
|
43e9a7ba4b | ||
|
62ce2999ae | ||
|
91cceccb4b | ||
|
13d5b029a4 | ||
|
ca40da5c03 | ||
|
b0a0988c5b | ||
|
d7d40db036 | ||
|
4fd19c3ad9 | ||
|
f0223aedde | ||
|
1980959c8b | ||
|
27ee4dd611 | ||
|
21e64f2aa9 | ||
|
03ab625265 | ||
|
dbd4ee44ed | ||
|
623c507e5f | ||
|
01f5337f69 | ||
|
d4a3a033b0 | ||
|
165d08f088 | ||
|
e0dbc53b58 | ||
|
f97551e2ad | ||
|
c252b3c8b0 | ||
|
30229a3bfd | ||
|
8a0f41da7a | ||
|
628a4300ba | ||
|
f88b5988c1 | ||
|
4bb2f7b3f6 | ||
|
95edca513c | ||
|
5004e4d2cc | ||
|
accade2390 | ||
|
e7fc89ba26 | ||
|
770804ffdc | ||
|
49ce2c7619 | ||
|
80d985a690 | ||
|
891f9c5358 | ||
|
cb0335446e | ||
|
70e7c4e63d | ||
|
12e6645058 | ||
|
714a4d871c | ||
|
72f1695014 | ||
|
17c1cef52b | ||
|
3247f3ad08 | ||
|
026fa0a7fd | ||
|
64e10b2dda | ||
|
e8a40d9d52 | ||
|
9abc001296 | ||
|
c61fc591c4 | ||
|
dcbc590302 | ||
|
4c6aca2eed | ||
|
64d50d6e5b | ||
|
b6c3619543 | ||
|
b2172af817 | ||
|
ef40cc3bae | ||
|
f5c07b8938 | ||
|
5ea4844d69 | ||
|
fa6564dbe3 | ||
|
ae72bd57f2 | ||
|
c433d9cfdb | ||
|
f5ee7aaf26 | ||
|
352b502559 | ||
|
546cca1251 | ||
|
456613e7fa | ||
|
5a45563f1b | ||
|
a21529f66a | ||
|
921947e368 | ||
|
93cbd4bf9a | ||
|
77d46aad05 | ||
|
0404a8e800 | ||
|
8eda32600f | ||
|
f3f9d5f4de | ||
|
0204de640f | ||
|
0b151bbe8f | ||
|
48e7057200 | ||
|
f2c22408da | ||
|
006f20858a | ||
|
cc02ebca75 | ||
|
3ab6760412 | ||
|
ca0b821aaf | ||
|
2e9b7e8334 | ||
|
b428327c6e | ||
|
8e0e2cec28 | ||
|
300d4ae593 | ||
|
ee62318348 | ||
|
29e0d5c1e3 | ||
|
91e8ef6776 | ||
|
5a96290cce | ||
|
acc2249288 | ||
|
e36776551e | ||
|
cd0145113f | ||
|
5a2b0ea1cc | ||
|
cc13783acd | ||
|
7e22af06f1 | ||
|
6f020901a8 | ||
|
306d55f636 | ||
|
27c9bb9b10 | ||
|
1cd253e05f | ||
|
e9360221b7 | ||
|
eb8a18304f | ||
|
e111958762 | ||
|
43a9194739 | ||
|
50da0cb732 | ||
|
a948334b8b | ||
|
b219b99380 | ||
|
21de3e2961 | ||
|
f69a19d01d | ||
|
ec0d426a6f | ||
|
678700f137 | ||
|
95f584c5d0 | ||
|
c0e1095e66 | ||
|
938004c090 | ||
|
a77a0b5eb0 | ||
|
3e77101b16 | ||
|
341071402c | ||
|
f15551b47b | ||
|
c2dd3bb5d2 | ||
|
b93c21a041 | ||
|
4d800a8487 | ||
|
11e537cb28 | ||
|
35214a34ee | ||
|
57afc5f035 | ||
|
907a54e9f6 | ||
|
a1928bd299 | ||
|
c7aed3c98c | ||
|
96cb9f13dd | ||
|
4d5676f455 | ||
|
5bdd49f36c | ||
|
d4da0a0a21 | ||
|
fe0d3b3e27 | ||
|
9d19001945 | ||
|
f55d15d9b9 | ||
|
237abda2eb | ||
|
08c9cc18d6 | ||
|
55da169fe7 | ||
|
a1211a0f5a | ||
|
179d54505a | ||
|
1514dcbb34 | ||
|
ac78184fe7 | ||
|
1777ff37c0 | ||
|
7dca59da56 | ||
|
6ac24fcf54 | ||
|
70e4933221 | ||
|
c1250ccb35 | ||
|
98e5ea5206 | ||
|
b22776d3a8 | ||
|
d4c1faad3b | ||
|
0fa45c3954 | ||
|
c802d7d58a | ||
|
5ab06caffc | ||
|
737d7484c5 | ||
|
b3a742304e | ||
|
6514d72fea | ||
|
c3d4b429d9 | ||
|
7ff848c2c8 | ||
|
a11ba3f437 | ||
|
6604d94271 | ||
|
e30fa1da4d | ||
|
7ea3345e0d | ||
|
1e97d73bd0 | ||
|
f0e9703384 | ||
|
514941627b | ||
|
c2096575ba | ||
|
3d9e5cc1cf | ||
|
acc08c65d9 | ||
|
f1e4403f98 | ||
|
b1d53f0472 | ||
|
5ffa2c9ca1 | ||
|
5a6fe87b7c | ||
|
bc3cfb008e | ||
|
9791a2ea40 | ||
|
9fb9d16737 | ||
|
6be1d0724a | ||
|
9238520490 | ||
|
dd1bb579df | ||
|
57943798b7 | ||
|
b43c3d75a2 |
6
.gitmodules
vendored
6
.gitmodules
vendored
@ -170,9 +170,6 @@
|
|||||||
[submodule "contrib/fast_float"]
|
[submodule "contrib/fast_float"]
|
||||||
path = contrib/fast_float
|
path = contrib/fast_float
|
||||||
url = https://github.com/fastfloat/fast_float
|
url = https://github.com/fastfloat/fast_float
|
||||||
[submodule "contrib/libpq"]
|
|
||||||
path = contrib/libpq
|
|
||||||
url = https://github.com/ClickHouse/libpq
|
|
||||||
[submodule "contrib/NuRaft"]
|
[submodule "contrib/NuRaft"]
|
||||||
path = contrib/NuRaft
|
path = contrib/NuRaft
|
||||||
url = https://github.com/ClickHouse/NuRaft
|
url = https://github.com/ClickHouse/NuRaft
|
||||||
@ -369,3 +366,6 @@
|
|||||||
[submodule "contrib/numactl"]
|
[submodule "contrib/numactl"]
|
||||||
path = contrib/numactl
|
path = contrib/numactl
|
||||||
url = https://github.com/ClickHouse/numactl.git
|
url = https://github.com/ClickHouse/numactl.git
|
||||||
|
[submodule "contrib/postgres"]
|
||||||
|
path = contrib/postgres
|
||||||
|
url = https://github.com/ClickHouse/postgres.git
|
||||||
|
20
README.md
20
README.md
@ -40,17 +40,8 @@ Every month we get together with the community (users, contributors, customers,
|
|||||||
|
|
||||||
Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
|
Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
|
||||||
|
|
||||||
The following upcoming meetups are featuring creator of ClickHouse & CTO, Alexey Milovidov:
|
Upcoming meetups
|
||||||
|
|
||||||
* [Raleigh Meetup (Deutsche Bank)](https://www.meetup.com/triangletechtalks/events/302723486/) - September 9
|
|
||||||
* [New York Meetup (Rokt)](https://www.meetup.com/clickhouse-new-york-user-group/events/302575342) - September 10
|
|
||||||
* [Chicago Meetup (Jump Capital)](https://lu.ma/43tvmrfw) - September 12
|
|
||||||
|
|
||||||
Other upcoming meetups
|
|
||||||
|
|
||||||
* [Toronto Meetup (Shopify)](https://www.meetup.com/clickhouse-toronto-user-group/events/301490855/) - September 10
|
|
||||||
* [Austin Meetup](https://www.meetup.com/clickhouse-austin-user-group/events/302558689/) - September 17
|
|
||||||
* [London Meetup](https://www.meetup.com/clickhouse-london-user-group/events/302977267) - September 17
|
|
||||||
* [Bangalore Meetup](https://www.meetup.com/clickhouse-bangalore-user-group/events/303208274/) - September 18
|
* [Bangalore Meetup](https://www.meetup.com/clickhouse-bangalore-user-group/events/303208274/) - September 18
|
||||||
* [Tel Aviv Meetup](https://www.meetup.com/clickhouse-meetup-israel/events/303095121) - September 22
|
* [Tel Aviv Meetup](https://www.meetup.com/clickhouse-meetup-israel/events/303095121) - September 22
|
||||||
* [Jakarta Meetup](https://www.meetup.com/clickhouse-indonesia-user-group/events/303191359/) - October 1
|
* [Jakarta Meetup](https://www.meetup.com/clickhouse-indonesia-user-group/events/303191359/) - October 1
|
||||||
@ -62,13 +53,20 @@ Other upcoming meetups
|
|||||||
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21
|
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21
|
||||||
* [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26
|
* [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26
|
||||||
|
|
||||||
Recently completed events
|
Recently completed meetups
|
||||||
|
|
||||||
* [ClickHouse Guangzhou User Group Meetup](https://mp.weixin.qq.com/s/GSvo-7xUoVzCsuUvlLTpCw) - August 25
|
* [ClickHouse Guangzhou User Group Meetup](https://mp.weixin.qq.com/s/GSvo-7xUoVzCsuUvlLTpCw) - August 25
|
||||||
* [Seattle Meetup (Statsig)](https://www.meetup.com/clickhouse-seattle-user-group/events/302518075/) - August 27
|
* [Seattle Meetup (Statsig)](https://www.meetup.com/clickhouse-seattle-user-group/events/302518075/) - August 27
|
||||||
* [Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302732666/) - August 27
|
* [Melbourne Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302732666/) - August 27
|
||||||
* [Sydney Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302862966/) - September 5
|
* [Sydney Meetup](https://www.meetup.com/clickhouse-australia-user-group/events/302862966/) - September 5
|
||||||
* [Zurich Meetup](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/302267429/) - September 5
|
* [Zurich Meetup](https://www.meetup.com/clickhouse-switzerland-meetup-group/events/302267429/) - September 5
|
||||||
* [San Francisco Meetup (Cloudflare)](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/302540575) - September 5
|
* [San Francisco Meetup (Cloudflare)](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/302540575) - September 5
|
||||||
|
* [Raleigh Meetup (Deutsche Bank)](https://www.meetup.com/triangletechtalks/events/302723486/) - September 9
|
||||||
|
* [New York Meetup (Rokt)](https://www.meetup.com/clickhouse-new-york-user-group/events/302575342) - September 10
|
||||||
|
* [Toronto Meetup (Shopify)](https://www.meetup.com/clickhouse-toronto-user-group/events/301490855/) - September 10
|
||||||
|
* [Chicago Meetup (Jump Capital)](https://lu.ma/43tvmrfw) - September 12
|
||||||
|
* [London Meetup](https://www.meetup.com/clickhouse-london-user-group/events/302977267) - September 17
|
||||||
|
* [Austin Meetup](https://www.meetup.com/clickhouse-austin-user-group/events/302558689/) - September 17
|
||||||
|
|
||||||
## Recent Recordings
|
## Recent Recordings
|
||||||
* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
|
* **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"
|
||||||
|
@ -188,8 +188,9 @@ namespace Crypto
|
|||||||
pFile = fopen(keyFile.c_str(), "r");
|
pFile = fopen(keyFile.c_str(), "r");
|
||||||
if (pFile)
|
if (pFile)
|
||||||
{
|
{
|
||||||
pem_password_cb * pCB = pass.empty() ? (pem_password_cb *)0 : &passCB;
|
pem_password_cb * pCB = &passCB;
|
||||||
void * pPassword = pass.empty() ? (void *)0 : (void *)pass.c_str();
|
static constexpr char * no_password = "";
|
||||||
|
void * pPassword = pass.empty() ? (void *)no_password : (void *)pass.c_str();
|
||||||
if (readFunc(pFile, &pKey, pCB, pPassword))
|
if (readFunc(pFile, &pKey, pCB, pPassword))
|
||||||
{
|
{
|
||||||
fclose(pFile);
|
fclose(pFile);
|
||||||
@ -225,6 +226,13 @@ namespace Crypto
|
|||||||
error:
|
error:
|
||||||
if (pFile)
|
if (pFile)
|
||||||
fclose(pFile);
|
fclose(pFile);
|
||||||
|
if (*ppKey)
|
||||||
|
{
|
||||||
|
if constexpr (std::is_same_v<K, EVP_PKEY>)
|
||||||
|
EVP_PKEY_free(*ppKey);
|
||||||
|
else
|
||||||
|
EC_KEY_free(*ppKey);
|
||||||
|
}
|
||||||
throw OpenSSLException("EVPKey::loadKey(string)");
|
throw OpenSSLException("EVPKey::loadKey(string)");
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -286,6 +294,13 @@ namespace Crypto
|
|||||||
error:
|
error:
|
||||||
if (pBIO)
|
if (pBIO)
|
||||||
BIO_free(pBIO);
|
BIO_free(pBIO);
|
||||||
|
if (*ppKey)
|
||||||
|
{
|
||||||
|
if constexpr (std::is_same_v<K, EVP_PKEY>)
|
||||||
|
EVP_PKEY_free(*ppKey);
|
||||||
|
else
|
||||||
|
EC_KEY_free(*ppKey);
|
||||||
|
}
|
||||||
throw OpenSSLException("EVPKey::loadKey(stream)");
|
throw OpenSSLException("EVPKey::loadKey(stream)");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -248,6 +248,9 @@ namespace Net
|
|||||||
SSL_CTX * sslContext() const;
|
SSL_CTX * sslContext() const;
|
||||||
/// Returns the underlying OpenSSL SSL Context object.
|
/// Returns the underlying OpenSSL SSL Context object.
|
||||||
|
|
||||||
|
SSL_CTX * takeSslContext();
|
||||||
|
/// Takes ownership of the underlying OpenSSL SSL Context object.
|
||||||
|
|
||||||
Usage usage() const;
|
Usage usage() const;
|
||||||
/// Returns whether the context is for use by a client or by a server
|
/// Returns whether the context is for use by a client or by a server
|
||||||
/// and whether TLSv1 is required.
|
/// and whether TLSv1 is required.
|
||||||
@ -401,6 +404,13 @@ namespace Net
|
|||||||
return _pSSLContext;
|
return _pSSLContext;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
inline SSL_CTX * Context::takeSslContext()
|
||||||
|
{
|
||||||
|
auto * result = _pSSLContext;
|
||||||
|
_pSSLContext = nullptr;
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
inline bool Context::extendedCertificateVerificationEnabled() const
|
inline bool Context::extendedCertificateVerificationEnabled() const
|
||||||
{
|
{
|
||||||
|
@ -106,6 +106,11 @@ Context::Context(
|
|||||||
|
|
||||||
Context::~Context()
|
Context::~Context()
|
||||||
{
|
{
|
||||||
|
if (_pSSLContext == nullptr)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
SSL_CTX_free(_pSSLContext);
|
SSL_CTX_free(_pSSLContext);
|
||||||
|
7
contrib/CMakeLists.txt
vendored
7
contrib/CMakeLists.txt
vendored
@ -145,8 +145,13 @@ add_contrib (isa-l-cmake isa-l)
|
|||||||
add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5, isa-l
|
add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5, isa-l
|
||||||
add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arrow, libhdfs3
|
add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arrow, libhdfs3
|
||||||
add_contrib (cppkafka-cmake cppkafka)
|
add_contrib (cppkafka-cmake cppkafka)
|
||||||
|
|
||||||
|
option(ENABLE_LIBPQXX "Enable PostgreSQL" ${ENABLE_LIBRARIES})
|
||||||
|
if (ENABLE_LIBPQXX)
|
||||||
|
add_contrib (postgres-cmake postgres)
|
||||||
add_contrib (libpqxx-cmake libpqxx)
|
add_contrib (libpqxx-cmake libpqxx)
|
||||||
add_contrib (libpq-cmake libpq)
|
endif()
|
||||||
|
|
||||||
add_contrib (rocksdb-cmake rocksdb) # requires: jemalloc, snappy, zlib, lz4, zstd, liburing
|
add_contrib (rocksdb-cmake rocksdb) # requires: jemalloc, snappy, zlib, lz4, zstd, liburing
|
||||||
add_contrib (nuraft-cmake NuRaft)
|
add_contrib (nuraft-cmake NuRaft)
|
||||||
add_contrib (fast_float-cmake fast_float)
|
add_contrib (fast_float-cmake fast_float)
|
||||||
|
1
contrib/libpq
vendored
1
contrib/libpq
vendored
@ -1 +0,0 @@
|
|||||||
Subproject commit 2446f2c85650b56df9d4ebc4c2ea7f4b01beee57
|
|
@ -1,78 +0,0 @@
|
|||||||
if (NOT ENABLE_LIBPQXX)
|
|
||||||
return()
|
|
||||||
endif()
|
|
||||||
|
|
||||||
set(LIBPQ_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpq")
|
|
||||||
|
|
||||||
set(SRCS
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-auth.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-auth-scram.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-connect.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-exec.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-lobj.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-misc.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-print.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-trace.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-protocol3.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-secure.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-secure-common.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/fe-secure-openssl.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/legacy-pqsignal.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/libpq-events.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/pqexpbuffer.c"
|
|
||||||
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/scram-common.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/sha2.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/sha1.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/md5.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/md5_common.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/hmac_openssl.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/cryptohash.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/saslprep.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/unicode_norm.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/ip.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/jsonapi.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/wchar.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/base64.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/link-canary.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/fe_memutils.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/string.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/pg_get_line.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/stringinfo.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/psprintf.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/encnames.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/common/logging.c"
|
|
||||||
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/snprintf.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/strlcpy.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/strerror.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/inet_net_ntop.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/getpeereid.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/chklocale.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/noblock.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/pg_strong_random.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/pgstrcasecmp.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/thread.c"
|
|
||||||
"${LIBPQ_SOURCE_DIR}/port/path.c"
|
|
||||||
)
|
|
||||||
|
|
||||||
add_library(_libpq ${SRCS})
|
|
||||||
|
|
||||||
add_definitions(-DHAVE_BIO_METH_NEW)
|
|
||||||
add_definitions(-DHAVE_HMAC_CTX_NEW)
|
|
||||||
add_definitions(-DHAVE_HMAC_CTX_FREE)
|
|
||||||
|
|
||||||
target_include_directories (_libpq SYSTEM PUBLIC ${LIBPQ_SOURCE_DIR})
|
|
||||||
target_include_directories (_libpq SYSTEM PUBLIC "${LIBPQ_SOURCE_DIR}/include")
|
|
||||||
target_include_directories (_libpq SYSTEM PRIVATE "${LIBPQ_SOURCE_DIR}/configs")
|
|
||||||
|
|
||||||
# NOTE: this is a dirty hack to avoid and instead pg_config.h should be shipped
|
|
||||||
# for different OS'es like for jemalloc, not one generic for all OS'es like
|
|
||||||
# now.
|
|
||||||
if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL)
|
|
||||||
target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1)
|
|
||||||
endif()
|
|
||||||
|
|
||||||
target_link_libraries (_libpq PRIVATE OpenSSL::SSL)
|
|
||||||
|
|
||||||
add_library(ch_contrib::libpq ALIAS _libpq)
|
|
2
contrib/libpqxx
vendored
2
contrib/libpqxx
vendored
@ -1 +1 @@
|
|||||||
Subproject commit c995193a3a14d71f4711f1f421f65a1a1db64640
|
Subproject commit 41e4c331564167cca97ad6eccbd5b8879c2ca044
|
@ -1,16 +1,9 @@
|
|||||||
option(ENABLE_LIBPQXX "Enalbe libpqxx" ${ENABLE_LIBRARIES})
|
|
||||||
|
|
||||||
if (NOT ENABLE_LIBPQXX)
|
|
||||||
message(STATUS "Not using libpqxx")
|
|
||||||
return()
|
|
||||||
endif()
|
|
||||||
|
|
||||||
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpqxx")
|
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/libpqxx")
|
||||||
|
|
||||||
set (SRCS
|
set (SRCS
|
||||||
"${LIBRARY_DIR}/src/strconv.cxx"
|
|
||||||
"${LIBRARY_DIR}/src/array.cxx"
|
"${LIBRARY_DIR}/src/array.cxx"
|
||||||
"${LIBRARY_DIR}/src/binarystring.cxx"
|
"${LIBRARY_DIR}/src/binarystring.cxx"
|
||||||
|
"${LIBRARY_DIR}/src/blob.cxx"
|
||||||
"${LIBRARY_DIR}/src/connection.cxx"
|
"${LIBRARY_DIR}/src/connection.cxx"
|
||||||
"${LIBRARY_DIR}/src/cursor.cxx"
|
"${LIBRARY_DIR}/src/cursor.cxx"
|
||||||
"${LIBRARY_DIR}/src/encodings.cxx"
|
"${LIBRARY_DIR}/src/encodings.cxx"
|
||||||
@ -19,59 +12,25 @@ set (SRCS
|
|||||||
"${LIBRARY_DIR}/src/field.cxx"
|
"${LIBRARY_DIR}/src/field.cxx"
|
||||||
"${LIBRARY_DIR}/src/largeobject.cxx"
|
"${LIBRARY_DIR}/src/largeobject.cxx"
|
||||||
"${LIBRARY_DIR}/src/notification.cxx"
|
"${LIBRARY_DIR}/src/notification.cxx"
|
||||||
|
"${LIBRARY_DIR}/src/params.cxx"
|
||||||
"${LIBRARY_DIR}/src/pipeline.cxx"
|
"${LIBRARY_DIR}/src/pipeline.cxx"
|
||||||
"${LIBRARY_DIR}/src/result.cxx"
|
"${LIBRARY_DIR}/src/result.cxx"
|
||||||
"${LIBRARY_DIR}/src/robusttransaction.cxx"
|
"${LIBRARY_DIR}/src/robusttransaction.cxx"
|
||||||
|
"${LIBRARY_DIR}/src/row.cxx"
|
||||||
"${LIBRARY_DIR}/src/sql_cursor.cxx"
|
"${LIBRARY_DIR}/src/sql_cursor.cxx"
|
||||||
|
"${LIBRARY_DIR}/src/strconv.cxx"
|
||||||
"${LIBRARY_DIR}/src/stream_from.cxx"
|
"${LIBRARY_DIR}/src/stream_from.cxx"
|
||||||
"${LIBRARY_DIR}/src/stream_to.cxx"
|
"${LIBRARY_DIR}/src/stream_to.cxx"
|
||||||
"${LIBRARY_DIR}/src/subtransaction.cxx"
|
"${LIBRARY_DIR}/src/subtransaction.cxx"
|
||||||
|
"${LIBRARY_DIR}/src/time.cxx"
|
||||||
"${LIBRARY_DIR}/src/transaction.cxx"
|
"${LIBRARY_DIR}/src/transaction.cxx"
|
||||||
"${LIBRARY_DIR}/src/transaction_base.cxx"
|
"${LIBRARY_DIR}/src/transaction_base.cxx"
|
||||||
"${LIBRARY_DIR}/src/row.cxx"
|
|
||||||
"${LIBRARY_DIR}/src/params.cxx"
|
|
||||||
"${LIBRARY_DIR}/src/util.cxx"
|
"${LIBRARY_DIR}/src/util.cxx"
|
||||||
"${LIBRARY_DIR}/src/version.cxx"
|
"${LIBRARY_DIR}/src/version.cxx"
|
||||||
|
"${LIBRARY_DIR}/src/wait.cxx"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Need to explicitly include each header file, because in the directory include/pqxx there are also files
|
add_library(_libpqxx ${SRCS})
|
||||||
# like just 'array'. So if including the whole directory with `target_include_directories`, it will make
|
|
||||||
# conflicts with all includes of <array>.
|
|
||||||
set (HDRS
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/array.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/params.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/binarystring.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/composite.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/connection.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/cursor.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/dbtransaction.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/errorhandler.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/except.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/field.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/isolation.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/largeobject.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/nontransaction.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/notification.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/pipeline.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/prepared_statement.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/result.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/robusttransaction.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/row.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/separated_list.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/strconv.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/stream_from.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/stream_to.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/subtransaction.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/transaction.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/transaction_base.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/types.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/util.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/version.hxx"
|
|
||||||
"${LIBRARY_DIR}/include/pqxx/zview.hxx"
|
|
||||||
)
|
|
||||||
|
|
||||||
add_library(_libpqxx ${SRCS} ${HDRS})
|
|
||||||
|
|
||||||
target_link_libraries(_libpqxx PUBLIC ch_contrib::libpq)
|
target_link_libraries(_libpqxx PUBLIC ch_contrib::libpq)
|
||||||
target_include_directories (_libpqxx SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include")
|
target_include_directories (_libpqxx SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include")
|
||||||
|
|
||||||
|
1
contrib/postgres
vendored
Submodule
1
contrib/postgres
vendored
Submodule
@ -0,0 +1 @@
|
|||||||
|
Subproject commit 2e51f82e27f4be389cc239d1b8784bbf2f01d33a
|
81
contrib/postgres-cmake/CMakeLists.txt
Normal file
81
contrib/postgres-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
# Build description for libpq which is part of the PostgreSQL sources
|
||||||
|
|
||||||
|
set(POSTGRES_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/postgres")
|
||||||
|
set(LIBPQ_SOURCE_DIR "${POSTGRES_SOURCE_DIR}/src/interfaces/libpq")
|
||||||
|
set(LIBPQ_CMAKE_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/postgres-cmake")
|
||||||
|
|
||||||
|
set(SRCS
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-auth.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-auth-scram.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-connect.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-exec.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-lobj.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-misc.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-print.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-trace.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-protocol3.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-secure.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-secure-common.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/fe-secure-openssl.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/legacy-pqsignal.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/libpq-events.c"
|
||||||
|
"${LIBPQ_SOURCE_DIR}/pqexpbuffer.c"
|
||||||
|
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/scram-common.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/sha2.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/sha1.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/md5.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/md5_common.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/hmac_openssl.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/cryptohash.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/saslprep.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/unicode_norm.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/ip.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/jsonapi.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/wchar.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/base64.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/link-canary.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/fe_memutils.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/string.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/pg_get_line.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/pg_prng.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/stringinfo.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/psprintf.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/encnames.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/common/logging.c"
|
||||||
|
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/snprintf.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/strlcat.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/strlcpy.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/strerror.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/inet_net_ntop.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/getpeereid.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/chklocale.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/noblock.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/pg_strong_random.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/pgstrcasecmp.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/pg_bitutils.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/thread.c"
|
||||||
|
"${POSTGRES_SOURCE_DIR}/src/port/path.c"
|
||||||
|
)
|
||||||
|
|
||||||
|
add_library(_libpq ${SRCS})
|
||||||
|
|
||||||
|
add_definitions(-DHAVE_BIO_METH_NEW)
|
||||||
|
add_definitions(-DHAVE_HMAC_CTX_NEW)
|
||||||
|
add_definitions(-DHAVE_HMAC_CTX_FREE)
|
||||||
|
|
||||||
|
target_include_directories (_libpq SYSTEM PUBLIC ${LIBPQ_SOURCE_DIR})
|
||||||
|
target_include_directories (_libpq SYSTEM PUBLIC "${POSTGRES_SOURCE_DIR}/src/include")
|
||||||
|
target_include_directories (_libpq SYSTEM PUBLIC "${LIBPQ_CMAKE_SOURCE_DIR}") # pre-generated headers
|
||||||
|
|
||||||
|
# NOTE: this is a dirty hack to avoid and instead pg_config.h should be shipped
|
||||||
|
# for different OS'es like for jemalloc, not one generic for all OS'es like
|
||||||
|
# now.
|
||||||
|
if (OS_DARWIN OR OS_FREEBSD OR USE_MUSL)
|
||||||
|
target_compile_definitions(_libpq PRIVATE -DSTRERROR_R_INT=1)
|
||||||
|
endif()
|
||||||
|
|
||||||
|
target_link_libraries (_libpq PRIVATE OpenSSL::SSL)
|
||||||
|
|
||||||
|
add_library(ch_contrib::libpq ALIAS _libpq)
|
471
contrib/postgres-cmake/nodes/nodetags.h
Normal file
471
contrib/postgres-cmake/nodes/nodetags.h
Normal file
@ -0,0 +1,471 @@
|
|||||||
|
/*-------------------------------------------------------------------------
|
||||||
|
*
|
||||||
|
* nodetags.h
|
||||||
|
* Generated node infrastructure code
|
||||||
|
*
|
||||||
|
* Portions Copyright (c) 1996-2023, PostgreSQL Global Development Group
|
||||||
|
* Portions Copyright (c) 1994, Regents of the University of California
|
||||||
|
*
|
||||||
|
* NOTES
|
||||||
|
* ******************************
|
||||||
|
* *** DO NOT EDIT THIS FILE! ***
|
||||||
|
* ******************************
|
||||||
|
*
|
||||||
|
* It has been GENERATED by src/backend/nodes/gen_node_support.pl
|
||||||
|
*
|
||||||
|
*-------------------------------------------------------------------------
|
||||||
|
*/
|
||||||
|
T_List = 1,
|
||||||
|
T_Alias = 2,
|
||||||
|
T_RangeVar = 3,
|
||||||
|
T_TableFunc = 4,
|
||||||
|
T_IntoClause = 5,
|
||||||
|
T_Var = 6,
|
||||||
|
T_Const = 7,
|
||||||
|
T_Param = 8,
|
||||||
|
T_Aggref = 9,
|
||||||
|
T_GroupingFunc = 10,
|
||||||
|
T_WindowFunc = 11,
|
||||||
|
T_SubscriptingRef = 12,
|
||||||
|
T_FuncExpr = 13,
|
||||||
|
T_NamedArgExpr = 14,
|
||||||
|
T_OpExpr = 15,
|
||||||
|
T_DistinctExpr = 16,
|
||||||
|
T_NullIfExpr = 17,
|
||||||
|
T_ScalarArrayOpExpr = 18,
|
||||||
|
T_BoolExpr = 19,
|
||||||
|
T_SubLink = 20,
|
||||||
|
T_SubPlan = 21,
|
||||||
|
T_AlternativeSubPlan = 22,
|
||||||
|
T_FieldSelect = 23,
|
||||||
|
T_FieldStore = 24,
|
||||||
|
T_RelabelType = 25,
|
||||||
|
T_CoerceViaIO = 26,
|
||||||
|
T_ArrayCoerceExpr = 27,
|
||||||
|
T_ConvertRowtypeExpr = 28,
|
||||||
|
T_CollateExpr = 29,
|
||||||
|
T_CaseExpr = 30,
|
||||||
|
T_CaseWhen = 31,
|
||||||
|
T_CaseTestExpr = 32,
|
||||||
|
T_ArrayExpr = 33,
|
||||||
|
T_RowExpr = 34,
|
||||||
|
T_RowCompareExpr = 35,
|
||||||
|
T_CoalesceExpr = 36,
|
||||||
|
T_MinMaxExpr = 37,
|
||||||
|
T_SQLValueFunction = 38,
|
||||||
|
T_XmlExpr = 39,
|
||||||
|
T_JsonFormat = 40,
|
||||||
|
T_JsonReturning = 41,
|
||||||
|
T_JsonValueExpr = 42,
|
||||||
|
T_JsonConstructorExpr = 43,
|
||||||
|
T_JsonIsPredicate = 44,
|
||||||
|
T_NullTest = 45,
|
||||||
|
T_BooleanTest = 46,
|
||||||
|
T_CoerceToDomain = 47,
|
||||||
|
T_CoerceToDomainValue = 48,
|
||||||
|
T_SetToDefault = 49,
|
||||||
|
T_CurrentOfExpr = 50,
|
||||||
|
T_NextValueExpr = 51,
|
||||||
|
T_InferenceElem = 52,
|
||||||
|
T_TargetEntry = 53,
|
||||||
|
T_RangeTblRef = 54,
|
||||||
|
T_JoinExpr = 55,
|
||||||
|
T_FromExpr = 56,
|
||||||
|
T_OnConflictExpr = 57,
|
||||||
|
T_Query = 58,
|
||||||
|
T_TypeName = 59,
|
||||||
|
T_ColumnRef = 60,
|
||||||
|
T_ParamRef = 61,
|
||||||
|
T_A_Expr = 62,
|
||||||
|
T_A_Const = 63,
|
||||||
|
T_TypeCast = 64,
|
||||||
|
T_CollateClause = 65,
|
||||||
|
T_RoleSpec = 66,
|
||||||
|
T_FuncCall = 67,
|
||||||
|
T_A_Star = 68,
|
||||||
|
T_A_Indices = 69,
|
||||||
|
T_A_Indirection = 70,
|
||||||
|
T_A_ArrayExpr = 71,
|
||||||
|
T_ResTarget = 72,
|
||||||
|
T_MultiAssignRef = 73,
|
||||||
|
T_SortBy = 74,
|
||||||
|
T_WindowDef = 75,
|
||||||
|
T_RangeSubselect = 76,
|
||||||
|
T_RangeFunction = 77,
|
||||||
|
T_RangeTableFunc = 78,
|
||||||
|
T_RangeTableFuncCol = 79,
|
||||||
|
T_RangeTableSample = 80,
|
||||||
|
T_ColumnDef = 81,
|
||||||
|
T_TableLikeClause = 82,
|
||||||
|
T_IndexElem = 83,
|
||||||
|
T_DefElem = 84,
|
||||||
|
T_LockingClause = 85,
|
||||||
|
T_XmlSerialize = 86,
|
||||||
|
T_PartitionElem = 87,
|
||||||
|
T_PartitionSpec = 88,
|
||||||
|
T_PartitionBoundSpec = 89,
|
||||||
|
T_PartitionRangeDatum = 90,
|
||||||
|
T_PartitionCmd = 91,
|
||||||
|
T_RangeTblEntry = 92,
|
||||||
|
T_RTEPermissionInfo = 93,
|
||||||
|
T_RangeTblFunction = 94,
|
||||||
|
T_TableSampleClause = 95,
|
||||||
|
T_WithCheckOption = 96,
|
||||||
|
T_SortGroupClause = 97,
|
||||||
|
T_GroupingSet = 98,
|
||||||
|
T_WindowClause = 99,
|
||||||
|
T_RowMarkClause = 100,
|
||||||
|
T_WithClause = 101,
|
||||||
|
T_InferClause = 102,
|
||||||
|
T_OnConflictClause = 103,
|
||||||
|
T_CTESearchClause = 104,
|
||||||
|
T_CTECycleClause = 105,
|
||||||
|
T_CommonTableExpr = 106,
|
||||||
|
T_MergeWhenClause = 107,
|
||||||
|
T_MergeAction = 108,
|
||||||
|
T_TriggerTransition = 109,
|
||||||
|
T_JsonOutput = 110,
|
||||||
|
T_JsonKeyValue = 111,
|
||||||
|
T_JsonObjectConstructor = 112,
|
||||||
|
T_JsonArrayConstructor = 113,
|
||||||
|
T_JsonArrayQueryConstructor = 114,
|
||||||
|
T_JsonAggConstructor = 115,
|
||||||
|
T_JsonObjectAgg = 116,
|
||||||
|
T_JsonArrayAgg = 117,
|
||||||
|
T_RawStmt = 118,
|
||||||
|
T_InsertStmt = 119,
|
||||||
|
T_DeleteStmt = 120,
|
||||||
|
T_UpdateStmt = 121,
|
||||||
|
T_MergeStmt = 122,
|
||||||
|
T_SelectStmt = 123,
|
||||||
|
T_SetOperationStmt = 124,
|
||||||
|
T_ReturnStmt = 125,
|
||||||
|
T_PLAssignStmt = 126,
|
||||||
|
T_CreateSchemaStmt = 127,
|
||||||
|
T_AlterTableStmt = 128,
|
||||||
|
T_ReplicaIdentityStmt = 129,
|
||||||
|
T_AlterTableCmd = 130,
|
||||||
|
T_AlterCollationStmt = 131,
|
||||||
|
T_AlterDomainStmt = 132,
|
||||||
|
T_GrantStmt = 133,
|
||||||
|
T_ObjectWithArgs = 134,
|
||||||
|
T_AccessPriv = 135,
|
||||||
|
T_GrantRoleStmt = 136,
|
||||||
|
T_AlterDefaultPrivilegesStmt = 137,
|
||||||
|
T_CopyStmt = 138,
|
||||||
|
T_VariableSetStmt = 139,
|
||||||
|
T_VariableShowStmt = 140,
|
||||||
|
T_CreateStmt = 141,
|
||||||
|
T_Constraint = 142,
|
||||||
|
T_CreateTableSpaceStmt = 143,
|
||||||
|
T_DropTableSpaceStmt = 144,
|
||||||
|
T_AlterTableSpaceOptionsStmt = 145,
|
||||||
|
T_AlterTableMoveAllStmt = 146,
|
||||||
|
T_CreateExtensionStmt = 147,
|
||||||
|
T_AlterExtensionStmt = 148,
|
||||||
|
T_AlterExtensionContentsStmt = 149,
|
||||||
|
T_CreateFdwStmt = 150,
|
||||||
|
T_AlterFdwStmt = 151,
|
||||||
|
T_CreateForeignServerStmt = 152,
|
||||||
|
T_AlterForeignServerStmt = 153,
|
||||||
|
T_CreateForeignTableStmt = 154,
|
||||||
|
T_CreateUserMappingStmt = 155,
|
||||||
|
T_AlterUserMappingStmt = 156,
|
||||||
|
T_DropUserMappingStmt = 157,
|
||||||
|
T_ImportForeignSchemaStmt = 158,
|
||||||
|
T_CreatePolicyStmt = 159,
|
||||||
|
T_AlterPolicyStmt = 160,
|
||||||
|
T_CreateAmStmt = 161,
|
||||||
|
T_CreateTrigStmt = 162,
|
||||||
|
T_CreateEventTrigStmt = 163,
|
||||||
|
T_AlterEventTrigStmt = 164,
|
||||||
|
T_CreatePLangStmt = 165,
|
||||||
|
T_CreateRoleStmt = 166,
|
||||||
|
T_AlterRoleStmt = 167,
|
||||||
|
T_AlterRoleSetStmt = 168,
|
||||||
|
T_DropRoleStmt = 169,
|
||||||
|
T_CreateSeqStmt = 170,
|
||||||
|
T_AlterSeqStmt = 171,
|
||||||
|
T_DefineStmt = 172,
|
||||||
|
T_CreateDomainStmt = 173,
|
||||||
|
T_CreateOpClassStmt = 174,
|
||||||
|
T_CreateOpClassItem = 175,
|
||||||
|
T_CreateOpFamilyStmt = 176,
|
||||||
|
T_AlterOpFamilyStmt = 177,
|
||||||
|
T_DropStmt = 178,
|
||||||
|
T_TruncateStmt = 179,
|
||||||
|
T_CommentStmt = 180,
|
||||||
|
T_SecLabelStmt = 181,
|
||||||
|
T_DeclareCursorStmt = 182,
|
||||||
|
T_ClosePortalStmt = 183,
|
||||||
|
T_FetchStmt = 184,
|
||||||
|
T_IndexStmt = 185,
|
||||||
|
T_CreateStatsStmt = 186,
|
||||||
|
T_StatsElem = 187,
|
||||||
|
T_AlterStatsStmt = 188,
|
||||||
|
T_CreateFunctionStmt = 189,
|
||||||
|
T_FunctionParameter = 190,
|
||||||
|
T_AlterFunctionStmt = 191,
|
||||||
|
T_DoStmt = 192,
|
||||||
|
T_InlineCodeBlock = 193,
|
||||||
|
T_CallStmt = 194,
|
||||||
|
T_CallContext = 195,
|
||||||
|
T_RenameStmt = 196,
|
||||||
|
T_AlterObjectDependsStmt = 197,
|
||||||
|
T_AlterObjectSchemaStmt = 198,
|
||||||
|
T_AlterOwnerStmt = 199,
|
||||||
|
T_AlterOperatorStmt = 200,
|
||||||
|
T_AlterTypeStmt = 201,
|
||||||
|
T_RuleStmt = 202,
|
||||||
|
T_NotifyStmt = 203,
|
||||||
|
T_ListenStmt = 204,
|
||||||
|
T_UnlistenStmt = 205,
|
||||||
|
T_TransactionStmt = 206,
|
||||||
|
T_CompositeTypeStmt = 207,
|
||||||
|
T_CreateEnumStmt = 208,
|
||||||
|
T_CreateRangeStmt = 209,
|
||||||
|
T_AlterEnumStmt = 210,
|
||||||
|
T_ViewStmt = 211,
|
||||||
|
T_LoadStmt = 212,
|
||||||
|
T_CreatedbStmt = 213,
|
||||||
|
T_AlterDatabaseStmt = 214,
|
||||||
|
T_AlterDatabaseRefreshCollStmt = 215,
|
||||||
|
T_AlterDatabaseSetStmt = 216,
|
||||||
|
T_DropdbStmt = 217,
|
||||||
|
T_AlterSystemStmt = 218,
|
||||||
|
T_ClusterStmt = 219,
|
||||||
|
T_VacuumStmt = 220,
|
||||||
|
T_VacuumRelation = 221,
|
||||||
|
T_ExplainStmt = 222,
|
||||||
|
T_CreateTableAsStmt = 223,
|
||||||
|
T_RefreshMatViewStmt = 224,
|
||||||
|
T_CheckPointStmt = 225,
|
||||||
|
T_DiscardStmt = 226,
|
||||||
|
T_LockStmt = 227,
|
||||||
|
T_ConstraintsSetStmt = 228,
|
||||||
|
T_ReindexStmt = 229,
|
||||||
|
T_CreateConversionStmt = 230,
|
||||||
|
T_CreateCastStmt = 231,
|
||||||
|
T_CreateTransformStmt = 232,
|
||||||
|
T_PrepareStmt = 233,
|
||||||
|
T_ExecuteStmt = 234,
|
||||||
|
T_DeallocateStmt = 235,
|
||||||
|
T_DropOwnedStmt = 236,
|
||||||
|
T_ReassignOwnedStmt = 237,
|
||||||
|
T_AlterTSDictionaryStmt = 238,
|
||||||
|
T_AlterTSConfigurationStmt = 239,
|
||||||
|
T_PublicationTable = 240,
|
||||||
|
T_PublicationObjSpec = 241,
|
||||||
|
T_CreatePublicationStmt = 242,
|
||||||
|
T_AlterPublicationStmt = 243,
|
||||||
|
T_CreateSubscriptionStmt = 244,
|
||||||
|
T_AlterSubscriptionStmt = 245,
|
||||||
|
T_DropSubscriptionStmt = 246,
|
||||||
|
T_PlannerGlobal = 247,
|
||||||
|
T_PlannerInfo = 248,
|
||||||
|
T_RelOptInfo = 249,
|
||||||
|
T_IndexOptInfo = 250,
|
||||||
|
T_ForeignKeyOptInfo = 251,
|
||||||
|
T_StatisticExtInfo = 252,
|
||||||
|
T_JoinDomain = 253,
|
||||||
|
T_EquivalenceClass = 254,
|
||||||
|
T_EquivalenceMember = 255,
|
||||||
|
T_PathKey = 256,
|
||||||
|
T_PathTarget = 257,
|
||||||
|
T_ParamPathInfo = 258,
|
||||||
|
T_Path = 259,
|
||||||
|
T_IndexPath = 260,
|
||||||
|
T_IndexClause = 261,
|
||||||
|
T_BitmapHeapPath = 262,
|
||||||
|
T_BitmapAndPath = 263,
|
||||||
|
T_BitmapOrPath = 264,
|
||||||
|
T_TidPath = 265,
|
||||||
|
T_TidRangePath = 266,
|
||||||
|
T_SubqueryScanPath = 267,
|
||||||
|
T_ForeignPath = 268,
|
||||||
|
T_CustomPath = 269,
|
||||||
|
T_AppendPath = 270,
|
||||||
|
T_MergeAppendPath = 271,
|
||||||
|
T_GroupResultPath = 272,
|
||||||
|
T_MaterialPath = 273,
|
||||||
|
T_MemoizePath = 274,
|
||||||
|
T_UniquePath = 275,
|
||||||
|
T_GatherPath = 276,
|
||||||
|
T_GatherMergePath = 277,
|
||||||
|
T_NestPath = 278,
|
||||||
|
T_MergePath = 279,
|
||||||
|
T_HashPath = 280,
|
||||||
|
T_ProjectionPath = 281,
|
||||||
|
T_ProjectSetPath = 282,
|
||||||
|
T_SortPath = 283,
|
||||||
|
T_IncrementalSortPath = 284,
|
||||||
|
T_GroupPath = 285,
|
||||||
|
T_UpperUniquePath = 286,
|
||||||
|
T_AggPath = 287,
|
||||||
|
T_GroupingSetData = 288,
|
||||||
|
T_RollupData = 289,
|
||||||
|
T_GroupingSetsPath = 290,
|
||||||
|
T_MinMaxAggPath = 291,
|
||||||
|
T_WindowAggPath = 292,
|
||||||
|
T_SetOpPath = 293,
|
||||||
|
T_RecursiveUnionPath = 294,
|
||||||
|
T_LockRowsPath = 295,
|
||||||
|
T_ModifyTablePath = 296,
|
||||||
|
T_LimitPath = 297,
|
||||||
|
T_RestrictInfo = 298,
|
||||||
|
T_PlaceHolderVar = 299,
|
||||||
|
T_SpecialJoinInfo = 300,
|
||||||
|
T_OuterJoinClauseInfo = 301,
|
||||||
|
T_AppendRelInfo = 302,
|
||||||
|
T_RowIdentityVarInfo = 303,
|
||||||
|
T_PlaceHolderInfo = 304,
|
||||||
|
T_MinMaxAggInfo = 305,
|
||||||
|
T_PlannerParamItem = 306,
|
||||||
|
T_AggInfo = 307,
|
||||||
|
T_AggTransInfo = 308,
|
||||||
|
T_PlannedStmt = 309,
|
||||||
|
T_Result = 310,
|
||||||
|
T_ProjectSet = 311,
|
||||||
|
T_ModifyTable = 312,
|
||||||
|
T_Append = 313,
|
||||||
|
T_MergeAppend = 314,
|
||||||
|
T_RecursiveUnion = 315,
|
||||||
|
T_BitmapAnd = 316,
|
||||||
|
T_BitmapOr = 317,
|
||||||
|
T_SeqScan = 318,
|
||||||
|
T_SampleScan = 319,
|
||||||
|
T_IndexScan = 320,
|
||||||
|
T_IndexOnlyScan = 321,
|
||||||
|
T_BitmapIndexScan = 322,
|
||||||
|
T_BitmapHeapScan = 323,
|
||||||
|
T_TidScan = 324,
|
||||||
|
T_TidRangeScan = 325,
|
||||||
|
T_SubqueryScan = 326,
|
||||||
|
T_FunctionScan = 327,
|
||||||
|
T_ValuesScan = 328,
|
||||||
|
T_TableFuncScan = 329,
|
||||||
|
T_CteScan = 330,
|
||||||
|
T_NamedTuplestoreScan = 331,
|
||||||
|
T_WorkTableScan = 332,
|
||||||
|
T_ForeignScan = 333,
|
||||||
|
T_CustomScan = 334,
|
||||||
|
T_NestLoop = 335,
|
||||||
|
T_NestLoopParam = 336,
|
||||||
|
T_MergeJoin = 337,
|
||||||
|
T_HashJoin = 338,
|
||||||
|
T_Material = 339,
|
||||||
|
T_Memoize = 340,
|
||||||
|
T_Sort = 341,
|
||||||
|
T_IncrementalSort = 342,
|
||||||
|
T_Group = 343,
|
||||||
|
T_Agg = 344,
|
||||||
|
T_WindowAgg = 345,
|
||||||
|
T_Unique = 346,
|
||||||
|
T_Gather = 347,
|
||||||
|
T_GatherMerge = 348,
|
||||||
|
T_Hash = 349,
|
||||||
|
T_SetOp = 350,
|
||||||
|
T_LockRows = 351,
|
||||||
|
T_Limit = 352,
|
||||||
|
T_PlanRowMark = 353,
|
||||||
|
T_PartitionPruneInfo = 354,
|
||||||
|
T_PartitionedRelPruneInfo = 355,
|
||||||
|
T_PartitionPruneStepOp = 356,
|
||||||
|
T_PartitionPruneStepCombine = 357,
|
||||||
|
T_PlanInvalItem = 358,
|
||||||
|
T_ExprState = 359,
|
||||||
|
T_IndexInfo = 360,
|
||||||
|
T_ExprContext = 361,
|
||||||
|
T_ReturnSetInfo = 362,
|
||||||
|
T_ProjectionInfo = 363,
|
||||||
|
T_JunkFilter = 364,
|
||||||
|
T_OnConflictSetState = 365,
|
||||||
|
T_MergeActionState = 366,
|
||||||
|
T_ResultRelInfo = 367,
|
||||||
|
T_EState = 368,
|
||||||
|
T_WindowFuncExprState = 369,
|
||||||
|
T_SetExprState = 370,
|
||||||
|
T_SubPlanState = 371,
|
||||||
|
T_DomainConstraintState = 372,
|
||||||
|
T_ResultState = 373,
|
||||||
|
T_ProjectSetState = 374,
|
||||||
|
T_ModifyTableState = 375,
|
||||||
|
T_AppendState = 376,
|
||||||
|
T_MergeAppendState = 377,
|
||||||
|
T_RecursiveUnionState = 378,
|
||||||
|
T_BitmapAndState = 379,
|
||||||
|
T_BitmapOrState = 380,
|
||||||
|
T_ScanState = 381,
|
||||||
|
T_SeqScanState = 382,
|
||||||
|
T_SampleScanState = 383,
|
||||||
|
T_IndexScanState = 384,
|
||||||
|
T_IndexOnlyScanState = 385,
|
||||||
|
T_BitmapIndexScanState = 386,
|
||||||
|
T_BitmapHeapScanState = 387,
|
||||||
|
T_TidScanState = 388,
|
||||||
|
T_TidRangeScanState = 389,
|
||||||
|
T_SubqueryScanState = 390,
|
||||||
|
T_FunctionScanState = 391,
|
||||||
|
T_ValuesScanState = 392,
|
||||||
|
T_TableFuncScanState = 393,
|
||||||
|
T_CteScanState = 394,
|
||||||
|
T_NamedTuplestoreScanState = 395,
|
||||||
|
T_WorkTableScanState = 396,
|
||||||
|
T_ForeignScanState = 397,
|
||||||
|
T_CustomScanState = 398,
|
||||||
|
T_JoinState = 399,
|
||||||
|
T_NestLoopState = 400,
|
||||||
|
T_MergeJoinState = 401,
|
||||||
|
T_HashJoinState = 402,
|
||||||
|
T_MaterialState = 403,
|
||||||
|
T_MemoizeState = 404,
|
||||||
|
T_SortState = 405,
|
||||||
|
T_IncrementalSortState = 406,
|
||||||
|
T_GroupState = 407,
|
||||||
|
T_AggState = 408,
|
||||||
|
T_WindowAggState = 409,
|
||||||
|
T_UniqueState = 410,
|
||||||
|
T_GatherState = 411,
|
||||||
|
T_GatherMergeState = 412,
|
||||||
|
T_HashState = 413,
|
||||||
|
T_SetOpState = 414,
|
||||||
|
T_LockRowsState = 415,
|
||||||
|
T_LimitState = 416,
|
||||||
|
T_IndexAmRoutine = 417,
|
||||||
|
T_TableAmRoutine = 418,
|
||||||
|
T_TsmRoutine = 419,
|
||||||
|
T_EventTriggerData = 420,
|
||||||
|
T_TriggerData = 421,
|
||||||
|
T_TupleTableSlot = 422,
|
||||||
|
T_FdwRoutine = 423,
|
||||||
|
T_Bitmapset = 424,
|
||||||
|
T_ExtensibleNode = 425,
|
||||||
|
T_ErrorSaveContext = 426,
|
||||||
|
T_IdentifySystemCmd = 427,
|
||||||
|
T_BaseBackupCmd = 428,
|
||||||
|
T_CreateReplicationSlotCmd = 429,
|
||||||
|
T_DropReplicationSlotCmd = 430,
|
||||||
|
T_StartReplicationCmd = 431,
|
||||||
|
T_ReadReplicationSlotCmd = 432,
|
||||||
|
T_TimeLineHistoryCmd = 433,
|
||||||
|
T_SupportRequestSimplify = 434,
|
||||||
|
T_SupportRequestSelectivity = 435,
|
||||||
|
T_SupportRequestCost = 436,
|
||||||
|
T_SupportRequestRows = 437,
|
||||||
|
T_SupportRequestIndexCondition = 438,
|
||||||
|
T_SupportRequestWFuncMonotonic = 439,
|
||||||
|
T_SupportRequestOptimizeWindowClause = 440,
|
||||||
|
T_Integer = 441,
|
||||||
|
T_Float = 442,
|
||||||
|
T_Boolean = 443,
|
||||||
|
T_String = 444,
|
||||||
|
T_BitString = 445,
|
||||||
|
T_ForeignKeyCacheInfo = 446,
|
||||||
|
T_IntList = 447,
|
||||||
|
T_OidList = 448,
|
||||||
|
T_XidList = 449,
|
||||||
|
T_AllocSetContext = 450,
|
||||||
|
T_GenerationContext = 451,
|
||||||
|
T_SlabContext = 452,
|
||||||
|
T_TIDBitmap = 453,
|
||||||
|
T_WindowObjectData = 454,
|
803
contrib/postgres-cmake/pg_config.h
Normal file
803
contrib/postgres-cmake/pg_config.h
Normal file
@ -0,0 +1,803 @@
|
|||||||
|
/* src/include/pg_config.h. Generated from pg_config.h.in by configure. */
|
||||||
|
/* src/include/pg_config.h.in. Generated from configure.in by autoheader. */
|
||||||
|
|
||||||
|
/* Define if building universal (internal helper macro) */
|
||||||
|
/* #undef AC_APPLE_UNIVERSAL_BUILD */
|
||||||
|
|
||||||
|
/* The normal alignment of `double', in bytes. */
|
||||||
|
#define ALIGNOF_DOUBLE 4
|
||||||
|
|
||||||
|
/* The normal alignment of `int', in bytes. */
|
||||||
|
#define ALIGNOF_INT 4
|
||||||
|
|
||||||
|
/* The normal alignment of `long', in bytes. */
|
||||||
|
#define ALIGNOF_LONG 4
|
||||||
|
|
||||||
|
/* The normal alignment of `long long int', in bytes. */
|
||||||
|
#define ALIGNOF_LONG_LONG_INT 4
|
||||||
|
|
||||||
|
/* The normal alignment of `short', in bytes. */
|
||||||
|
#define ALIGNOF_SHORT 2
|
||||||
|
|
||||||
|
/* Size of a disk block --- this also limits the size of a tuple. You can set
|
||||||
|
it bigger if you need bigger tuples (although TOAST should reduce the need
|
||||||
|
to have large tuples, since fields can be spread across multiple tuples).
|
||||||
|
BLCKSZ must be a power of 2. The maximum possible value of BLCKSZ is
|
||||||
|
currently 2^15 (32768). This is determined by the 15-bit widths of the
|
||||||
|
lp_off and lp_len fields in ItemIdData (see include/storage/itemid.h).
|
||||||
|
Changing BLCKSZ requires an initdb. */
|
||||||
|
#define BLCKSZ 8192
|
||||||
|
|
||||||
|
/* Define to the default TCP port number on which the server listens and to
|
||||||
|
which clients will try to connect. This can be overridden at run-time, but
|
||||||
|
it's convenient if your clients have the right default compiled in.
|
||||||
|
(--with-pgport=PORTNUM) */
|
||||||
|
#define DEF_PGPORT 5432
|
||||||
|
|
||||||
|
/* Define to the default TCP port number as a string constant. */
|
||||||
|
#define DEF_PGPORT_STR "5432"
|
||||||
|
|
||||||
|
/* Define to the file name extension of dynamically-loadable modules. */
|
||||||
|
#define DLSUFFIX ".so"
|
||||||
|
|
||||||
|
/* Define to build with GSSAPI support. (--with-gssapi) */
|
||||||
|
//#define ENABLE_GSS 0
|
||||||
|
|
||||||
|
/* Define to 1 if you want National Language Support. (--enable-nls) */
|
||||||
|
/* #undef ENABLE_NLS */
|
||||||
|
|
||||||
|
/* Define to 1 to build client libraries as thread-safe code.
|
||||||
|
(--enable-thread-safety) */
|
||||||
|
#define ENABLE_THREAD_SAFETY 1
|
||||||
|
|
||||||
|
/* Define to nothing if C supports flexible array members, and to 1 if it does
|
||||||
|
not. That way, with a declaration like `struct s { int n; double
|
||||||
|
d[FLEXIBLE_ARRAY_MEMBER]; };', the struct hack can be used with pre-C99
|
||||||
|
compilers. When computing the size of such an object, don't use 'sizeof
|
||||||
|
(struct s)' as it overestimates the size. Use 'offsetof (struct s, d)'
|
||||||
|
instead. Don't use 'offsetof (struct s, d[0])', as this doesn't work with
|
||||||
|
MSVC and with C++ compilers. */
|
||||||
|
#define FLEXIBLE_ARRAY_MEMBER /**/
|
||||||
|
|
||||||
|
/* float4 values are passed by value if 'true', by reference if 'false' */
|
||||||
|
#define FLOAT4PASSBYVAL true
|
||||||
|
|
||||||
|
/* float8, int8, and related values are passed by value if 'true', by
|
||||||
|
reference if 'false' */
|
||||||
|
#define FLOAT8PASSBYVAL false
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `append_history' function. */
|
||||||
|
/* #undef HAVE_APPEND_HISTORY */
|
||||||
|
|
||||||
|
/* Define to 1 if you want to use atomics if available. */
|
||||||
|
#define HAVE_ATOMICS 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <atomic.h> header file. */
|
||||||
|
/* #undef HAVE_ATOMIC_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `cbrt' function. */
|
||||||
|
#define HAVE_CBRT 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `class' function. */
|
||||||
|
/* #undef HAVE_CLASS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <crtdefs.h> header file. */
|
||||||
|
/* #undef HAVE_CRTDEFS_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `crypt' function. */
|
||||||
|
#define HAVE_CRYPT 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <crypt.h> header file. */
|
||||||
|
#define HAVE_CRYPT_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `fdatasync', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#define HAVE_DECL_FDATASYNC 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `F_FULLFSYNC', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#define HAVE_DECL_F_FULLFSYNC 0
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `posix_fadvise', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#define HAVE_DECL_POSIX_FADVISE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `snprintf', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#define HAVE_DECL_SNPRINTF 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `strlcat', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#if OS_DARWIN
|
||||||
|
#define HAVE_DECL_STRLCAT 1
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `strlcpy', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#if OS_DARWIN
|
||||||
|
#define HAVE_DECL_STRLCPY 1
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `sys_siglist', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#define HAVE_DECL_SYS_SIGLIST 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `vsnprintf', and to 0 if you
|
||||||
|
don't. */
|
||||||
|
#define HAVE_DECL_VSNPRINTF 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <dld.h> header file. */
|
||||||
|
/* #undef HAVE_DLD_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <editline/history.h> header file. */
|
||||||
|
/* #undef HAVE_EDITLINE_HISTORY_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <editline/readline.h> header file. */
|
||||||
|
#define HAVE_EDITLINE_READLINE_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `fpclass' function. */
|
||||||
|
/* #undef HAVE_FPCLASS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `fp_class' function. */
|
||||||
|
/* #undef HAVE_FP_CLASS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `fp_class_d' function. */
|
||||||
|
/* #undef HAVE_FP_CLASS_D */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <fp_class.h> header file. */
|
||||||
|
/* #undef HAVE_FP_CLASS_H */
|
||||||
|
|
||||||
|
/* Define to 1 if fseeko (and presumably ftello) exists and is declared. */
|
||||||
|
#define HAVE_FSEEKO 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have __atomic_compare_exchange_n(int *, int *, int). */
|
||||||
|
/* #undef HAVE_GCC__ATOMIC_INT32_CAS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have __atomic_compare_exchange_n(int64 *, int *, int64).
|
||||||
|
*/
|
||||||
|
/* #undef HAVE_GCC__ATOMIC_INT64_CAS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have __sync_lock_test_and_set(char *) and friends. */
|
||||||
|
#define HAVE_GCC__SYNC_CHAR_TAS 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have __sync_compare_and_swap(int *, int, int). */
|
||||||
|
/* #undef HAVE_GCC__SYNC_INT32_CAS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have __sync_lock_test_and_set(int *) and friends. */
|
||||||
|
#define HAVE_GCC__SYNC_INT32_TAS 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have __sync_compare_and_swap(int64 *, int64, int64). */
|
||||||
|
/* #undef HAVE_GCC__SYNC_INT64_CAS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `getifaddrs' function. */
|
||||||
|
#define HAVE_GETIFADDRS 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `getopt' function. */
|
||||||
|
#define HAVE_GETOPT 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <getopt.h> header file. */
|
||||||
|
#define HAVE_GETOPT_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `getopt_long' function. */
|
||||||
|
#define HAVE_GETOPT_LONG 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `getpeereid' function. */
|
||||||
|
/* #undef HAVE_GETPEEREID */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `getpeerucred' function. */
|
||||||
|
/* #undef HAVE_GETPEERUCRED */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <gssapi_ext.h> header file. */
|
||||||
|
/* #undef HAVE_GSSAPI_EXT_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <gssapi/gssapi_ext.h> header file. */
|
||||||
|
/* #undef HAVE_GSSAPI_GSSAPI_EXT_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <gssapi/gssapi.h> header file. */
|
||||||
|
//#define HAVE_GSSAPI_GSSAPI_H 0
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <gssapi.h> header file. */
|
||||||
|
/* #undef HAVE_GSSAPI_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <history.h> header file. */
|
||||||
|
/* #undef HAVE_HISTORY_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `history_truncate_file' function. */
|
||||||
|
#define HAVE_HISTORY_TRUNCATE_FILE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <ieeefp.h> header file. */
|
||||||
|
/* #undef HAVE_IEEEFP_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <ifaddrs.h> header file. */
|
||||||
|
#define HAVE_IFADDRS_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `inet_aton' function. */
|
||||||
|
#define HAVE_INET_ATON 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `inet_pton' function. */
|
||||||
|
#define HAVE_INET_PTON 1
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `int64'. */
|
||||||
|
/* #undef HAVE_INT64 */
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `int8'. */
|
||||||
|
/* #undef HAVE_INT8 */
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `intptr_t'. */
|
||||||
|
#define HAVE_INTPTR_T 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <inttypes.h> header file. */
|
||||||
|
#define HAVE_INTTYPES_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the global variable 'int opterr'. */
|
||||||
|
#define HAVE_INT_OPTERR 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the global variable 'int optreset'. */
|
||||||
|
/* #undef HAVE_INT_OPTRESET */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the global variable 'int timezone'. */
|
||||||
|
#define HAVE_INT_TIMEZONE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have isinf(). */
|
||||||
|
#define HAVE_ISINF 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <langinfo.h> header file. */
|
||||||
|
#define HAVE_LANGINFO_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `crypto' library (-lcrypto). */
|
||||||
|
#define HAVE_LIBCRYPTO 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `ldap' library (-lldap). */
|
||||||
|
//#define HAVE_LIBLDAP 0
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `m' library (-lm). */
|
||||||
|
#define HAVE_LIBM 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `pam' library (-lpam). */
|
||||||
|
#define HAVE_LIBPAM 1
|
||||||
|
|
||||||
|
/* Define if you have a function readline library */
|
||||||
|
#define HAVE_LIBREADLINE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `selinux' library (-lselinux). */
|
||||||
|
/* #undef HAVE_LIBSELINUX */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `ssl' library (-lssl). */
|
||||||
|
#define HAVE_LIBSSL 0
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `wldap32' library (-lwldap32). */
|
||||||
|
/* #undef HAVE_LIBWLDAP32 */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `xml2' library (-lxml2). */
|
||||||
|
#define HAVE_LIBXML2 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `xslt' library (-lxslt). */
|
||||||
|
#define HAVE_LIBXSLT 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `z' library (-lz). */
|
||||||
|
#define HAVE_LIBZ 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `zstd' library (-lzstd). */
|
||||||
|
/* #undef HAVE_LIBZSTD */
|
||||||
|
|
||||||
|
/* Define to 1 if constants of type 'long long int' should have the suffix LL.
|
||||||
|
*/
|
||||||
|
#define HAVE_LL_CONSTANTS 1
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `locale_t'. */
|
||||||
|
#define HAVE_LOCALE_T 1
|
||||||
|
|
||||||
|
/* Define to 1 if `long int' works and is 64 bits. */
|
||||||
|
/* #undef HAVE_LONG_INT_64 */
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `long long int'. */
|
||||||
|
#define HAVE_LONG_LONG_INT 1
|
||||||
|
|
||||||
|
/* Define to 1 if `long long int' works and is 64 bits. */
|
||||||
|
#define HAVE_LONG_LONG_INT_64 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <mbarrier.h> header file. */
|
||||||
|
/* #undef HAVE_MBARRIER_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `mbstowcs_l' function. */
|
||||||
|
/* #undef HAVE_MBSTOWCS_L */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `memmove' function. */
|
||||||
|
#define HAVE_MEMMOVE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <memory.h> header file. */
|
||||||
|
#define HAVE_MEMORY_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `mkdtemp' function. */
|
||||||
|
#define HAVE_MKDTEMP 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <net/if.h> header file. */
|
||||||
|
#define HAVE_NET_IF_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <ossp/uuid.h> header file. */
|
||||||
|
/* #undef HAVE_OSSP_UUID_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <pam/pam_appl.h> header file. */
|
||||||
|
/* #undef HAVE_PAM_PAM_APPL_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `posix_fadvise' function. */
|
||||||
|
#define HAVE_POSIX_FADVISE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `preadv', and to 0 if you don't. */
|
||||||
|
/* #undef HAVE_DECL_PREADV */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the declaration of `pwritev', and to 0 if you don't. */
|
||||||
|
/* #define HAVE_DECL_PWRITEV */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `X509_get_signature_info' function. */
|
||||||
|
/* #undef HAVE_X509_GET_SIGNATURE_INFO */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the POSIX signal interface. */
|
||||||
|
#define HAVE_POSIX_SIGNALS 1
|
||||||
|
|
||||||
|
/* Define to 1 if the assembler supports PPC's LWARX mutex hint bit. */
|
||||||
|
/* #undef HAVE_PPC_LWARX_MUTEX_HINT */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `pthread_is_threaded_np' function. */
|
||||||
|
/* #undef HAVE_PTHREAD_IS_THREADED_NP */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <pwd.h> header file. */
|
||||||
|
#define HAVE_PWD_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <readline.h> header file. */
|
||||||
|
/* #undef HAVE_READLINE_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <readline/history.h> header file. */
|
||||||
|
#define HAVE_READLINE_HISTORY_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <readline/readline.h> header file. */
|
||||||
|
/* #undef HAVE_READLINE_READLINE_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `rint' function. */
|
||||||
|
#define HAVE_RINT 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `rl_completion_matches' function. */
|
||||||
|
#define HAVE_RL_COMPLETION_MATCHES 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `rl_filename_completion_function' function. */
|
||||||
|
#define HAVE_RL_FILENAME_COMPLETION_FUNCTION 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `rl_reset_screen_size' function. */
|
||||||
|
/* #undef HAVE_RL_RESET_SCREEN_SIZE */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `rl_variable_bind' function. */
|
||||||
|
#define HAVE_RL_VARIABLE_BIND 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <security/pam_appl.h> header file. */
|
||||||
|
#define HAVE_SECURITY_PAM_APPL_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `setproctitle' function. */
|
||||||
|
/* #undef HAVE_SETPROCTITLE */
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `socklen_t'. */
|
||||||
|
#define HAVE_SOCKLEN_T 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `sigprocmask' function. */
|
||||||
|
#define HAVE_SIGPROCMASK 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have sigsetjmp(). */
|
||||||
|
#define HAVE_SIGSETJMP 1
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `sig_atomic_t'. */
|
||||||
|
#define HAVE_SIG_ATOMIC_T 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `snprintf' function. */
|
||||||
|
#define HAVE_SNPRINTF 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have spinlocks. */
|
||||||
|
#define HAVE_SPINLOCKS 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `SSL_CTX_set_cert_cb' function. */
|
||||||
|
#define HAVE_SSL_CTX_SET_CERT_CB 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `SSL_CTX_set_num_tickets' function. */
|
||||||
|
/* #define HAVE_SSL_CTX_SET_NUM_TICKETS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `SSL_get_current_compression' function. */
|
||||||
|
#define HAVE_SSL_GET_CURRENT_COMPRESSION 0
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <stdint.h> header file. */
|
||||||
|
#define HAVE_STDINT_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <stdlib.h> header file. */
|
||||||
|
#define HAVE_STDLIB_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `strerror' function. */
|
||||||
|
#define HAVE_STRERROR 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `strerror_r' function. */
|
||||||
|
#define HAVE_STRERROR_R 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <strings.h> header file. */
|
||||||
|
//#define HAVE_STRINGS_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <string.h> header file. */
|
||||||
|
#define HAVE_STRING_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `strlcat' function. */
|
||||||
|
/* #undef HAVE_STRLCAT */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `strlcpy' function. */
|
||||||
|
/* #undef HAVE_STRLCPY */
|
||||||
|
|
||||||
|
#if (!OS_DARWIN)
|
||||||
|
#define HAVE_STRCHRNUL 1
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `struct option'. */
|
||||||
|
#define HAVE_STRUCT_OPTION 1
|
||||||
|
|
||||||
|
/* Define to 1 if `sa_len' is a member of `struct sockaddr'. */
|
||||||
|
/* #undef HAVE_STRUCT_SOCKADDR_SA_LEN */
|
||||||
|
|
||||||
|
/* Define to 1 if `tm_zone' is a member of `struct tm'. */
|
||||||
|
#define HAVE_STRUCT_TM_TM_ZONE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `sync_file_range' function. */
|
||||||
|
/* #undef HAVE_SYNC_FILE_RANGE */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the syslog interface. */
|
||||||
|
#define HAVE_SYSLOG 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/ioctl.h> header file. */
|
||||||
|
#define HAVE_SYS_IOCTL_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/personality.h> header file. */
|
||||||
|
/* #undef HAVE_SYS_PERSONALITY_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/poll.h> header file. */
|
||||||
|
#define HAVE_SYS_POLL_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/signalfd.h> header file. */
|
||||||
|
/* #undef HAVE_SYS_SIGNALFD_H */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/socket.h> header file. */
|
||||||
|
#define HAVE_SYS_SOCKET_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/stat.h> header file. */
|
||||||
|
#define HAVE_SYS_STAT_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/time.h> header file. */
|
||||||
|
#define HAVE_SYS_TIME_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/types.h> header file. */
|
||||||
|
#define HAVE_SYS_TYPES_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/ucred.h> header file. */
|
||||||
|
#if (OS_DARWIN || OS_FREEBSD)
|
||||||
|
#define HAVE_SYS_UCRED_H 1
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <sys/un.h> header file. */
|
||||||
|
#define _GNU_SOURCE 1 /* Needed for glibc struct ucred */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <termios.h> header file. */
|
||||||
|
#define HAVE_TERMIOS_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if your `struct tm' has `tm_zone'. Deprecated, use
|
||||||
|
`HAVE_STRUCT_TM_TM_ZONE' instead. */
|
||||||
|
#define HAVE_TM_ZONE 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `towlower' function. */
|
||||||
|
#define HAVE_TOWLOWER 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the external array `tzname'. */
|
||||||
|
#define HAVE_TZNAME 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <ucred.h> header file. */
|
||||||
|
/* #undef HAVE_UCRED_H */
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `uint64'. */
|
||||||
|
/* #undef HAVE_UINT64 */
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `uint8'. */
|
||||||
|
/* #undef HAVE_UINT8 */
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `uintptr_t'. */
|
||||||
|
#define HAVE_UINTPTR_T 1
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `union semun'. */
|
||||||
|
/* #undef HAVE_UNION_SEMUN */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <unistd.h> header file. */
|
||||||
|
#define HAVE_UNISTD_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have unix sockets. */
|
||||||
|
#define HAVE_UNIX_SOCKETS 1
|
||||||
|
|
||||||
|
/* Define to 1 if the system has the type `unsigned long long int'. */
|
||||||
|
#define HAVE_UNSIGNED_LONG_LONG_INT 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `utime' function. */
|
||||||
|
#define HAVE_UTIME 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `utimes' function. */
|
||||||
|
#define HAVE_UTIMES 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <utime.h> header file. */
|
||||||
|
#define HAVE_UTIME_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have BSD UUID support. */
|
||||||
|
/* #undef HAVE_UUID_BSD */
|
||||||
|
|
||||||
|
/* Define to 1 if you have E2FS UUID support. */
|
||||||
|
/* #undef HAVE_UUID_E2FS */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <uuid.h> header file. */
|
||||||
|
#define HAVE_UUID_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have OSSP UUID support. */
|
||||||
|
#define HAVE_UUID_OSSP 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <uuid/uuid.h> header file. */
|
||||||
|
/* #undef HAVE_UUID_UUID_H */
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler knows the visibility("hidden") attribute. */
|
||||||
|
/* #undef HAVE_VISIBILITY_ATTRIBUTE */
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `vsnprintf' function. */
|
||||||
|
#define HAVE_VSNPRINTF 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the <wchar.h> header file. */
|
||||||
|
#define HAVE_WCHAR_H 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `wcstombs' function. */
|
||||||
|
#define HAVE_WCSTOMBS 1
|
||||||
|
|
||||||
|
/* Define to 1 if you have the `wcstombs_l' function. */
|
||||||
|
/* #undef HAVE_WCSTOMBS_L */
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler understands __builtin_bswap32. */
|
||||||
|
/* #undef HAVE__BUILTIN_BSWAP32 */
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler understands __builtin_constant_p. */
|
||||||
|
#define HAVE__BUILTIN_CONSTANT_P 1
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler understands __builtin_frame_address. */
|
||||||
|
/* #undef HAVE__BUILTIN_FRAME_ADDRESS */
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler understands __builtin_types_compatible_p. */
|
||||||
|
#define HAVE__BUILTIN_TYPES_COMPATIBLE_P 1
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler understands __builtin_unreachable. */
|
||||||
|
/* #undef HAVE__BUILTIN_UNREACHABLE */
|
||||||
|
|
||||||
|
/* Define to 1 if you have __cpuid. */
|
||||||
|
/* #undef HAVE__CPUID */
|
||||||
|
|
||||||
|
/* Define to 1 if you have __get_cpuid. */
|
||||||
|
/* #undef HAVE__GET_CPUID */
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler understands _Static_assert. */
|
||||||
|
/* #undef HAVE__STATIC_ASSERT */
|
||||||
|
|
||||||
|
/* Define to 1 if your compiler understands __VA_ARGS__ in macros. */
|
||||||
|
#define HAVE__VA_ARGS 1
|
||||||
|
|
||||||
|
/* Define to the appropriate snprintf length modifier for 64-bit ints. */
|
||||||
|
#define INT64_MODIFIER "ll"
|
||||||
|
|
||||||
|
/* Define to 1 if `locale_t' requires <xlocale.h>. */
|
||||||
|
/* #undef LOCALE_T_IN_XLOCALE */
|
||||||
|
|
||||||
|
/* Define as the maximum alignment requirement of any C data type. */
|
||||||
|
#define MAXIMUM_ALIGNOF 4
|
||||||
|
|
||||||
|
/* Define bytes to use libc memset(). */
|
||||||
|
#define MEMSET_LOOP_LIMIT 1024
|
||||||
|
|
||||||
|
/* Define to the address where bug reports for this package should be sent. */
|
||||||
|
#define PACKAGE_BUGREPORT "pgsql-bugs@postgresql.org"
|
||||||
|
|
||||||
|
/* Define to the full name of this package. */
|
||||||
|
#define PACKAGE_NAME "PostgreSQL"
|
||||||
|
|
||||||
|
/* Define to the full name and version of this package. */
|
||||||
|
#define PACKAGE_STRING "PostgreSQL 9.5.4"
|
||||||
|
|
||||||
|
/* Define to the one symbol short name of this package. */
|
||||||
|
#define PACKAGE_TARNAME "postgresql"
|
||||||
|
|
||||||
|
/* Define to the home page for this package. */
|
||||||
|
#define PACKAGE_URL ""
|
||||||
|
|
||||||
|
/* Define to the version of this package. */
|
||||||
|
#define PACKAGE_VERSION "9.5.4"
|
||||||
|
|
||||||
|
/* Define to the name of a signed 128-bit integer type. */
|
||||||
|
/* #undef PG_INT128_TYPE */
|
||||||
|
|
||||||
|
/* Define to the name of a signed 64-bit integer type. */
|
||||||
|
#define PG_INT64_TYPE long long int
|
||||||
|
|
||||||
|
/* Define to the name of the default PostgreSQL service principal in Kerberos
|
||||||
|
(GSSAPI). (--with-krb-srvnam=NAME) */
|
||||||
|
#define PG_KRB_SRVNAM "postgres"
|
||||||
|
|
||||||
|
/* PostgreSQL major version as a string */
|
||||||
|
#define PG_MAJORVERSION "9.5"
|
||||||
|
|
||||||
|
/* Define to gnu_printf if compiler supports it, else printf. */
|
||||||
|
#define PG_PRINTF_ATTRIBUTE printf
|
||||||
|
|
||||||
|
/* Define to 1 if "static inline" works without unwanted warnings from
|
||||||
|
compilations where static inline functions are defined but not called. */
|
||||||
|
#define PG_USE_INLINE 1
|
||||||
|
|
||||||
|
/* PostgreSQL version as a string */
|
||||||
|
#define PG_VERSION "9.5.4"
|
||||||
|
|
||||||
|
/* PostgreSQL version as a number */
|
||||||
|
#define PG_VERSION_NUM 90504
|
||||||
|
|
||||||
|
/* A string containing the version number, platform, and C compiler */
|
||||||
|
#define PG_VERSION_STR "PostgreSQL 9.5.4 on i686-pc-linux-gnu, compiled by gcc (GCC) 4.1.2 20080704 (Red Hat 4.1.2-55), 32-bit"
|
||||||
|
|
||||||
|
/* Define to 1 to allow profiling output to be saved separately for each
|
||||||
|
process. */
|
||||||
|
/* #undef PROFILE_PID_DIR */
|
||||||
|
|
||||||
|
/* RELSEG_SIZE is the maximum number of blocks allowed in one disk file. Thus,
|
||||||
|
the maximum size of a single file is RELSEG_SIZE * BLCKSZ; relations bigger
|
||||||
|
than that are divided into multiple files. RELSEG_SIZE * BLCKSZ must be
|
||||||
|
less than your OS' limit on file size. This is often 2 GB or 4GB in a
|
||||||
|
32-bit operating system, unless you have large file support enabled. By
|
||||||
|
default, we make the limit 1 GB to avoid any possible integer-overflow
|
||||||
|
problems within the OS. A limit smaller than necessary only means we divide
|
||||||
|
a large relation into more chunks than necessary, so it seems best to err
|
||||||
|
in the direction of a small limit. A power-of-2 value is recommended to
|
||||||
|
save a few cycles in md.c, but is not absolutely required. Changing
|
||||||
|
RELSEG_SIZE requires an initdb. */
|
||||||
|
#define RELSEG_SIZE 131072
|
||||||
|
|
||||||
|
/* The size of `long', as computed by sizeof. */
|
||||||
|
#define SIZEOF_LONG 4
|
||||||
|
|
||||||
|
/* The size of `off_t', as computed by sizeof. */
|
||||||
|
#define SIZEOF_OFF_T 8
|
||||||
|
|
||||||
|
/* The size of `size_t', as computed by sizeof. */
|
||||||
|
#define SIZEOF_SIZE_T 4
|
||||||
|
|
||||||
|
/* The size of `void *', as computed by sizeof. */
|
||||||
|
#define SIZEOF_VOID_P 4
|
||||||
|
|
||||||
|
/* Define to 1 if you have the ANSI C header files. */
|
||||||
|
#define STDC_HEADERS 1
|
||||||
|
|
||||||
|
/* Define to 1 if strerror_r() returns a int. */
|
||||||
|
/* #undef STRERROR_R_INT */
|
||||||
|
|
||||||
|
/* Define to 1 if your <sys/time.h> declares `struct tm'. */
|
||||||
|
/* #undef TM_IN_SYS_TIME */
|
||||||
|
|
||||||
|
/* Define to 1 to build with assertion checks. (--enable-cassert) */
|
||||||
|
/* #undef USE_ASSERT_CHECKING */
|
||||||
|
|
||||||
|
/* Define to 1 to build with Bonjour support. (--with-bonjour) */
|
||||||
|
/* #undef USE_BONJOUR */
|
||||||
|
|
||||||
|
/* Define to 1 if you want float4 values to be passed by value.
|
||||||
|
(--enable-float4-byval) */
|
||||||
|
#define USE_FLOAT4_BYVAL 1
|
||||||
|
|
||||||
|
/* Define to 1 if you want float8, int8, etc values to be passed by value.
|
||||||
|
(--enable-float8-byval) */
|
||||||
|
/* #undef USE_FLOAT8_BYVAL */
|
||||||
|
|
||||||
|
/* Define to 1 if you want 64-bit integer timestamp and interval support.
|
||||||
|
(--enable-integer-datetimes) */
|
||||||
|
#define USE_INTEGER_DATETIMES 1
|
||||||
|
|
||||||
|
/* Define to 1 to build with LDAP support. (--with-ldap) */
|
||||||
|
//#define USE_LDAP 0
|
||||||
|
|
||||||
|
/* Define to 1 to build with XML support. (--with-libxml) */
|
||||||
|
#define USE_LIBXML 1
|
||||||
|
|
||||||
|
/* Define to 1 to use XSLT support when building contrib/xml2.
|
||||||
|
(--with-libxslt) */
|
||||||
|
#define USE_LIBXSLT 1
|
||||||
|
|
||||||
|
/* Define to select named POSIX semaphores. */
|
||||||
|
/* #undef USE_NAMED_POSIX_SEMAPHORES */
|
||||||
|
|
||||||
|
/* Define to build with OpenSSL support. (--with-openssl) */
|
||||||
|
#define USE_OPENSSL 0
|
||||||
|
|
||||||
|
#define USE_OPENSSL_RANDOM 0
|
||||||
|
|
||||||
|
#define FRONTEND 1
|
||||||
|
|
||||||
|
/* Define to 1 to build with PAM support. (--with-pam) */
|
||||||
|
#define USE_PAM 1
|
||||||
|
|
||||||
|
/* Use replacement snprintf() functions. */
|
||||||
|
/* #undef USE_REPL_SNPRINTF */
|
||||||
|
|
||||||
|
/* Define to 1 to use Intel SSE 4.2 CRC instructions with a runtime check. */
|
||||||
|
#define USE_SLICING_BY_8_CRC32C 1
|
||||||
|
|
||||||
|
/* Define to 1 use Intel SSE 4.2 CRC instructions. */
|
||||||
|
/* #undef USE_SSE42_CRC32C */
|
||||||
|
|
||||||
|
/* Define to 1 to use Intel SSSE 4.2 CRC instructions with a runtime check. */
|
||||||
|
/* #undef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK */
|
||||||
|
|
||||||
|
/* Define to select SysV-style semaphores. */
|
||||||
|
#define USE_SYSV_SEMAPHORES 1
|
||||||
|
|
||||||
|
/* Define to select SysV-style shared memory. */
|
||||||
|
#define USE_SYSV_SHARED_MEMORY 1
|
||||||
|
|
||||||
|
/* Define to select unnamed POSIX semaphores. */
|
||||||
|
/* #undef USE_UNNAMED_POSIX_SEMAPHORES */
|
||||||
|
|
||||||
|
/* Define to select Win32-style semaphores. */
|
||||||
|
/* #undef USE_WIN32_SEMAPHORES */
|
||||||
|
|
||||||
|
/* Define to select Win32-style shared memory. */
|
||||||
|
/* #undef USE_WIN32_SHARED_MEMORY */
|
||||||
|
|
||||||
|
/* Define to 1 to build with ZSTD support. (--with-zstd) */
|
||||||
|
/* #undef USE_ZSTD */
|
||||||
|
|
||||||
|
/* Define to 1 if `wcstombs_l' requires <xlocale.h>. */
|
||||||
|
/* #undef WCSTOMBS_L_IN_XLOCALE */
|
||||||
|
|
||||||
|
/* Define WORDS_BIGENDIAN to 1 if your processor stores words with the most
|
||||||
|
significant byte first (like Motorola and SPARC, unlike Intel). */
|
||||||
|
#if defined AC_APPLE_UNIVERSAL_BUILD
|
||||||
|
# if defined __BIG_ENDIAN__
|
||||||
|
# define WORDS_BIGENDIAN 1
|
||||||
|
# endif
|
||||||
|
#else
|
||||||
|
# ifndef WORDS_BIGENDIAN
|
||||||
|
/* # undef WORDS_BIGENDIAN */
|
||||||
|
# endif
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Size of a WAL file block. This need have no particular relation to BLCKSZ.
|
||||||
|
XLOG_BLCKSZ must be a power of 2, and if your system supports O_DIRECT I/O,
|
||||||
|
XLOG_BLCKSZ must be a multiple of the alignment requirement for direct-I/O
|
||||||
|
buffers, else direct I/O may fail. Changing XLOG_BLCKSZ requires an initdb.
|
||||||
|
*/
|
||||||
|
#define XLOG_BLCKSZ 8192
|
||||||
|
|
||||||
|
/* XLOG_SEG_SIZE is the size of a single WAL file. This must be a power of 2
|
||||||
|
and larger than XLOG_BLCKSZ (preferably, a great deal larger than
|
||||||
|
XLOG_BLCKSZ). Changing XLOG_SEG_SIZE requires an initdb. */
|
||||||
|
#define XLOG_SEG_SIZE (16 * 1024 * 1024)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/* Number of bits in a file offset, on hosts where this is settable. */
|
||||||
|
#define _FILE_OFFSET_BITS 64
|
||||||
|
|
||||||
|
/* Define to 1 to make fseeko visible on some hosts (e.g. glibc 2.2). */
|
||||||
|
/* #undef _LARGEFILE_SOURCE */
|
||||||
|
|
||||||
|
/* Define for large files, on AIX-style hosts. */
|
||||||
|
/* #undef _LARGE_FILES */
|
||||||
|
|
||||||
|
/* Define to `__inline__' or `__inline' if that's what the C compiler
|
||||||
|
calls it, or to nothing if 'inline' is not supported under any name. */
|
||||||
|
#ifndef __cplusplus
|
||||||
|
/* #undef inline */
|
||||||
|
#endif
|
||||||
|
|
||||||
|
/* Define to the type of a signed integer type wide enough to hold a pointer,
|
||||||
|
if such a type exists, and if the system does not define it. */
|
||||||
|
/* #undef intptr_t */
|
||||||
|
|
||||||
|
/* Define to empty if the C compiler does not understand signed types. */
|
||||||
|
/* #undef signed */
|
||||||
|
|
||||||
|
/* Define to the type of an unsigned integer type wide enough to hold a
|
||||||
|
pointer, if such a type exists, and if the system does not define it. */
|
||||||
|
/* #undef uintptr_t */
|
7
contrib/postgres-cmake/pg_config_ext.h
Normal file
7
contrib/postgres-cmake/pg_config_ext.h
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
/*
|
||||||
|
* * src/include/pg_config_ext.h.in. This is generated manually, not by
|
||||||
|
* * autoheader, since we want to limit which symbols get defined here.
|
||||||
|
* */
|
||||||
|
|
||||||
|
/* Define to the name of a signed 64-bit integer type. */
|
||||||
|
#define PG_INT64_TYPE long long int
|
34
contrib/postgres-cmake/pg_config_os.h
Normal file
34
contrib/postgres-cmake/pg_config_os.h
Normal file
@ -0,0 +1,34 @@
|
|||||||
|
#if defined(OS_DARWIN)
|
||||||
|
|
||||||
|
/* src/include/port/darwin.h */
|
||||||
|
#define __darwin__ 1
|
||||||
|
|
||||||
|
#if HAVE_DECL_F_FULLFSYNC /* not present before macOS 10.3 */
|
||||||
|
#define HAVE_FSYNC_WRITETHROUGH
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#else
|
||||||
|
/* src/include/port/linux.h */
|
||||||
|
/*
|
||||||
|
* As of July 2007, all known versions of the Linux kernel will sometimes
|
||||||
|
* return EIDRM for a shmctl() operation when EINVAL is correct (it happens
|
||||||
|
* when the low-order 15 bits of the supplied shm ID match the slot number
|
||||||
|
* assigned to a newer shmem segment). We deal with this by assuming that
|
||||||
|
* EIDRM means EINVAL in PGSharedMemoryIsInUse(). This is reasonably safe
|
||||||
|
* since in fact Linux has no excuse for ever returning EIDRM; it doesn't
|
||||||
|
* track removed segments in a way that would allow distinguishing them from
|
||||||
|
* private ones. But someday that code might get upgraded, and we'd have
|
||||||
|
* to have a kernel version test here.
|
||||||
|
*/
|
||||||
|
#define HAVE_LINUX_EIDRM_BUG
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Set the default wal_sync_method to fdatasync. With recent Linux versions,
|
||||||
|
* xlogdefs.h's normal rules will prefer open_datasync, which (a) doesn't
|
||||||
|
* perform better and (b) causes outright failures on ext4 data=journal
|
||||||
|
* filesystems, because those don't support O_DIRECT.
|
||||||
|
*/
|
||||||
|
#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC
|
||||||
|
|
||||||
|
#endif
|
||||||
|
|
12
contrib/postgres-cmake/pg_config_paths.h
Normal file
12
contrib/postgres-cmake/pg_config_paths.h
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
#define PGBINDIR "/bin"
|
||||||
|
#define PGSHAREDIR "/share"
|
||||||
|
#define SYSCONFDIR "/etc"
|
||||||
|
#define INCLUDEDIR "/include"
|
||||||
|
#define PKGINCLUDEDIR "/include"
|
||||||
|
#define INCLUDEDIRSERVER "/include/server"
|
||||||
|
#define LIBDIR "/lib"
|
||||||
|
#define PKGLIBDIR "/lib"
|
||||||
|
#define LOCALEDIR "/share/locale"
|
||||||
|
#define DOCDIR "/doc"
|
||||||
|
#define HTMLDIR "/doc"
|
||||||
|
#define MANDIR "/man"
|
0
contrib/postgres-cmake/utils/errcodes.h
Normal file
0
contrib/postgres-cmake/utils/errcodes.h
Normal file
@ -14,5 +14,6 @@ git config submodule."contrib/icu".update '!../sparse-checkout/update-icu.sh'
|
|||||||
git config submodule."contrib/boost".update '!../sparse-checkout/update-boost.sh'
|
git config submodule."contrib/boost".update '!../sparse-checkout/update-boost.sh'
|
||||||
git config submodule."contrib/aws-s2n-tls".update '!../sparse-checkout/update-aws-s2n-tls.sh'
|
git config submodule."contrib/aws-s2n-tls".update '!../sparse-checkout/update-aws-s2n-tls.sh'
|
||||||
git config submodule."contrib/protobuf".update '!../sparse-checkout/update-protobuf.sh'
|
git config submodule."contrib/protobuf".update '!../sparse-checkout/update-protobuf.sh'
|
||||||
|
git config submodule."contrib/postgres".update '!../sparse-checkout/update-postgres.sh'
|
||||||
git config submodule."contrib/libxml2".update '!../sparse-checkout/update-libxml2.sh'
|
git config submodule."contrib/libxml2".update '!../sparse-checkout/update-libxml2.sh'
|
||||||
git config submodule."contrib/brotli".update '!../sparse-checkout/update-brotli.sh'
|
git config submodule."contrib/brotli".update '!../sparse-checkout/update-brotli.sh'
|
||||||
|
16
contrib/sparse-checkout/update-postgres.sh
Executable file
16
contrib/sparse-checkout/update-postgres.sh
Executable file
@ -0,0 +1,16 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
echo "Using sparse checkout for postgres"
|
||||||
|
|
||||||
|
FILES_TO_CHECKOUT=$(git rev-parse --git-dir)/info/sparse-checkout
|
||||||
|
echo '!/*' > $FILES_TO_CHECKOUT
|
||||||
|
echo '/src/interfaces/libpq/*' >> $FILES_TO_CHECKOUT
|
||||||
|
echo '!/src/interfaces/libpq/*/*' >> $FILES_TO_CHECKOUT
|
||||||
|
echo '/src/common/*' >> $FILES_TO_CHECKOUT
|
||||||
|
echo '!/src/port/*/*' >> $FILES_TO_CHECKOUT
|
||||||
|
echo '/src/port/*' >> $FILES_TO_CHECKOUT
|
||||||
|
echo '/src/include/*' >> $FILES_TO_CHECKOUT
|
||||||
|
|
||||||
|
git config core.sparsecheckout true
|
||||||
|
git checkout $1
|
||||||
|
git read-tree -mu HEAD
|
@ -155,6 +155,12 @@ Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.htm
|
|||||||
|
|
||||||
Sets a comma-separated list of PostgreSQL database tables, which will be replicated via [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) database engine.
|
Sets a comma-separated list of PostgreSQL database tables, which will be replicated via [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) database engine.
|
||||||
|
|
||||||
|
Each table can have subset of replicated columns in brackets. If subset of columns is omitted, then all columns for table will be replicated.
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
materialized_postgresql_tables_list = 'table1(co1, col2),table2,table3(co3, col5, col7)
|
||||||
|
```
|
||||||
|
|
||||||
Default value: empty list — means whole PostgreSQL database will be replicated.
|
Default value: empty list — means whole PostgreSQL database will be replicated.
|
||||||
|
|
||||||
### `materialized_postgresql_schema` {#materialized-postgresql-schema}
|
### `materialized_postgresql_schema` {#materialized-postgresql-schema}
|
||||||
|
@ -167,7 +167,7 @@ If you want to change the target table by using `ALTER`, we recommend disabling
|
|||||||
|
|
||||||
- `_subject` - NATS message subject. Data type: `String`.
|
- `_subject` - NATS message subject. Data type: `String`.
|
||||||
|
|
||||||
Additional virtual columns when `kafka_handle_error_mode='stream'`:
|
Additional virtual columns when `nats_handle_error_mode='stream'`:
|
||||||
|
|
||||||
- `_raw_message` - Raw message that couldn't be parsed successfully. Data type: `Nullable(String)`.
|
- `_raw_message` - Raw message that couldn't be parsed successfully. Data type: `Nullable(String)`.
|
||||||
- `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`.
|
- `_error` - Exception message happened during failed parsing. Data type: `Nullable(String)`.
|
||||||
|
@ -3150,3 +3150,15 @@ Default value: "default"
|
|||||||
|
|
||||||
**See Also**
|
**See Also**
|
||||||
- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
|
- [Workload Scheduling](/docs/en/operations/workload-scheduling.md)
|
||||||
|
|
||||||
|
## max_authentication_methods_per_user {#max_authentication_methods_per_user}
|
||||||
|
|
||||||
|
The maximum number of authentication methods a user can be created with or altered to.
|
||||||
|
Changing this setting does not affect existing users. Create/alter authentication-related queries will fail if they exceed the limit specified in this setting.
|
||||||
|
Non authentication create/alter queries will succeed.
|
||||||
|
|
||||||
|
Type: UInt64
|
||||||
|
|
||||||
|
Default value: 100
|
||||||
|
|
||||||
|
Zero means unlimited
|
||||||
|
@ -9,7 +9,7 @@ Computes an approximate [quantile](https://en.wikipedia.org/wiki/Quantile) of a
|
|||||||
**Syntax**
|
**Syntax**
|
||||||
|
|
||||||
``` sql
|
``` sql
|
||||||
quantileDDsketch[relative_accuracy, (level)](expr)
|
quantileDD(relative_accuracy, [level])(expr)
|
||||||
```
|
```
|
||||||
|
|
||||||
**Arguments**
|
**Arguments**
|
||||||
|
@ -2088,13 +2088,14 @@ Calculate AUC (Area Under the Curve, which is a concept in machine learning, see
|
|||||||
**Syntax**
|
**Syntax**
|
||||||
|
|
||||||
``` sql
|
``` sql
|
||||||
arrayAUC(arr_scores, arr_labels)
|
arrayAUC(arr_scores, arr_labels[, scale])
|
||||||
```
|
```
|
||||||
|
|
||||||
**Arguments**
|
**Arguments**
|
||||||
|
|
||||||
- `arr_scores` — scores prediction model gives.
|
- `arr_scores` — scores prediction model gives.
|
||||||
- `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negative sample.
|
- `arr_labels` — labels of samples, usually 1 for positive sample and 0 for negative sample.
|
||||||
|
- `scale` - Optional. Wether to return the normalized area. Default value: true. [Bool]
|
||||||
|
|
||||||
**Returned value**
|
**Returned value**
|
||||||
|
|
||||||
|
@ -12,9 +12,10 @@ Syntax:
|
|||||||
``` sql
|
``` sql
|
||||||
ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
ALTER USER [IF EXISTS] name1 [ON CLUSTER cluster_name1] [RENAME TO new_name1]
|
||||||
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
[, name2 [ON CLUSTER cluster_name2] [RENAME TO new_name2] ...]
|
||||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}]
|
[NOT IDENTIFIED | IDENTIFIED | ADD IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'}]
|
||||||
[[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
[[ADD | DROP] HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||||
[VALID UNTIL datetime]
|
[VALID UNTIL datetime]
|
||||||
|
[RESET AUTHENTICATION METHODS TO NEW]
|
||||||
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
[DEFAULT ROLE role [,...] | ALL | ALL EXCEPT role [,...] ]
|
||||||
[GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
[GRANTEES {user | role | ANY | NONE} [,...] [EXCEPT {user | role} [,...]]]
|
||||||
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]
|
[SETTINGS variable [= value] [MIN [=] min_value] [MAX [=] max_value] [READONLY | WRITABLE] | PROFILE 'profile_name'] [,...]
|
||||||
@ -62,3 +63,31 @@ Allows the user with `john` account to grant his privileges to the user with `ja
|
|||||||
``` sql
|
``` sql
|
||||||
ALTER USER john GRANTEES jack;
|
ALTER USER john GRANTEES jack;
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Adds new authentication methods to the user while keeping the existing ones:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
ALTER USER user1 ADD IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3'
|
||||||
|
```
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped.
|
||||||
|
2. `no_password` can not co-exist with other authentication methods for security reasons.
|
||||||
|
Because of that, it is not possible to `ADD` a `no_password` authentication method. The below query will throw an error:
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
ALTER USER user1 ADD IDENTIFIED WITH no_password
|
||||||
|
```
|
||||||
|
|
||||||
|
If you want to drop authentication methods for a user and rely on `no_password`, you must specify in the below replacing form.
|
||||||
|
|
||||||
|
Reset authentication methods and adds the ones specified in the query (effect of leading IDENTIFIED without the ADD keyword):
|
||||||
|
|
||||||
|
``` sql
|
||||||
|
ALTER USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3'
|
||||||
|
```
|
||||||
|
|
||||||
|
Reset authentication methods and keep the most recent added one:
|
||||||
|
``` sql
|
||||||
|
ALTER USER user1 RESET AUTHENTICATION METHODS TO NEW
|
||||||
|
```
|
||||||
|
@ -15,6 +15,7 @@ CREATE USER [IF NOT EXISTS | OR REPLACE] name1 [ON CLUSTER cluster_name1]
|
|||||||
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}]
|
[NOT IDENTIFIED | IDENTIFIED {[WITH {no_password | plaintext_password | sha256_password | sha256_hash | double_sha1_password | double_sha1_hash}] BY {'password' | 'hash'}} | {WITH ldap SERVER 'server_name'} | {WITH kerberos [REALM 'realm']} | {WITH ssl_certificate CN 'common_name' | SAN 'TYPE:subject_alt_name'} | {WITH ssh_key BY KEY 'public_key' TYPE 'ssh-rsa|...'} | {WITH http SERVER 'server_name' [SCHEME 'Basic']}]
|
||||||
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
[HOST {LOCAL | NAME 'name' | REGEXP 'name_regexp' | IP 'address' | LIKE 'pattern'} [,...] | ANY | NONE]
|
||||||
[VALID UNTIL datetime]
|
[VALID UNTIL datetime]
|
||||||
|
[RESET AUTHENTICATION METHODS TO NEW]
|
||||||
[IN access_storage_type]
|
[IN access_storage_type]
|
||||||
[DEFAULT ROLE role [,...]]
|
[DEFAULT ROLE role [,...]]
|
||||||
[DEFAULT DATABASE database | NONE]
|
[DEFAULT DATABASE database | NONE]
|
||||||
@ -144,6 +145,17 @@ In ClickHouse Cloud, by default, passwords must meet the following complexity re
|
|||||||
|
|
||||||
The available password types are: `plaintext_password`, `sha256_password`, `double_sha1_password`.
|
The available password types are: `plaintext_password`, `sha256_password`, `double_sha1_password`.
|
||||||
|
|
||||||
|
7. Multiple authentication methods can be specified:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
CREATE USER user1 IDENTIFIED WITH plaintext_password by '1', bcrypt_password by '2', plaintext_password by '3''
|
||||||
|
```
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
1. Older versions of ClickHouse might not support the syntax of multiple authentication methods. Therefore, if the ClickHouse server contains such users and is downgraded to a version that does not support it, such users will become unusable and some user related operations will be broken. In order to downgrade gracefully, one must set all users to contain a single authentication method prior to downgrading. Alternatively, if the server was downgraded without the proper procedure, the faulty users should be dropped.
|
||||||
|
2. `no_password` can not co-exist with other authentication methods for security reasons. Therefore, you can only specify
|
||||||
|
`no_password` if it is the only authentication method in the query.
|
||||||
|
|
||||||
## User Host
|
## User Host
|
||||||
|
|
||||||
User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways:
|
User host is a host from which a connection to ClickHouse server could be established. The host can be specified in the `HOST` query section in the following ways:
|
||||||
|
@ -29,6 +29,7 @@ namespace DB
|
|||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
{
|
{
|
||||||
extern const int CANNOT_RESTORE_TABLE;
|
extern const int CANNOT_RESTORE_TABLE;
|
||||||
|
extern const int ACCESS_ENTITY_ALREADY_EXISTS;
|
||||||
extern const int LOGICAL_ERROR;
|
extern const int LOGICAL_ERROR;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -175,9 +176,46 @@ namespace
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
std::unordered_map<UUID, UUID> resolveDependencies(const std::unordered_map<UUID, std::pair<String, AccessEntityType>> & dependencies, const AccessControl & access_control, bool allow_unresolved_dependencies)
|
/// Checks if new entities (which we're going to restore) already exist,
|
||||||
|
/// and either skips them or throws an exception depending on the restore settings.
|
||||||
|
void checkExistingEntities(std::vector<std::pair<UUID, AccessEntityPtr>> & entities,
|
||||||
|
std::unordered_map<UUID, UUID> & old_to_new_id,
|
||||||
|
const AccessControl & access_control,
|
||||||
|
RestoreAccessCreationMode creation_mode)
|
||||||
|
{
|
||||||
|
if (creation_mode == RestoreAccessCreationMode::kReplace)
|
||||||
|
return;
|
||||||
|
|
||||||
|
auto should_skip = [&](const std::pair<UUID, AccessEntityPtr> & id_and_entity)
|
||||||
|
{
|
||||||
|
const auto & id = id_and_entity.first;
|
||||||
|
const auto & entity = *id_and_entity.second;
|
||||||
|
auto existing_id = access_control.find(entity.getType(), entity.getName());
|
||||||
|
if (!existing_id)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
else if (creation_mode == RestoreAccessCreationMode::kCreateIfNotExists)
|
||||||
|
{
|
||||||
|
old_to_new_id[id] = *existing_id;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
throw Exception(ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS, "Cannot restore {} because it already exists", entity.formatTypeWithName());
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
std::erase_if(entities, should_skip);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// If new entities (which we're going to restore) depend on other entities which are not going to be restored or not present in the backup
|
||||||
|
/// then we should try to replace those dependencies with already existing entities.
|
||||||
|
void resolveDependencies(const std::unordered_map<UUID, std::pair<String, AccessEntityType>> & dependencies,
|
||||||
|
std::unordered_map<UUID, UUID> & old_to_new_ids,
|
||||||
|
const AccessControl & access_control,
|
||||||
|
bool allow_unresolved_dependencies)
|
||||||
{
|
{
|
||||||
std::unordered_map<UUID, UUID> old_to_new_ids;
|
|
||||||
for (const auto & [id, name_and_type] : dependencies)
|
for (const auto & [id, name_and_type] : dependencies)
|
||||||
{
|
{
|
||||||
std::optional<UUID> new_id;
|
std::optional<UUID> new_id;
|
||||||
@ -188,9 +226,9 @@ namespace
|
|||||||
if (new_id)
|
if (new_id)
|
||||||
old_to_new_ids.emplace(id, *new_id);
|
old_to_new_ids.emplace(id, *new_id);
|
||||||
}
|
}
|
||||||
return old_to_new_ids;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Generates random IDs for the new entities.
|
||||||
void generateRandomIDs(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, std::unordered_map<UUID, UUID> & old_to_new_ids)
|
void generateRandomIDs(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
{
|
{
|
||||||
Poco::UUIDGenerator generator;
|
Poco::UUIDGenerator generator;
|
||||||
@ -203,27 +241,12 @@ namespace
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
void replaceDependencies(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
/// Updates dependencies of the new entities using a specified map.
|
||||||
|
void replaceDependencies(std::vector<std::pair<UUID, AccessEntityPtr>> & entities,
|
||||||
|
const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
{
|
{
|
||||||
for (auto & entity : entities | boost::adaptors::map_values)
|
for (auto & entity : entities | boost::adaptors::map_values)
|
||||||
{
|
IAccessEntity::replaceDependencies(entity, old_to_new_ids);
|
||||||
bool need_replace = false;
|
|
||||||
for (const auto & dependency : entity->findDependencies())
|
|
||||||
{
|
|
||||||
if (old_to_new_ids.contains(dependency))
|
|
||||||
{
|
|
||||||
need_replace = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!need_replace)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
auto new_entity = entity->clone();
|
|
||||||
new_entity->replaceDependencies(old_to_new_ids);
|
|
||||||
entity = new_entity;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
AccessRightsElements getRequiredAccessToRestore(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities)
|
AccessRightsElements getRequiredAccessToRestore(const std::vector<std::pair<UUID, AccessEntityPtr>> & entities)
|
||||||
@ -314,7 +337,9 @@ std::pair<String, BackupEntryPtr> makeBackupEntryForAccess(
|
|||||||
|
|
||||||
AccessRestorerFromBackup::AccessRestorerFromBackup(
|
AccessRestorerFromBackup::AccessRestorerFromBackup(
|
||||||
const BackupPtr & backup_, const RestoreSettings & restore_settings_)
|
const BackupPtr & backup_, const RestoreSettings & restore_settings_)
|
||||||
: backup(backup_), allow_unresolved_access_dependencies(restore_settings_.allow_unresolved_access_dependencies)
|
: backup(backup_)
|
||||||
|
, creation_mode(restore_settings_.create_access)
|
||||||
|
, allow_unresolved_dependencies(restore_settings_.allow_unresolved_access_dependencies)
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -362,7 +387,9 @@ std::vector<std::pair<UUID, AccessEntityPtr>> AccessRestorerFromBackup::getAcces
|
|||||||
{
|
{
|
||||||
auto new_entities = entities;
|
auto new_entities = entities;
|
||||||
|
|
||||||
auto old_to_new_ids = resolveDependencies(dependencies, access_control, allow_unresolved_access_dependencies);
|
std::unordered_map<UUID, UUID> old_to_new_ids;
|
||||||
|
checkExistingEntities(new_entities, old_to_new_ids, access_control, creation_mode);
|
||||||
|
resolveDependencies(dependencies, old_to_new_ids, access_control, allow_unresolved_dependencies);
|
||||||
generateRandomIDs(new_entities, old_to_new_ids);
|
generateRandomIDs(new_entities, old_to_new_ids);
|
||||||
replaceDependencies(new_entities, old_to_new_ids);
|
replaceDependencies(new_entities, old_to_new_ids);
|
||||||
|
|
||||||
|
@ -17,6 +17,7 @@ using BackupPtr = std::shared_ptr<const IBackup>;
|
|||||||
class IBackupEntry;
|
class IBackupEntry;
|
||||||
using BackupEntryPtr = std::shared_ptr<const IBackupEntry>;
|
using BackupEntryPtr = std::shared_ptr<const IBackupEntry>;
|
||||||
struct RestoreSettings;
|
struct RestoreSettings;
|
||||||
|
enum class RestoreAccessCreationMode : uint8_t;
|
||||||
|
|
||||||
|
|
||||||
/// Makes a backup of access entities of a specified type.
|
/// Makes a backup of access entities of a specified type.
|
||||||
@ -45,7 +46,8 @@ public:
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
BackupPtr backup;
|
BackupPtr backup;
|
||||||
bool allow_unresolved_access_dependencies = false;
|
RestoreAccessCreationMode creation_mode;
|
||||||
|
bool allow_unresolved_dependencies = false;
|
||||||
std::vector<std::pair<UUID, AccessEntityPtr>> entities;
|
std::vector<std::pair<UUID, AccessEntityPtr>> entities;
|
||||||
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
|
std::unordered_map<UUID, std::pair<String, AccessEntityType>> dependencies;
|
||||||
std::unordered_set<String> data_paths;
|
std::unordered_set<String> data_paths;
|
||||||
|
@ -544,9 +544,9 @@ scope_guard AccessControl::subscribeForChanges(const std::vector<UUID> & ids, co
|
|||||||
return changes_notifier->subscribeForChanges(ids, handler);
|
return changes_notifier->subscribeForChanges(ids, handler);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
|
bool AccessControl::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
if (MultipleAccessStorage::insertImpl(id, entity, replace_if_exists, throw_if_exists))
|
if (MultipleAccessStorage::insertImpl(id, entity, replace_if_exists, throw_if_exists, conflicting_id))
|
||||||
{
|
{
|
||||||
changes_notifier->sendNotifications();
|
changes_notifier->sendNotifications();
|
||||||
return true;
|
return true;
|
||||||
|
@ -243,7 +243,7 @@ private:
|
|||||||
class CustomSettingsPrefixes;
|
class CustomSettingsPrefixes;
|
||||||
class PasswordComplexityRules;
|
class PasswordComplexityRules;
|
||||||
|
|
||||||
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
|
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
|
||||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||||
|
|
||||||
|
@ -82,7 +82,7 @@ AccessEntityPtr deserializeAccessEntityImpl(const String & definition)
|
|||||||
if (res)
|
if (res)
|
||||||
throw Exception(ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION, "Two access entities attached in the same file");
|
throw Exception(ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION, "Two access entities attached in the same file");
|
||||||
res = user = std::make_unique<User>();
|
res = user = std::make_unique<User>();
|
||||||
InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true);
|
InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query, /* allow_no_password = */ true, /* allow_plaintext_password = */ true, /* max_number_of_authentication_methods = zero is unlimited*/ 0);
|
||||||
}
|
}
|
||||||
else if (auto * create_role_query = query->as<ASTCreateRoleQuery>())
|
else if (auto * create_role_query = query->as<ASTCreateRoleQuery>())
|
||||||
{
|
{
|
||||||
|
@ -14,11 +14,6 @@
|
|||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
namespace ErrorCodes
|
|
||||||
{
|
|
||||||
extern const int NOT_IMPLEMENTED;
|
|
||||||
extern const int SUPPORT_IS_DISABLED;
|
|
||||||
}
|
|
||||||
|
|
||||||
namespace
|
namespace
|
||||||
{
|
{
|
||||||
@ -84,165 +79,98 @@ namespace
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
bool checkKerberosAuthentication(
|
||||||
|
const GSSAcceptorContext * gss_acceptor_context,
|
||||||
|
const AuthenticationData & authentication_method,
|
||||||
|
const ExternalAuthenticators & external_authenticators)
|
||||||
|
{
|
||||||
|
return authentication_method.getType() == AuthenticationType::KERBEROS
|
||||||
|
&& external_authenticators.checkKerberosCredentials(authentication_method.getKerberosRealm(), *gss_acceptor_context);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
bool checkMySQLAuthentication(
|
||||||
|
const MySQLNative41Credentials * mysql_credentials,
|
||||||
|
const AuthenticationData & authentication_method)
|
||||||
|
{
|
||||||
|
switch (authentication_method.getType())
|
||||||
|
{
|
||||||
|
case AuthenticationType::PLAINTEXT_PASSWORD:
|
||||||
|
return checkPasswordPlainTextMySQL(
|
||||||
|
mysql_credentials->getScramble(),
|
||||||
|
mysql_credentials->getScrambledPassword(),
|
||||||
|
authentication_method.getPasswordHashBinary());
|
||||||
|
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
||||||
|
return checkPasswordDoubleSHA1MySQL(
|
||||||
|
mysql_credentials->getScramble(),
|
||||||
|
mysql_credentials->getScrambledPassword(),
|
||||||
|
authentication_method.getPasswordHashBinary());
|
||||||
|
default:
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
bool Authentication::areCredentialsValid(
|
bool checkBasicAuthentication(
|
||||||
const Credentials & credentials,
|
const BasicCredentials * basic_credentials,
|
||||||
const AuthenticationData & auth_data,
|
const AuthenticationData & authentication_method,
|
||||||
const ExternalAuthenticators & external_authenticators,
|
const ExternalAuthenticators & external_authenticators,
|
||||||
SettingsChanges & settings)
|
SettingsChanges & settings)
|
||||||
{
|
{
|
||||||
if (!credentials.isReady())
|
switch (authentication_method.getType())
|
||||||
return false;
|
|
||||||
|
|
||||||
if (const auto * gss_acceptor_context = typeid_cast<const GSSAcceptorContext *>(&credentials))
|
|
||||||
{
|
|
||||||
switch (auth_data.getType())
|
|
||||||
{
|
{
|
||||||
case AuthenticationType::NO_PASSWORD:
|
case AuthenticationType::NO_PASSWORD:
|
||||||
case AuthenticationType::PLAINTEXT_PASSWORD:
|
|
||||||
case AuthenticationType::SHA256_PASSWORD:
|
|
||||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
|
||||||
case AuthenticationType::BCRYPT_PASSWORD:
|
|
||||||
case AuthenticationType::LDAP:
|
|
||||||
case AuthenticationType::HTTP:
|
|
||||||
throw Authentication::Require<BasicCredentials>("ClickHouse Basic Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::JWT:
|
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
|
|
||||||
|
|
||||||
case AuthenticationType::KERBEROS:
|
|
||||||
return external_authenticators.checkKerberosCredentials(auth_data.getKerberosRealm(), *gss_acceptor_context);
|
|
||||||
|
|
||||||
case AuthenticationType::SSL_CERTIFICATE:
|
|
||||||
throw Authentication::Require<BasicCredentials>("ClickHouse X.509 Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::SSH_KEY:
|
|
||||||
#if USE_SSH
|
|
||||||
throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
|
|
||||||
#else
|
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
case AuthenticationType::MAX:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (const auto * mysql_credentials = typeid_cast<const MySQLNative41Credentials *>(&credentials))
|
|
||||||
{
|
{
|
||||||
switch (auth_data.getType())
|
|
||||||
{
|
|
||||||
case AuthenticationType::NO_PASSWORD:
|
|
||||||
return true; // N.B. even if the password is not empty!
|
return true; // N.B. even if the password is not empty!
|
||||||
|
|
||||||
case AuthenticationType::PLAINTEXT_PASSWORD:
|
|
||||||
return checkPasswordPlainTextMySQL(mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), auth_data.getPasswordHashBinary());
|
|
||||||
|
|
||||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
|
||||||
return checkPasswordDoubleSHA1MySQL(mysql_credentials->getScramble(), mysql_credentials->getScrambledPassword(), auth_data.getPasswordHashBinary());
|
|
||||||
|
|
||||||
case AuthenticationType::SHA256_PASSWORD:
|
|
||||||
case AuthenticationType::BCRYPT_PASSWORD:
|
|
||||||
case AuthenticationType::LDAP:
|
|
||||||
case AuthenticationType::KERBEROS:
|
|
||||||
case AuthenticationType::HTTP:
|
|
||||||
throw Authentication::Require<BasicCredentials>("ClickHouse Basic Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::SSL_CERTIFICATE:
|
|
||||||
throw Authentication::Require<BasicCredentials>("ClickHouse X.509 Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::JWT:
|
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
|
|
||||||
|
|
||||||
case AuthenticationType::SSH_KEY:
|
|
||||||
#if USE_SSH
|
|
||||||
throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
|
|
||||||
#else
|
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
case AuthenticationType::MAX:
|
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
if (const auto * basic_credentials = typeid_cast<const BasicCredentials *>(&credentials))
|
|
||||||
{
|
|
||||||
switch (auth_data.getType())
|
|
||||||
{
|
|
||||||
case AuthenticationType::NO_PASSWORD:
|
|
||||||
return true; // N.B. even if the password is not empty!
|
|
||||||
|
|
||||||
case AuthenticationType::PLAINTEXT_PASSWORD:
|
case AuthenticationType::PLAINTEXT_PASSWORD:
|
||||||
return checkPasswordPlainText(basic_credentials->getPassword(), auth_data.getPasswordHashBinary());
|
|
||||||
|
|
||||||
case AuthenticationType::SHA256_PASSWORD:
|
|
||||||
return checkPasswordSHA256(basic_credentials->getPassword(), auth_data.getPasswordHashBinary(), auth_data.getSalt());
|
|
||||||
|
|
||||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
|
||||||
return checkPasswordDoubleSHA1(basic_credentials->getPassword(), auth_data.getPasswordHashBinary());
|
|
||||||
|
|
||||||
case AuthenticationType::LDAP:
|
|
||||||
return external_authenticators.checkLDAPCredentials(auth_data.getLDAPServerName(), *basic_credentials);
|
|
||||||
|
|
||||||
case AuthenticationType::KERBEROS:
|
|
||||||
throw Authentication::Require<GSSAcceptorContext>(auth_data.getKerberosRealm());
|
|
||||||
|
|
||||||
case AuthenticationType::SSL_CERTIFICATE:
|
|
||||||
throw Authentication::Require<BasicCredentials>("ClickHouse X.509 Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::SSH_KEY:
|
|
||||||
#if USE_SSH
|
|
||||||
throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
|
|
||||||
#else
|
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
|
|
||||||
#endif
|
|
||||||
|
|
||||||
case AuthenticationType::JWT:
|
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
|
|
||||||
|
|
||||||
case AuthenticationType::BCRYPT_PASSWORD:
|
|
||||||
return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary());
|
|
||||||
|
|
||||||
case AuthenticationType::HTTP:
|
|
||||||
switch (auth_data.getHTTPAuthenticationScheme())
|
|
||||||
{
|
{
|
||||||
case HTTPAuthenticationScheme::BASIC:
|
return checkPasswordPlainText(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary());
|
||||||
|
}
|
||||||
|
case AuthenticationType::SHA256_PASSWORD:
|
||||||
|
{
|
||||||
|
return checkPasswordSHA256(
|
||||||
|
basic_credentials->getPassword(), authentication_method.getPasswordHashBinary(), authentication_method.getSalt());
|
||||||
|
}
|
||||||
|
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
||||||
|
{
|
||||||
|
return checkPasswordDoubleSHA1(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary());
|
||||||
|
}
|
||||||
|
case AuthenticationType::LDAP:
|
||||||
|
{
|
||||||
|
return external_authenticators.checkLDAPCredentials(authentication_method.getLDAPServerName(), *basic_credentials);
|
||||||
|
}
|
||||||
|
case AuthenticationType::BCRYPT_PASSWORD:
|
||||||
|
{
|
||||||
|
return checkPasswordBcrypt(basic_credentials->getPassword(), authentication_method.getPasswordHashBinary());
|
||||||
|
}
|
||||||
|
case AuthenticationType::HTTP:
|
||||||
|
{
|
||||||
|
if (authentication_method.getHTTPAuthenticationScheme() == HTTPAuthenticationScheme::BASIC)
|
||||||
|
{
|
||||||
return external_authenticators.checkHTTPBasicCredentials(
|
return external_authenticators.checkHTTPBasicCredentials(
|
||||||
auth_data.getHTTPAuthenticationServerName(), *basic_credentials, settings);
|
authentication_method.getHTTPAuthenticationServerName(), *basic_credentials, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
case AuthenticationType::MAX:
|
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
default:
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (const auto * ssl_certificate_credentials = typeid_cast<const SSLCertificateCredentials *>(&credentials))
|
return false;
|
||||||
{
|
}
|
||||||
switch (auth_data.getType())
|
|
||||||
{
|
|
||||||
case AuthenticationType::NO_PASSWORD:
|
|
||||||
case AuthenticationType::PLAINTEXT_PASSWORD:
|
|
||||||
case AuthenticationType::SHA256_PASSWORD:
|
|
||||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
|
||||||
case AuthenticationType::BCRYPT_PASSWORD:
|
|
||||||
case AuthenticationType::LDAP:
|
|
||||||
case AuthenticationType::HTTP:
|
|
||||||
throw Authentication::Require<BasicCredentials>("ClickHouse Basic Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::JWT:
|
bool checkSSLCertificateAuthentication(
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
|
const SSLCertificateCredentials * ssl_certificate_credentials,
|
||||||
|
const AuthenticationData & authentication_method)
|
||||||
case AuthenticationType::KERBEROS:
|
|
||||||
throw Authentication::Require<GSSAcceptorContext>(auth_data.getKerberosRealm());
|
|
||||||
|
|
||||||
case AuthenticationType::SSL_CERTIFICATE:
|
|
||||||
{
|
{
|
||||||
|
if (AuthenticationType::SSL_CERTIFICATE != authentication_method.getType())
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
for (SSLCertificateSubjects::Type type : {SSLCertificateSubjects::Type::CN, SSLCertificateSubjects::Type::SAN})
|
for (SSLCertificateSubjects::Type type : {SSLCertificateSubjects::Type::CN, SSLCertificateSubjects::Type::SAN})
|
||||||
{
|
{
|
||||||
for (const auto & subject : auth_data.getSSLCertificateSubjects().at(type))
|
for (const auto & subject : authentication_method.getSSLCertificateSubjects().at(type))
|
||||||
{
|
{
|
||||||
if (ssl_certificate_credentials->getSSLCertificateSubjects().at(type).contains(subject))
|
if (ssl_certificate_credentials->getSSLCertificateSubjects().at(type).contains(subject))
|
||||||
return true;
|
return true;
|
||||||
@ -267,56 +195,61 @@ bool Authentication::areCredentialsValid(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
case AuthenticationType::SSH_KEY:
|
|
||||||
#if USE_SSH
|
#if USE_SSH
|
||||||
throw Authentication::Require<SshCredentials>("SSH Keys Authentication");
|
bool checkSshAuthentication(
|
||||||
#else
|
const SshCredentials * ssh_credentials,
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
|
const AuthenticationData & authentication_method)
|
||||||
#endif
|
{
|
||||||
|
return AuthenticationType::SSH_KEY == authentication_method.getType()
|
||||||
case AuthenticationType::MAX:
|
&& checkSshSignature(authentication_method.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal());
|
||||||
break;
|
|
||||||
}
|
}
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
bool Authentication::areCredentialsValid(
|
||||||
|
const Credentials & credentials,
|
||||||
|
const AuthenticationData & authentication_method,
|
||||||
|
const ExternalAuthenticators & external_authenticators,
|
||||||
|
SettingsChanges & settings)
|
||||||
|
{
|
||||||
|
if (!credentials.isReady())
|
||||||
|
return false;
|
||||||
|
|
||||||
|
if (const auto * gss_acceptor_context = typeid_cast<const GSSAcceptorContext *>(&credentials))
|
||||||
|
{
|
||||||
|
return checkKerberosAuthentication(gss_acceptor_context, authentication_method, external_authenticators);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (const auto * mysql_credentials = typeid_cast<const MySQLNative41Credentials *>(&credentials))
|
||||||
|
{
|
||||||
|
return checkMySQLAuthentication(mysql_credentials, authentication_method);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (const auto * basic_credentials = typeid_cast<const BasicCredentials *>(&credentials))
|
||||||
|
{
|
||||||
|
return checkBasicAuthentication(basic_credentials, authentication_method, external_authenticators, settings);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (const auto * ssl_certificate_credentials = typeid_cast<const SSLCertificateCredentials *>(&credentials))
|
||||||
|
{
|
||||||
|
return checkSSLCertificateAuthentication(ssl_certificate_credentials, authentication_method);
|
||||||
}
|
}
|
||||||
|
|
||||||
#if USE_SSH
|
#if USE_SSH
|
||||||
if (const auto * ssh_credentials = typeid_cast<const SshCredentials *>(&credentials))
|
if (const auto * ssh_credentials = typeid_cast<const SshCredentials *>(&credentials))
|
||||||
{
|
{
|
||||||
switch (auth_data.getType())
|
return checkSshAuthentication(ssh_credentials, authentication_method);
|
||||||
{
|
|
||||||
case AuthenticationType::NO_PASSWORD:
|
|
||||||
case AuthenticationType::PLAINTEXT_PASSWORD:
|
|
||||||
case AuthenticationType::SHA256_PASSWORD:
|
|
||||||
case AuthenticationType::DOUBLE_SHA1_PASSWORD:
|
|
||||||
case AuthenticationType::BCRYPT_PASSWORD:
|
|
||||||
case AuthenticationType::LDAP:
|
|
||||||
case AuthenticationType::HTTP:
|
|
||||||
throw Authentication::Require<BasicCredentials>("ClickHouse Basic Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::JWT:
|
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
|
|
||||||
|
|
||||||
case AuthenticationType::KERBEROS:
|
|
||||||
throw Authentication::Require<GSSAcceptorContext>(auth_data.getKerberosRealm());
|
|
||||||
|
|
||||||
case AuthenticationType::SSL_CERTIFICATE:
|
|
||||||
throw Authentication::Require<SSLCertificateCredentials>("ClickHouse X.509 Authentication");
|
|
||||||
|
|
||||||
case AuthenticationType::SSH_KEY:
|
|
||||||
return checkSshSignature(auth_data.getSSHKeys(), ssh_credentials->getSignature(), ssh_credentials->getOriginal());
|
|
||||||
case AuthenticationType::MAX:
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast<const AlwaysAllowCredentials *>(&credentials))
|
if ([[maybe_unused]] const auto * always_allow_credentials = typeid_cast<const AlwaysAllowCredentials *>(&credentials))
|
||||||
return true;
|
return true;
|
||||||
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "areCredentialsValid(): authentication type {} not supported", toString(auth_data.getType()));
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -24,7 +24,7 @@ struct Authentication
|
|||||||
/// returned by the authentication server
|
/// returned by the authentication server
|
||||||
static bool areCredentialsValid(
|
static bool areCredentialsValid(
|
||||||
const Credentials & credentials,
|
const Credentials & credentials,
|
||||||
const AuthenticationData & auth_data,
|
const AuthenticationData & authentication_method,
|
||||||
const ExternalAuthenticators & external_authenticators,
|
const ExternalAuthenticators & external_authenticators,
|
||||||
SettingsChanges & settings);
|
SettingsChanges & settings);
|
||||||
|
|
||||||
|
@ -375,7 +375,8 @@ std::shared_ptr<ASTAuthenticationData> AuthenticationData::toAST() const
|
|||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
|
||||||
case AuthenticationType::NO_PASSWORD: [[fallthrough]];
|
case AuthenticationType::NO_PASSWORD:
|
||||||
|
break;
|
||||||
case AuthenticationType::MAX:
|
case AuthenticationType::MAX:
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type));
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "AST: Unexpected authentication type {}", toString(auth_type));
|
||||||
}
|
}
|
||||||
|
@ -1,8 +1,6 @@
|
|||||||
#include <Access/DiskAccessStorage.h>
|
#include <Access/DiskAccessStorage.h>
|
||||||
#include <Access/AccessEntityIO.h>
|
#include <Access/AccessEntityIO.h>
|
||||||
#include <Access/AccessChangesNotifier.h>
|
#include <Access/AccessChangesNotifier.h>
|
||||||
#include <Backups/RestorerFromBackup.h>
|
|
||||||
#include <Backups/RestoreSettings.h>
|
|
||||||
#include <IO/WriteHelpers.h>
|
#include <IO/WriteHelpers.h>
|
||||||
#include <IO/ReadHelpers.h>
|
#include <IO/ReadHelpers.h>
|
||||||
#include <IO/ReadBufferFromFile.h>
|
#include <IO/ReadBufferFromFile.h>
|
||||||
@ -418,7 +416,7 @@ void DiskAccessStorage::setAllInMemory(const std::vector<std::pair<UUID, AccessE
|
|||||||
|
|
||||||
/// Insert or update entities.
|
/// Insert or update entities.
|
||||||
for (const auto & [id, entity] : entities_without_conflicts)
|
for (const auto & [id, entity] : entities_without_conflicts)
|
||||||
insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false, /* write_on_disk= */ false);
|
insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false, /* conflicting_id = */ nullptr, /* write_on_disk= */ false);
|
||||||
}
|
}
|
||||||
|
|
||||||
void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set<UUID> & ids_to_keep)
|
void DiskAccessStorage::removeAllExceptInMemory(const boost::container::flat_set<UUID> & ids_to_keep)
|
||||||
@ -507,14 +505,14 @@ std::optional<std::pair<String, AccessEntityType>> DiskAccessStorage::readNameWi
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
bool DiskAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
std::lock_guard lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, /* write_on_disk = */ true);
|
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, conflicting_id, /* write_on_disk = */ true);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk)
|
bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk)
|
||||||
{
|
{
|
||||||
const String & name = new_entity->getName();
|
const String & name = new_entity->getName();
|
||||||
AccessEntityType type = new_entity->getType();
|
AccessEntityType type = new_entity->getType();
|
||||||
@ -533,10 +531,16 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne
|
|||||||
if (name_collision && !replace_if_exists)
|
if (name_collision && !replace_if_exists)
|
||||||
{
|
{
|
||||||
if (throw_if_exists)
|
if (throw_if_exists)
|
||||||
|
{
|
||||||
throwNameCollisionCannotInsert(type, name);
|
throwNameCollisionCannotInsert(type, name);
|
||||||
|
}
|
||||||
else
|
else
|
||||||
|
{
|
||||||
|
if (conflicting_id)
|
||||||
|
*conflicting_id = id_by_name;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
auto it_by_id = entries_by_id.find(id);
|
auto it_by_id = entries_by_id.find(id);
|
||||||
bool id_collision = (it_by_id != entries_by_id.end());
|
bool id_collision = (it_by_id != entries_by_id.end());
|
||||||
@ -548,8 +552,12 @@ bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne
|
|||||||
throwIDCollisionCannotInsert(id, type, name, existing_entry.type, existing_entry.name);
|
throwIDCollisionCannotInsert(id, type, name, existing_entry.type, existing_entry.name);
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
{
|
||||||
|
if (conflicting_id)
|
||||||
|
*conflicting_id = id;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (write_on_disk)
|
if (write_on_disk)
|
||||||
scheduleWriteLists(type);
|
scheduleWriteLists(type);
|
||||||
@ -727,25 +735,4 @@ void DiskAccessStorage::deleteAccessEntityOnDisk(const UUID & id) const
|
|||||||
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Couldn't delete {}", file_path);
|
throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Couldn't delete {}", file_path);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void DiskAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
|
|
||||||
{
|
|
||||||
if (!isRestoreAllowed())
|
|
||||||
throwRestoreNotAllowed();
|
|
||||||
|
|
||||||
auto entities = restorer.getAccessEntitiesToRestore();
|
|
||||||
if (entities.empty())
|
|
||||||
return;
|
|
||||||
|
|
||||||
auto create_access = restorer.getRestoreSettings().create_access;
|
|
||||||
bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
|
|
||||||
bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
|
|
||||||
|
|
||||||
restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
|
|
||||||
{
|
|
||||||
for (const auto & [id, entity] : my_entities)
|
|
||||||
insert(id, entity, replace_if_exists, throw_if_exists);
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -34,14 +34,13 @@ public:
|
|||||||
bool exists(const UUID & id) const override;
|
bool exists(const UUID & id) const override;
|
||||||
|
|
||||||
bool isBackupAllowed() const override { return backup_allowed; }
|
bool isBackupAllowed() const override { return backup_allowed; }
|
||||||
void restoreFromBackup(RestorerFromBackup & restorer) override;
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||||
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||||
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
|
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
|
||||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||||
|
|
||||||
@ -55,7 +54,7 @@ private:
|
|||||||
void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS;
|
void listsWritingThreadFunc() TSA_NO_THREAD_SAFETY_ANALYSIS;
|
||||||
void stopListsWritingThread();
|
void stopListsWritingThread();
|
||||||
|
|
||||||
bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, bool write_on_disk) TSA_REQUIRES(mutex);
|
bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id, bool write_on_disk) TSA_REQUIRES(mutex);
|
||||||
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
|
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
|
||||||
bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
|
bool removeNoLock(const UUID & id, bool throw_if_not_exists, bool write_on_disk) TSA_REQUIRES(mutex);
|
||||||
|
|
||||||
|
@ -9,4 +9,28 @@ bool IAccessEntity::equal(const IAccessEntity & other) const
|
|||||||
return (name == other.name) && (getType() == other.getType());
|
return (name == other.name) && (getType() == other.getType());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
void IAccessEntity::replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
|
{
|
||||||
|
if (old_to_new_ids.empty())
|
||||||
|
return;
|
||||||
|
|
||||||
|
bool need_replace_dependencies = false;
|
||||||
|
auto dependencies = entity->findDependencies();
|
||||||
|
for (const auto & dependency : dependencies)
|
||||||
|
{
|
||||||
|
if (old_to_new_ids.contains(dependency))
|
||||||
|
{
|
||||||
|
need_replace_dependencies = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!need_replace_dependencies)
|
||||||
|
return;
|
||||||
|
|
||||||
|
auto new_entity = entity->clone();
|
||||||
|
new_entity->replaceDependencies(old_to_new_ids);
|
||||||
|
entity = new_entity;
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -50,7 +50,8 @@ struct IAccessEntity
|
|||||||
virtual std::vector<UUID> findDependencies() const { return {}; }
|
virtual std::vector<UUID> findDependencies() const { return {}; }
|
||||||
|
|
||||||
/// Replaces dependencies according to a specified map.
|
/// Replaces dependencies according to a specified map.
|
||||||
virtual void replaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
|
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) { doReplaceDependencies(old_to_new_ids); }
|
||||||
|
static void replaceDependencies(std::shared_ptr<const IAccessEntity> & entity, const std::unordered_map<UUID, UUID> & old_to_new_ids);
|
||||||
|
|
||||||
/// Whether this access entity should be written to a backup.
|
/// Whether this access entity should be written to a backup.
|
||||||
virtual bool isBackupAllowed() const { return false; }
|
virtual bool isBackupAllowed() const { return false; }
|
||||||
@ -66,6 +67,8 @@ protected:
|
|||||||
{
|
{
|
||||||
return std::make_shared<EntityClassT>(typeid_cast<const EntityClassT &>(*this));
|
return std::make_shared<EntityClassT>(typeid_cast<const EntityClassT &>(*this));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
virtual void doReplaceDependencies(const std::unordered_map<UUID, UUID> & /* old_to_new_ids */) {}
|
||||||
};
|
};
|
||||||
|
|
||||||
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
|
using AccessEntityPtr = std::shared_ptr<const IAccessEntity>;
|
||||||
|
@ -4,6 +4,8 @@
|
|||||||
#include <Access/User.h>
|
#include <Access/User.h>
|
||||||
#include <Access/AccessBackup.h>
|
#include <Access/AccessBackup.h>
|
||||||
#include <Backups/BackupEntriesCollector.h>
|
#include <Backups/BackupEntriesCollector.h>
|
||||||
|
#include <Backups/RestorerFromBackup.h>
|
||||||
|
#include <Backups/RestoreSettings.h>
|
||||||
#include <Common/Exception.h>
|
#include <Common/Exception.h>
|
||||||
#include <Common/quoteString.h>
|
#include <Common/quoteString.h>
|
||||||
#include <Common/callOnce.h>
|
#include <Common/callOnce.h>
|
||||||
@ -14,10 +16,11 @@
|
|||||||
#include <base/FnTraits.h>
|
#include <base/FnTraits.h>
|
||||||
#include <boost/algorithm/string/join.hpp>
|
#include <boost/algorithm/string/join.hpp>
|
||||||
#include <boost/algorithm/string/replace.hpp>
|
#include <boost/algorithm/string/replace.hpp>
|
||||||
|
#include <boost/range/adaptor/map.hpp>
|
||||||
#include <boost/range/adaptor/reversed.hpp>
|
#include <boost/range/adaptor/reversed.hpp>
|
||||||
|
#include <boost/range/algorithm/copy.hpp>
|
||||||
#include <boost/range/algorithm_ext/erase.hpp>
|
#include <boost/range/algorithm_ext/erase.hpp>
|
||||||
|
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
namespace ErrorCodes
|
namespace ErrorCodes
|
||||||
@ -30,7 +33,6 @@ namespace ErrorCodes
|
|||||||
extern const int IP_ADDRESS_NOT_ALLOWED;
|
extern const int IP_ADDRESS_NOT_ALLOWED;
|
||||||
extern const int LOGICAL_ERROR;
|
extern const int LOGICAL_ERROR;
|
||||||
extern const int NOT_IMPLEMENTED;
|
extern const int NOT_IMPLEMENTED;
|
||||||
extern const int AUTHENTICATION_FAILED;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -179,20 +181,20 @@ UUID IAccessStorage::insert(const AccessEntityPtr & entity)
|
|||||||
return *insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true);
|
return *insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
|
std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
auto id = generateRandomID();
|
auto id = generateRandomID();
|
||||||
|
|
||||||
if (insert(id, entity, replace_if_exists, throw_if_exists))
|
if (insert(id, entity, replace_if_exists, throw_if_exists, conflicting_id))
|
||||||
return id;
|
return id;
|
||||||
|
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
|
bool IAccessStorage::insert(const DB::UUID & id, const DB::AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
return insertImpl(id, entity, replace_if_exists, throw_if_exists);
|
return insertImpl(id, entity, replace_if_exists, throw_if_exists, conflicting_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -286,7 +288,7 @@ std::vector<UUID> IAccessStorage::insertOrReplace(const std::vector<AccessEntity
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, bool, bool)
|
bool IAccessStorage::insertImpl(const UUID &, const AccessEntityPtr & entity, bool, bool, UUID *)
|
||||||
{
|
{
|
||||||
if (isReadOnly())
|
if (isReadOnly())
|
||||||
throwReadonlyCannotInsert(entity->getType(), entity->getName());
|
throwReadonlyCannotInsert(entity->getType(), entity->getName());
|
||||||
@ -525,27 +527,45 @@ std::optional<AuthResult> IAccessStorage::authenticateImpl(
|
|||||||
if (!isAddressAllowed(*user, address))
|
if (!isAddressAllowed(*user, address))
|
||||||
throwAddressNotAllowed(address);
|
throwAddressNotAllowed(address);
|
||||||
|
|
||||||
auto auth_type = user->auth_data.getType();
|
bool skipped_not_allowed_authentication_methods = false;
|
||||||
|
|
||||||
|
for (const auto & auth_method : user->authentication_methods)
|
||||||
|
{
|
||||||
|
auto auth_type = auth_method.getType();
|
||||||
if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) ||
|
if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) ||
|
||||||
((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password))
|
((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password))
|
||||||
throwAuthenticationTypeNotAllowed(auth_type);
|
{
|
||||||
|
skipped_not_allowed_authentication_methods = true;
|
||||||
if (!areCredentialsValid(*user, credentials, external_authenticators, auth_result.settings))
|
continue;
|
||||||
throwInvalidCredentials();
|
}
|
||||||
|
|
||||||
|
if (areCredentialsValid(user->getName(), user->valid_until, auth_method, credentials, external_authenticators, auth_result.settings))
|
||||||
|
{
|
||||||
|
auth_result.authentication_data = auth_method;
|
||||||
return auth_result;
|
return auth_result;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (skipped_not_allowed_authentication_methods)
|
||||||
|
{
|
||||||
|
LOG_INFO(log, "Skipped the check for not allowed authentication methods,"
|
||||||
|
"check allow_no_password and allow_plaintext_password settings in the server configuration");
|
||||||
|
}
|
||||||
|
|
||||||
|
throwInvalidCredentials();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (throw_if_user_not_exists)
|
if (throw_if_user_not_exists)
|
||||||
throwNotFound(AccessEntityType::USER, credentials.getUserName());
|
throwNotFound(AccessEntityType::USER, credentials.getUserName());
|
||||||
else
|
else
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool IAccessStorage::areCredentialsValid(
|
bool IAccessStorage::areCredentialsValid(
|
||||||
const User & user,
|
const std::string & user_name,
|
||||||
|
time_t valid_until,
|
||||||
|
const AuthenticationData & authentication_method,
|
||||||
const Credentials & credentials,
|
const Credentials & credentials,
|
||||||
const ExternalAuthenticators & external_authenticators,
|
const ExternalAuthenticators & external_authenticators,
|
||||||
SettingsChanges & settings) const
|
SettingsChanges & settings) const
|
||||||
@ -553,21 +573,20 @@ bool IAccessStorage::areCredentialsValid(
|
|||||||
if (!credentials.isReady())
|
if (!credentials.isReady())
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (credentials.getUserName() != user.getName())
|
if (credentials.getUserName() != user_name)
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
if (user.valid_until)
|
if (valid_until)
|
||||||
{
|
{
|
||||||
const time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
|
const time_t now = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now());
|
||||||
|
|
||||||
if (now > user.valid_until)
|
if (now > valid_until)
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
return Authentication::areCredentialsValid(credentials, user.auth_data, external_authenticators, settings);
|
return Authentication::areCredentialsValid(credentials, authentication_method, external_authenticators, settings);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const
|
bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const
|
||||||
{
|
{
|
||||||
return user.allowed_client_hosts.contains(address);
|
return user.allowed_client_hosts.contains(address);
|
||||||
@ -595,12 +614,51 @@ void IAccessStorage::backup(BackupEntriesCollector & backup_entries_collector, c
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void IAccessStorage::restoreFromBackup(RestorerFromBackup &)
|
void IAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
|
||||||
{
|
{
|
||||||
if (!isRestoreAllowed())
|
if (!isRestoreAllowed())
|
||||||
throwRestoreNotAllowed();
|
throwRestoreNotAllowed();
|
||||||
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "restoreFromBackup() is not implemented in {}", getStorageType());
|
if (isReplicated() && !acquireReplicatedRestore(restorer))
|
||||||
|
return;
|
||||||
|
|
||||||
|
auto entities = restorer.getAccessEntitiesToRestore();
|
||||||
|
if (entities.empty())
|
||||||
|
return;
|
||||||
|
|
||||||
|
auto create_access = restorer.getRestoreSettings().create_access;
|
||||||
|
bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
|
||||||
|
bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
|
||||||
|
|
||||||
|
restorer.addDataRestoreTask([this, entities_to_restore = std::move(entities), replace_if_exists, throw_if_exists] mutable
|
||||||
|
{
|
||||||
|
std::unordered_map<UUID, UUID> new_to_existing_ids;
|
||||||
|
for (auto & [id, entity] : entities_to_restore)
|
||||||
|
{
|
||||||
|
UUID existing_entity_id;
|
||||||
|
if (!insert(id, entity, replace_if_exists, throw_if_exists, &existing_entity_id))
|
||||||
|
{
|
||||||
|
/// Couldn't insert `entity` because there is an existing entity with the same name.
|
||||||
|
new_to_existing_ids[id] = existing_entity_id;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!new_to_existing_ids.empty())
|
||||||
|
{
|
||||||
|
/// If new entities restored from backup have dependencies on other entities from backup which were not restored because they existed,
|
||||||
|
/// then we should correct those dependencies.
|
||||||
|
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
|
||||||
|
{
|
||||||
|
auto res = entity;
|
||||||
|
IAccessEntity::replaceDependencies(res, new_to_existing_ids);
|
||||||
|
return res;
|
||||||
|
};
|
||||||
|
std::vector<UUID> ids;
|
||||||
|
ids.reserve(entities_to_restore.size());
|
||||||
|
boost::copy(entities_to_restore | boost::adaptors::map_keys, std::back_inserter(ids));
|
||||||
|
tryUpdate(ids, update_func);
|
||||||
|
}
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -747,14 +805,6 @@ void IAccessStorage::throwAddressNotAllowed(const Poco::Net::IPAddress & address
|
|||||||
throw Exception(ErrorCodes::IP_ADDRESS_NOT_ALLOWED, "Connections from {} are not allowed", address.toString());
|
throw Exception(ErrorCodes::IP_ADDRESS_NOT_ALLOWED, "Connections from {} are not allowed", address.toString());
|
||||||
}
|
}
|
||||||
|
|
||||||
void IAccessStorage::throwAuthenticationTypeNotAllowed(AuthenticationType auth_type)
|
|
||||||
{
|
|
||||||
throw Exception(
|
|
||||||
ErrorCodes::AUTHENTICATION_FAILED,
|
|
||||||
"Authentication type {} is not allowed, check the setting allow_{} in the server configuration",
|
|
||||||
toString(auth_type), AuthenticationTypeInfo::get(auth_type).name);
|
|
||||||
}
|
|
||||||
|
|
||||||
void IAccessStorage::throwInvalidCredentials()
|
void IAccessStorage::throwInvalidCredentials()
|
||||||
{
|
{
|
||||||
throw Exception(ErrorCodes::WRONG_PASSWORD, "Invalid credentials");
|
throw Exception(ErrorCodes::WRONG_PASSWORD, "Invalid credentials");
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <Access/IAccessEntity.h>
|
#include <Access/IAccessEntity.h>
|
||||||
|
#include <Access/AuthenticationData.h>
|
||||||
#include <Core/Types.h>
|
#include <Core/Types.h>
|
||||||
#include <Core/UUID.h>
|
#include <Core/UUID.h>
|
||||||
#include <Parsers/IParser.h>
|
#include <Parsers/IParser.h>
|
||||||
@ -34,6 +35,7 @@ struct AuthResult
|
|||||||
UUID user_id;
|
UUID user_id;
|
||||||
/// Session settings received from authentication server (if any)
|
/// Session settings received from authentication server (if any)
|
||||||
SettingsChanges settings{};
|
SettingsChanges settings{};
|
||||||
|
AuthenticationData authentication_data {};
|
||||||
};
|
};
|
||||||
|
|
||||||
/// Contains entities, i.e. instances of classes derived from IAccessEntity.
|
/// Contains entities, i.e. instances of classes derived from IAccessEntity.
|
||||||
@ -62,6 +64,9 @@ public:
|
|||||||
/// Returns true if this entity is readonly.
|
/// Returns true if this entity is readonly.
|
||||||
virtual bool isReadOnly(const UUID &) const { return isReadOnly(); }
|
virtual bool isReadOnly(const UUID &) const { return isReadOnly(); }
|
||||||
|
|
||||||
|
/// Returns true if this storage is replicated.
|
||||||
|
virtual bool isReplicated() const { return false; }
|
||||||
|
|
||||||
/// Starts periodic reloading and updating of entities in this storage.
|
/// Starts periodic reloading and updating of entities in this storage.
|
||||||
virtual void startPeriodicReloading() {}
|
virtual void startPeriodicReloading() {}
|
||||||
|
|
||||||
@ -151,8 +156,8 @@ public:
|
|||||||
/// Inserts an entity to the storage. Returns ID of a new entry in the storage.
|
/// Inserts an entity to the storage. Returns ID of a new entry in the storage.
|
||||||
/// Throws an exception if the specified name already exists.
|
/// Throws an exception if the specified name already exists.
|
||||||
UUID insert(const AccessEntityPtr & entity);
|
UUID insert(const AccessEntityPtr & entity);
|
||||||
std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id = nullptr);
|
||||||
bool insert(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
bool insert(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id = nullptr);
|
||||||
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists = false, bool throw_if_exists = true);
|
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists = false, bool throw_if_exists = true);
|
||||||
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists = false, bool throw_if_exists = true);
|
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, const std::vector<UUID> & ids, bool replace_if_exists = false, bool throw_if_exists = true);
|
||||||
|
|
||||||
@ -216,7 +221,7 @@ protected:
|
|||||||
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
|
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
|
||||||
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
|
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
|
||||||
virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
|
virtual std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const;
|
||||||
virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
virtual bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id);
|
||||||
virtual bool removeImpl(const UUID & id, bool throw_if_not_exists);
|
virtual bool removeImpl(const UUID & id, bool throw_if_not_exists);
|
||||||
virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||||
virtual std::optional<AuthResult> authenticateImpl(
|
virtual std::optional<AuthResult> authenticateImpl(
|
||||||
@ -227,7 +232,9 @@ protected:
|
|||||||
bool allow_no_password,
|
bool allow_no_password,
|
||||||
bool allow_plaintext_password) const;
|
bool allow_plaintext_password) const;
|
||||||
virtual bool areCredentialsValid(
|
virtual bool areCredentialsValid(
|
||||||
const User & user,
|
const std::string & user_name,
|
||||||
|
time_t valid_until,
|
||||||
|
const AuthenticationData & authentication_method,
|
||||||
const Credentials & credentials,
|
const Credentials & credentials,
|
||||||
const ExternalAuthenticators & external_authenticators,
|
const ExternalAuthenticators & external_authenticators,
|
||||||
SettingsChanges & settings) const;
|
SettingsChanges & settings) const;
|
||||||
@ -236,6 +243,7 @@ protected:
|
|||||||
LoggerPtr getLogger() const;
|
LoggerPtr getLogger() const;
|
||||||
static String formatEntityTypeWithName(AccessEntityType type, const String & name) { return AccessEntityTypeInfo::get(type).formatEntityNameWithType(name); }
|
static String formatEntityTypeWithName(AccessEntityType type, const String & name) { return AccessEntityTypeInfo::get(type).formatEntityNameWithType(name); }
|
||||||
static void clearConflictsInEntitiesList(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, LoggerPtr log_);
|
static void clearConflictsInEntitiesList(std::vector<std::pair<UUID, AccessEntityPtr>> & entities, LoggerPtr log_);
|
||||||
|
virtual bool acquireReplicatedRestore(RestorerFromBackup &) const { return false; }
|
||||||
[[noreturn]] void throwNotFound(const UUID & id) const;
|
[[noreturn]] void throwNotFound(const UUID & id) const;
|
||||||
[[noreturn]] void throwNotFound(AccessEntityType type, const String & name) const;
|
[[noreturn]] void throwNotFound(AccessEntityType type, const String & name) const;
|
||||||
[[noreturn]] static void throwBadCast(const UUID & id, AccessEntityType type, const String & name, AccessEntityType required_type);
|
[[noreturn]] static void throwBadCast(const UUID & id, AccessEntityType type, const String & name, AccessEntityType required_type);
|
||||||
@ -248,7 +256,6 @@ protected:
|
|||||||
[[noreturn]] void throwReadonlyCannotRemove(AccessEntityType type, const String & name) const;
|
[[noreturn]] void throwReadonlyCannotRemove(AccessEntityType type, const String & name) const;
|
||||||
[[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address);
|
[[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address);
|
||||||
[[noreturn]] static void throwInvalidCredentials();
|
[[noreturn]] static void throwInvalidCredentials();
|
||||||
[[noreturn]] static void throwAuthenticationTypeNotAllowed(AuthenticationType auth_type);
|
|
||||||
[[noreturn]] void throwBackupNotAllowed() const;
|
[[noreturn]] void throwBackupNotAllowed() const;
|
||||||
[[noreturn]] void throwRestoreNotAllowed() const;
|
[[noreturn]] void throwRestoreNotAllowed() const;
|
||||||
|
|
||||||
|
@ -468,8 +468,8 @@ std::optional<AuthResult> LDAPAccessStorage::authenticateImpl(
|
|||||||
// User does not exist, so we create one, and will add it if authentication is successful.
|
// User does not exist, so we create one, and will add it if authentication is successful.
|
||||||
new_user = std::make_shared<User>();
|
new_user = std::make_shared<User>();
|
||||||
new_user->setName(credentials.getUserName());
|
new_user->setName(credentials.getUserName());
|
||||||
new_user->auth_data = AuthenticationData(AuthenticationType::LDAP);
|
new_user->authentication_methods.emplace_back(AuthenticationType::LDAP);
|
||||||
new_user->auth_data.setLDAPServerName(ldap_server_name);
|
new_user->authentication_methods.back().setLDAPServerName(ldap_server_name);
|
||||||
user = new_user;
|
user = new_user;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -504,7 +504,7 @@ std::optional<AuthResult> LDAPAccessStorage::authenticateImpl(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (id)
|
if (id)
|
||||||
return AuthResult{ .user_id = *id };
|
return AuthResult{ .user_id = *id, .authentication_data = AuthenticationData(AuthenticationType::LDAP) };
|
||||||
return std::nullopt;
|
return std::nullopt;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -1,7 +1,5 @@
|
|||||||
#include <Access/MemoryAccessStorage.h>
|
#include <Access/MemoryAccessStorage.h>
|
||||||
#include <Access/AccessChangesNotifier.h>
|
#include <Access/AccessChangesNotifier.h>
|
||||||
#include <Backups/RestorerFromBackup.h>
|
|
||||||
#include <Backups/RestoreSettings.h>
|
|
||||||
#include <base/scope_guard.h>
|
#include <base/scope_guard.h>
|
||||||
#include <boost/container/flat_set.hpp>
|
#include <boost/container/flat_set.hpp>
|
||||||
#include <boost/range/adaptor/map.hpp>
|
#include <boost/range/adaptor/map.hpp>
|
||||||
@ -63,14 +61,14 @@ AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool MemoryAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
bool MemoryAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
std::lock_guard lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists);
|
return insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, conflicting_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
const String & name = new_entity->getName();
|
const String & name = new_entity->getName();
|
||||||
AccessEntityType type = new_entity->getType();
|
AccessEntityType type = new_entity->getType();
|
||||||
@ -86,10 +84,16 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr &
|
|||||||
if (name_collision && !replace_if_exists)
|
if (name_collision && !replace_if_exists)
|
||||||
{
|
{
|
||||||
if (throw_if_exists)
|
if (throw_if_exists)
|
||||||
|
{
|
||||||
throwNameCollisionCannotInsert(type, name);
|
throwNameCollisionCannotInsert(type, name);
|
||||||
|
}
|
||||||
else
|
else
|
||||||
|
{
|
||||||
|
if (conflicting_id)
|
||||||
|
*conflicting_id = id_by_name;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
auto it_by_id = entries_by_id.find(id);
|
auto it_by_id = entries_by_id.find(id);
|
||||||
bool id_collision = (it_by_id != entries_by_id.end());
|
bool id_collision = (it_by_id != entries_by_id.end());
|
||||||
@ -97,10 +101,16 @@ bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr &
|
|||||||
{
|
{
|
||||||
const auto & existing_entry = it_by_id->second;
|
const auto & existing_entry = it_by_id->second;
|
||||||
if (throw_if_exists)
|
if (throw_if_exists)
|
||||||
|
{
|
||||||
throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName());
|
throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName());
|
||||||
|
}
|
||||||
else
|
else
|
||||||
|
{
|
||||||
|
if (conflicting_id)
|
||||||
|
*conflicting_id = id;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/// Remove collisions if necessary.
|
/// Remove collisions if necessary.
|
||||||
if (name_collision && (id_by_name != id))
|
if (name_collision && (id_by_name != id))
|
||||||
@ -270,28 +280,7 @@ void MemoryAccessStorage::setAll(const std::vector<std::pair<UUID, AccessEntityP
|
|||||||
|
|
||||||
/// Insert or update entities.
|
/// Insert or update entities.
|
||||||
for (const auto & [id, entity] : entities_without_conflicts)
|
for (const auto & [id, entity] : entities_without_conflicts)
|
||||||
insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false);
|
insertNoLock(id, entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false, /* conflicting_id = */ nullptr);
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
void MemoryAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
|
|
||||||
{
|
|
||||||
if (!isRestoreAllowed())
|
|
||||||
throwRestoreNotAllowed();
|
|
||||||
|
|
||||||
auto entities = restorer.getAccessEntitiesToRestore();
|
|
||||||
if (entities.empty())
|
|
||||||
return;
|
|
||||||
|
|
||||||
auto create_access = restorer.getRestoreSettings().create_access;
|
|
||||||
bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
|
|
||||||
bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
|
|
||||||
|
|
||||||
restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
|
|
||||||
{
|
|
||||||
for (const auto & [id, entity] : my_entities)
|
|
||||||
insert(id, entity, replace_if_exists, throw_if_exists);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -34,17 +34,16 @@ public:
|
|||||||
bool exists(const UUID & id) const override;
|
bool exists(const UUID & id) const override;
|
||||||
|
|
||||||
bool isBackupAllowed() const override { return backup_allowed; }
|
bool isBackupAllowed() const override { return backup_allowed; }
|
||||||
void restoreFromBackup(RestorerFromBackup & restorer) override;
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||||
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
|
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
|
||||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||||
|
|
||||||
bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id);
|
||||||
bool removeNoLock(const UUID & id, bool throw_if_not_exists);
|
bool removeNoLock(const UUID & id, bool throw_if_not_exists);
|
||||||
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||||
|
|
||||||
|
@ -353,7 +353,7 @@ void MultipleAccessStorage::reload(ReloadMode reload_mode)
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
|
bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
std::shared_ptr<IAccessStorage> storage_for_insertion;
|
std::shared_ptr<IAccessStorage> storage_for_insertion;
|
||||||
|
|
||||||
@ -376,7 +376,7 @@ bool MultipleAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr &
|
|||||||
getStorageName());
|
getStorageName());
|
||||||
}
|
}
|
||||||
|
|
||||||
if (storage_for_insertion->insert(id, entity, replace_if_exists, throw_if_exists))
|
if (storage_for_insertion->insert(id, entity, replace_if_exists, throw_if_exists, conflicting_id))
|
||||||
{
|
{
|
||||||
std::lock_guard lock{mutex};
|
std::lock_guard lock{mutex};
|
||||||
ids_cache.set(id, storage_for_insertion);
|
ids_cache.set(id, storage_for_insertion);
|
||||||
|
@ -67,7 +67,7 @@ protected:
|
|||||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||||
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
std::optional<std::pair<String, AccessEntityType>> readNameWithTypeImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||||
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
|
bool insertImpl(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
|
||||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||||
std::optional<AuthResult> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
|
std::optional<AuthResult> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists, bool allow_no_password, bool allow_plaintext_password) const override;
|
||||||
|
@ -24,7 +24,7 @@ std::vector<UUID> Quota::findDependencies() const
|
|||||||
return to_roles.findDependencies();
|
return to_roles.findDependencies();
|
||||||
}
|
}
|
||||||
|
|
||||||
void Quota::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
void Quota::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
{
|
{
|
||||||
to_roles.replaceDependencies(old_to_new_ids);
|
to_roles.replaceDependencies(old_to_new_ids);
|
||||||
}
|
}
|
||||||
|
@ -47,7 +47,7 @@ struct Quota : public IAccessEntity
|
|||||||
AccessEntityType getType() const override { return TYPE; }
|
AccessEntityType getType() const override { return TYPE; }
|
||||||
|
|
||||||
std::vector<UUID> findDependencies() const override;
|
std::vector<UUID> findDependencies() const override;
|
||||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||||
bool isBackupAllowed() const override { return true; }
|
bool isBackupAllowed() const override { return true; }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -5,10 +5,9 @@
|
|||||||
#include <Access/AccessChangesNotifier.h>
|
#include <Access/AccessChangesNotifier.h>
|
||||||
#include <Access/AccessBackup.h>
|
#include <Access/AccessBackup.h>
|
||||||
#include <Backups/BackupEntriesCollector.h>
|
#include <Backups/BackupEntriesCollector.h>
|
||||||
#include <Backups/RestorerFromBackup.h>
|
|
||||||
#include <Backups/RestoreSettings.h>
|
|
||||||
#include <Backups/IBackupCoordination.h>
|
#include <Backups/IBackupCoordination.h>
|
||||||
#include <Backups/IRestoreCoordination.h>
|
#include <Backups/IRestoreCoordination.h>
|
||||||
|
#include <Backups/RestorerFromBackup.h>
|
||||||
#include <IO/ReadHelpers.h>
|
#include <IO/ReadHelpers.h>
|
||||||
#include <Interpreters/Context.h>
|
#include <Interpreters/Context.h>
|
||||||
#include <Common/ZooKeeper/KeeperException.h>
|
#include <Common/ZooKeeper/KeeperException.h>
|
||||||
@ -120,7 +119,7 @@ static void retryOnZooKeeperUserError(size_t attempts, Func && function)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
|
bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(new_entity->getType());
|
const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(new_entity->getType());
|
||||||
const String & name = new_entity->getName();
|
const String & name = new_entity->getName();
|
||||||
@ -128,7 +127,7 @@ bool ReplicatedAccessStorage::insertImpl(const UUID & id, const AccessEntityPtr
|
|||||||
|
|
||||||
auto zookeeper = getZooKeeper();
|
auto zookeeper = getZooKeeper();
|
||||||
bool ok = false;
|
bool ok = false;
|
||||||
retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists); });
|
retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists, conflicting_id); });
|
||||||
|
|
||||||
if (!ok)
|
if (!ok)
|
||||||
return false;
|
return false;
|
||||||
@ -143,7 +142,8 @@ bool ReplicatedAccessStorage::insertZooKeeper(
|
|||||||
const UUID & id,
|
const UUID & id,
|
||||||
const AccessEntityPtr & new_entity,
|
const AccessEntityPtr & new_entity,
|
||||||
bool replace_if_exists,
|
bool replace_if_exists,
|
||||||
bool throw_if_exists)
|
bool throw_if_exists,
|
||||||
|
UUID * conflicting_id)
|
||||||
{
|
{
|
||||||
const String & name = new_entity->getName();
|
const String & name = new_entity->getName();
|
||||||
const AccessEntityType type = new_entity->getType();
|
const AccessEntityType type = new_entity->getType();
|
||||||
@ -167,16 +167,16 @@ bool ReplicatedAccessStorage::insertZooKeeper(
|
|||||||
|
|
||||||
if (res == Coordination::Error::ZNODEEXISTS)
|
if (res == Coordination::Error::ZNODEEXISTS)
|
||||||
{
|
{
|
||||||
if (!throw_if_exists && !replace_if_exists)
|
if (!replace_if_exists)
|
||||||
return false; /// Couldn't insert a new entity.
|
|
||||||
|
|
||||||
if (throw_if_exists)
|
|
||||||
{
|
{
|
||||||
if (responses[0]->error == Coordination::Error::ZNODEEXISTS)
|
if (responses[0]->error == Coordination::Error::ZNODEEXISTS)
|
||||||
|
{
|
||||||
|
/// Couldn't insert the new entity because there is an existing entity with such UUID.
|
||||||
|
if (throw_if_exists)
|
||||||
{
|
{
|
||||||
/// To fail with a nice error message, we need info about what already exists.
|
/// To fail with a nice error message, we need info about what already exists.
|
||||||
/// This itself could fail if the conflicting uuid disappears in the meantime.
|
/// This itself can fail if the conflicting uuid disappears in the meantime.
|
||||||
/// If that happens, then we'll just retry from the start.
|
/// If that happens, then retryOnZooKeeperUserError() will just retry the operation from the start.
|
||||||
String existing_entity_definition = zookeeper->get(entity_path);
|
String existing_entity_definition = zookeeper->get(entity_path);
|
||||||
|
|
||||||
AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path);
|
AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path);
|
||||||
@ -186,9 +186,34 @@ bool ReplicatedAccessStorage::insertZooKeeper(
|
|||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
/// Couldn't insert the new entity because there is an existing entity with such name.
|
if (conflicting_id)
|
||||||
|
*conflicting_id = id;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (responses[1]->error == Coordination::Error::ZNODEEXISTS)
|
||||||
|
{
|
||||||
|
/// Couldn't insert the new entity because there is an existing entity with the same name.
|
||||||
|
if (throw_if_exists)
|
||||||
|
{
|
||||||
throwNameCollisionCannotInsert(type, name);
|
throwNameCollisionCannotInsert(type, name);
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
if (conflicting_id)
|
||||||
|
{
|
||||||
|
/// Get UUID of the existing entry with the same name.
|
||||||
|
/// This itself can fail if the conflicting name disappears in the meantime.
|
||||||
|
/// If that happens, then retryOnZooKeeperUserError() will just retry the operation from the start.
|
||||||
|
*conflicting_id = parseUUID(zookeeper->get(name_path));
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
zkutil::KeeperMultiException::check(res, ops, responses);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
assert(replace_if_exists);
|
assert(replace_if_exists);
|
||||||
@ -693,28 +718,10 @@ void ReplicatedAccessStorage::backup(BackupEntriesCollector & backup_entries_col
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
void ReplicatedAccessStorage::restoreFromBackup(RestorerFromBackup & restorer)
|
bool ReplicatedAccessStorage::acquireReplicatedRestore(RestorerFromBackup & restorer) const
|
||||||
{
|
{
|
||||||
if (!isRestoreAllowed())
|
|
||||||
throwRestoreNotAllowed();
|
|
||||||
|
|
||||||
auto restore_coordination = restorer.getRestoreCoordination();
|
auto restore_coordination = restorer.getRestoreCoordination();
|
||||||
if (!restore_coordination->acquireReplicatedAccessStorage(zookeeper_path))
|
return restore_coordination->acquireReplicatedAccessStorage(zookeeper_path);
|
||||||
return;
|
|
||||||
|
|
||||||
auto entities = restorer.getAccessEntitiesToRestore();
|
|
||||||
if (entities.empty())
|
|
||||||
return;
|
|
||||||
|
|
||||||
auto create_access = restorer.getRestoreSettings().create_access;
|
|
||||||
bool replace_if_exists = (create_access == RestoreAccessCreationMode::kReplace);
|
|
||||||
bool throw_if_exists = (create_access == RestoreAccessCreationMode::kCreate);
|
|
||||||
|
|
||||||
restorer.addDataRestoreTask([this, my_entities = std::move(entities), replace_if_exists, throw_if_exists]
|
|
||||||
{
|
|
||||||
for (const auto & [id, entity] : my_entities)
|
|
||||||
insert(id, entity, replace_if_exists, throw_if_exists);
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -26,6 +26,7 @@ public:
|
|||||||
void shutdown() override;
|
void shutdown() override;
|
||||||
|
|
||||||
const char * getStorageType() const override { return STORAGE_TYPE; }
|
const char * getStorageType() const override { return STORAGE_TYPE; }
|
||||||
|
bool isReplicated() const override { return true; }
|
||||||
|
|
||||||
void startPeriodicReloading() override { startWatchingThread(); }
|
void startPeriodicReloading() override { startWatchingThread(); }
|
||||||
void stopPeriodicReloading() override { stopWatchingThread(); }
|
void stopPeriodicReloading() override { stopWatchingThread(); }
|
||||||
@ -35,7 +36,6 @@ public:
|
|||||||
|
|
||||||
bool isBackupAllowed() const override { return backup_allowed; }
|
bool isBackupAllowed() const override { return backup_allowed; }
|
||||||
void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override;
|
void backup(BackupEntriesCollector & backup_entries_collector, const String & data_path_in_backup, AccessEntityType type) const override;
|
||||||
void restoreFromBackup(RestorerFromBackup & restorer) override;
|
|
||||||
|
|
||||||
private:
|
private:
|
||||||
String zookeeper_path;
|
String zookeeper_path;
|
||||||
@ -48,11 +48,11 @@ private:
|
|||||||
std::unique_ptr<ThreadFromGlobalPool> watching_thread;
|
std::unique_ptr<ThreadFromGlobalPool> watching_thread;
|
||||||
std::shared_ptr<ConcurrentBoundedQueue<UUID>> watched_queue;
|
std::shared_ptr<ConcurrentBoundedQueue<UUID>> watched_queue;
|
||||||
|
|
||||||
bool insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists) override;
|
bool insertImpl(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id) override;
|
||||||
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
|
||||||
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
|
||||||
|
|
||||||
bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
|
bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, UUID * conflicting_id);
|
||||||
bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists);
|
bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists);
|
||||||
bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
|
||||||
|
|
||||||
@ -80,6 +80,7 @@ private:
|
|||||||
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
|
||||||
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
|
||||||
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
|
||||||
|
bool acquireReplicatedRestore(RestorerFromBackup & restorer) const override;
|
||||||
|
|
||||||
mutable std::mutex mutex;
|
mutable std::mutex mutex;
|
||||||
MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex);
|
MemoryAccessStorage memory_storage TSA_GUARDED_BY(mutex);
|
||||||
|
@ -21,7 +21,7 @@ std::vector<UUID> Role::findDependencies() const
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void Role::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
void Role::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
{
|
{
|
||||||
granted_roles.replaceDependencies(old_to_new_ids);
|
granted_roles.replaceDependencies(old_to_new_ids);
|
||||||
settings.replaceDependencies(old_to_new_ids);
|
settings.replaceDependencies(old_to_new_ids);
|
||||||
|
@ -21,7 +21,7 @@ struct Role : public IAccessEntity
|
|||||||
AccessEntityType getType() const override { return TYPE; }
|
AccessEntityType getType() const override { return TYPE; }
|
||||||
|
|
||||||
std::vector<UUID> findDependencies() const override;
|
std::vector<UUID> findDependencies() const override;
|
||||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||||
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
|
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -63,7 +63,7 @@ std::vector<UUID> RowPolicy::findDependencies() const
|
|||||||
return to_roles.findDependencies();
|
return to_roles.findDependencies();
|
||||||
}
|
}
|
||||||
|
|
||||||
void RowPolicy::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
void RowPolicy::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
{
|
{
|
||||||
to_roles.replaceDependencies(old_to_new_ids);
|
to_roles.replaceDependencies(old_to_new_ids);
|
||||||
}
|
}
|
||||||
|
@ -50,7 +50,7 @@ struct RowPolicy : public IAccessEntity
|
|||||||
AccessEntityType getType() const override { return TYPE; }
|
AccessEntityType getType() const override { return TYPE; }
|
||||||
|
|
||||||
std::vector<UUID> findDependencies() const override;
|
std::vector<UUID> findDependencies() const override;
|
||||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||||
bool isBackupAllowed() const override { return true; }
|
bool isBackupAllowed() const override { return true; }
|
||||||
|
|
||||||
/// Which roles or users should use this row policy.
|
/// Which roles or users should use this row policy.
|
||||||
|
@ -21,7 +21,7 @@ std::vector<UUID> SettingsProfile::findDependencies() const
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SettingsProfile::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
void SettingsProfile::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
{
|
{
|
||||||
elements.replaceDependencies(old_to_new_ids);
|
elements.replaceDependencies(old_to_new_ids);
|
||||||
to_roles.replaceDependencies(old_to_new_ids);
|
to_roles.replaceDependencies(old_to_new_ids);
|
||||||
|
@ -22,7 +22,7 @@ struct SettingsProfile : public IAccessEntity
|
|||||||
AccessEntityType getType() const override { return TYPE; }
|
AccessEntityType getType() const override { return TYPE; }
|
||||||
|
|
||||||
std::vector<UUID> findDependencies() const override;
|
std::vector<UUID> findDependencies() const override;
|
||||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||||
bool isBackupAllowed() const override { return elements.isBackupAllowed(); }
|
bool isBackupAllowed() const override { return elements.isBackupAllowed(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -16,7 +16,8 @@ bool User::equal(const IAccessEntity & other) const
|
|||||||
if (!IAccessEntity::equal(other))
|
if (!IAccessEntity::equal(other))
|
||||||
return false;
|
return false;
|
||||||
const auto & other_user = typeid_cast<const User &>(other);
|
const auto & other_user = typeid_cast<const User &>(other);
|
||||||
return (auth_data == other_user.auth_data) && (allowed_client_hosts == other_user.allowed_client_hosts)
|
return (authentication_methods == other_user.authentication_methods)
|
||||||
|
&& (allowed_client_hosts == other_user.allowed_client_hosts)
|
||||||
&& (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles)
|
&& (access == other_user.access) && (granted_roles == other_user.granted_roles) && (default_roles == other_user.default_roles)
|
||||||
&& (settings == other_user.settings) && (grantees == other_user.grantees) && (default_database == other_user.default_database)
|
&& (settings == other_user.settings) && (grantees == other_user.grantees) && (default_database == other_user.default_database)
|
||||||
&& (valid_until == other_user.valid_until);
|
&& (valid_until == other_user.valid_until);
|
||||||
@ -48,7 +49,7 @@ std::vector<UUID> User::findDependencies() const
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
void User::replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
void User::doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids)
|
||||||
{
|
{
|
||||||
default_roles.replaceDependencies(old_to_new_ids);
|
default_roles.replaceDependencies(old_to_new_ids);
|
||||||
granted_roles.replaceDependencies(old_to_new_ids);
|
granted_roles.replaceDependencies(old_to_new_ids);
|
||||||
|
@ -15,7 +15,7 @@ namespace DB
|
|||||||
*/
|
*/
|
||||||
struct User : public IAccessEntity
|
struct User : public IAccessEntity
|
||||||
{
|
{
|
||||||
AuthenticationData auth_data;
|
std::vector<AuthenticationData> authentication_methods;
|
||||||
AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{};
|
AllowedClientHosts allowed_client_hosts = AllowedClientHosts::AnyHostTag{};
|
||||||
AccessRights access;
|
AccessRights access;
|
||||||
GrantedRoles granted_roles;
|
GrantedRoles granted_roles;
|
||||||
@ -32,7 +32,7 @@ struct User : public IAccessEntity
|
|||||||
void setName(const String & name_) override;
|
void setName(const String & name_) override;
|
||||||
|
|
||||||
std::vector<UUID> findDependencies() const override;
|
std::vector<UUID> findDependencies() const override;
|
||||||
void replaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
void doReplaceDependencies(const std::unordered_map<UUID, UUID> & old_to_new_ids) override;
|
||||||
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
|
bool isBackupAllowed() const override { return settings.isBackupAllowed(); }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -155,18 +155,18 @@ namespace
|
|||||||
|
|
||||||
if (has_password_plaintext)
|
if (has_password_plaintext)
|
||||||
{
|
{
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::PLAINTEXT_PASSWORD};
|
user->authentication_methods.emplace_back(AuthenticationType::PLAINTEXT_PASSWORD);
|
||||||
user->auth_data.setPassword(config.getString(user_config + ".password"));
|
user->authentication_methods.back().setPassword(config.getString(user_config + ".password"));
|
||||||
}
|
}
|
||||||
else if (has_password_sha256_hex)
|
else if (has_password_sha256_hex)
|
||||||
{
|
{
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::SHA256_PASSWORD};
|
user->authentication_methods.emplace_back(AuthenticationType::SHA256_PASSWORD);
|
||||||
user->auth_data.setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"));
|
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_sha256_hex"));
|
||||||
}
|
}
|
||||||
else if (has_password_double_sha1_hex)
|
else if (has_password_double_sha1_hex)
|
||||||
{
|
{
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::DOUBLE_SHA1_PASSWORD};
|
user->authentication_methods.emplace_back(AuthenticationType::DOUBLE_SHA1_PASSWORD);
|
||||||
user->auth_data.setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"));
|
user->authentication_methods.back().setPasswordHashHex(config.getString(user_config + ".password_double_sha1_hex"));
|
||||||
}
|
}
|
||||||
else if (has_ldap)
|
else if (has_ldap)
|
||||||
{
|
{
|
||||||
@ -178,19 +178,19 @@ namespace
|
|||||||
if (ldap_server_name.empty())
|
if (ldap_server_name.empty())
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "LDAP server name cannot be empty for user {}.", user_name);
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "LDAP server name cannot be empty for user {}.", user_name);
|
||||||
|
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::LDAP};
|
user->authentication_methods.emplace_back(AuthenticationType::LDAP);
|
||||||
user->auth_data.setLDAPServerName(ldap_server_name);
|
user->authentication_methods.back().setLDAPServerName(ldap_server_name);
|
||||||
}
|
}
|
||||||
else if (has_kerberos)
|
else if (has_kerberos)
|
||||||
{
|
{
|
||||||
const auto realm = config.getString(user_config + ".kerberos.realm", "");
|
const auto realm = config.getString(user_config + ".kerberos.realm", "");
|
||||||
|
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::KERBEROS};
|
user->authentication_methods.emplace_back(AuthenticationType::KERBEROS);
|
||||||
user->auth_data.setKerberosRealm(realm);
|
user->authentication_methods.back().setKerberosRealm(realm);
|
||||||
}
|
}
|
||||||
else if (has_certificates)
|
else if (has_certificates)
|
||||||
{
|
{
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::SSL_CERTIFICATE};
|
user->authentication_methods.emplace_back(AuthenticationType::SSL_CERTIFICATE);
|
||||||
|
|
||||||
/// Fill list of allowed certificates.
|
/// Fill list of allowed certificates.
|
||||||
Poco::Util::AbstractConfiguration::Keys keys;
|
Poco::Util::AbstractConfiguration::Keys keys;
|
||||||
@ -200,14 +200,14 @@ namespace
|
|||||||
if (key.starts_with("common_name"))
|
if (key.starts_with("common_name"))
|
||||||
{
|
{
|
||||||
String value = config.getString(certificates_config + "." + key);
|
String value = config.getString(certificates_config + "." + key);
|
||||||
user->auth_data.addSSLCertificateSubject(SSLCertificateSubjects::Type::CN, std::move(value));
|
user->authentication_methods.back().addSSLCertificateSubject(SSLCertificateSubjects::Type::CN, std::move(value));
|
||||||
}
|
}
|
||||||
else if (key.starts_with("subject_alt_name"))
|
else if (key.starts_with("subject_alt_name"))
|
||||||
{
|
{
|
||||||
String value = config.getString(certificates_config + "." + key);
|
String value = config.getString(certificates_config + "." + key);
|
||||||
if (value.empty())
|
if (value.empty())
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected ssl_certificates.subject_alt_name to not be empty");
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected ssl_certificates.subject_alt_name to not be empty");
|
||||||
user->auth_data.addSSLCertificateSubject(SSLCertificateSubjects::Type::SAN, std::move(value));
|
user->authentication_methods.back().addSSLCertificateSubject(SSLCertificateSubjects::Type::SAN, std::move(value));
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown certificate pattern type: {}", key);
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown certificate pattern type: {}", key);
|
||||||
@ -216,7 +216,7 @@ namespace
|
|||||||
else if (has_ssh_keys)
|
else if (has_ssh_keys)
|
||||||
{
|
{
|
||||||
#if USE_SSH
|
#if USE_SSH
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::SSH_KEY};
|
user->authentication_methods.emplace_back(AuthenticationType::SSH_KEY);
|
||||||
|
|
||||||
Poco::Util::AbstractConfiguration::Keys entries;
|
Poco::Util::AbstractConfiguration::Keys entries;
|
||||||
config.keys(ssh_keys_config, entries);
|
config.keys(ssh_keys_config, entries);
|
||||||
@ -253,20 +253,26 @@ namespace
|
|||||||
else
|
else
|
||||||
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown ssh_key entry pattern type: {}", entry);
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown ssh_key entry pattern type: {}", entry);
|
||||||
}
|
}
|
||||||
user->auth_data.setSSHKeys(std::move(keys));
|
user->authentication_methods.back().setSSHKeys(std::move(keys));
|
||||||
#else
|
#else
|
||||||
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
|
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without libssh");
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
else if (has_http_auth)
|
else if (has_http_auth)
|
||||||
{
|
{
|
||||||
user->auth_data = AuthenticationData{AuthenticationType::HTTP};
|
user->authentication_methods.emplace_back(AuthenticationType::HTTP);
|
||||||
user->auth_data.setHTTPAuthenticationServerName(config.getString(http_auth_config + ".server"));
|
user->authentication_methods.back().setHTTPAuthenticationServerName(config.getString(http_auth_config + ".server"));
|
||||||
auto scheme = config.getString(http_auth_config + ".scheme");
|
auto scheme = config.getString(http_auth_config + ".scheme");
|
||||||
user->auth_data.setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme));
|
user->authentication_methods.back().setHTTPAuthenticationScheme(parseHTTPAuthenticationScheme(scheme));
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
user->authentication_methods.emplace_back();
|
||||||
}
|
}
|
||||||
|
|
||||||
auto auth_type = user->auth_data.getType();
|
for (const auto & authentication_method : user->authentication_methods)
|
||||||
|
{
|
||||||
|
auto auth_type = authentication_method.getType();
|
||||||
if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) ||
|
if (((auth_type == AuthenticationType::NO_PASSWORD) && !allow_no_password) ||
|
||||||
((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password))
|
((auth_type == AuthenticationType::PLAINTEXT_PASSWORD) && !allow_plaintext_password))
|
||||||
{
|
{
|
||||||
@ -274,6 +280,7 @@ namespace
|
|||||||
"Authentication type {} is not allowed, check the setting allow_{} in the server configuration",
|
"Authentication type {} is not allowed, check the setting allow_{} in the server configuration",
|
||||||
toString(auth_type), AuthenticationTypeInfo::get(auth_type).name);
|
toString(auth_type), AuthenticationTypeInfo::get(auth_type).name);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const auto profile_name_config = user_config + ".profile";
|
const auto profile_name_config = user_config + ".profile";
|
||||||
if (config.has(profile_name_config))
|
if (config.has(profile_name_config))
|
||||||
|
@ -3,195 +3,27 @@
|
|||||||
#include <Parsers/FunctionSecretArgumentsFinder.h>
|
#include <Parsers/FunctionSecretArgumentsFinder.h>
|
||||||
#include <Analyzer/ConstantNode.h>
|
#include <Analyzer/ConstantNode.h>
|
||||||
#include <Analyzer/FunctionNode.h>
|
#include <Analyzer/FunctionNode.h>
|
||||||
#include <Analyzer/IQueryTreeNode.h>
|
|
||||||
#include <Analyzer/IdentifierNode.h>
|
#include <Analyzer/IdentifierNode.h>
|
||||||
#include <Analyzer/ListNode.h>
|
|
||||||
#include <Common/KnownObjectNames.h>
|
|
||||||
#include <Core/QualifiedTableName.h>
|
|
||||||
|
|
||||||
#include <boost/algorithm/string/predicate.hpp>
|
|
||||||
|
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
|
class FunctionTreeNode : public AbstractFunction
|
||||||
/// Finds arguments of a specified function which should not be displayed for most users for security reasons.
|
|
||||||
/// That involves passwords and secret keys.
|
|
||||||
class FunctionSecretArgumentsFinderTreeNode
|
|
||||||
{
|
{
|
||||||
public:
|
public:
|
||||||
explicit FunctionSecretArgumentsFinderTreeNode(const FunctionNode & function_) : function(function_), arguments(function.getArguments())
|
class ArgumentTreeNode : public Argument
|
||||||
{
|
{
|
||||||
if (arguments.getNodes().empty())
|
public:
|
||||||
return;
|
explicit ArgumentTreeNode(const IQueryTreeNode * argument_) : argument(argument_) {}
|
||||||
|
std::unique_ptr<AbstractFunction> getFunction() const override
|
||||||
findFunctionSecretArguments();
|
{
|
||||||
|
if (const auto * f = argument->as<FunctionNode>())
|
||||||
|
return std::make_unique<FunctionTreeNode>(*f);
|
||||||
|
return nullptr;
|
||||||
}
|
}
|
||||||
|
bool isIdentifier() const override { return argument->as<IdentifierNode>(); }
|
||||||
struct Result
|
bool tryGetString(String * res, bool allow_identifier) const override
|
||||||
{
|
|
||||||
/// Result constructed by default means no arguments will be hidden.
|
|
||||||
size_t start = static_cast<size_t>(-1);
|
|
||||||
size_t count = 0; /// Mostly it's either 0 or 1. There are only a few cases where `count` can be greater than 1 (e.g. see `encrypt`).
|
|
||||||
/// In all known cases secret arguments are consecutive
|
|
||||||
bool are_named = false; /// Arguments like `password = 'password'` are considered as named arguments.
|
|
||||||
/// E.g. "headers" in `url('..', headers('foo' = '[HIDDEN]'))`
|
|
||||||
std::vector<std::string> nested_maps;
|
|
||||||
|
|
||||||
bool hasSecrets() const
|
|
||||||
{
|
|
||||||
return count != 0 || !nested_maps.empty();
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
FunctionSecretArgumentsFinder::Result getResult() const { return result; }
|
|
||||||
|
|
||||||
private:
|
|
||||||
const FunctionNode & function;
|
|
||||||
const ListNode & arguments;
|
|
||||||
FunctionSecretArgumentsFinder::Result result;
|
|
||||||
|
|
||||||
void markSecretArgument(size_t index, bool argument_is_named = false)
|
|
||||||
{
|
|
||||||
if (index >= arguments.getNodes().size())
|
|
||||||
return;
|
|
||||||
if (!result.count)
|
|
||||||
{
|
|
||||||
result.start = index;
|
|
||||||
result.are_named = argument_is_named;
|
|
||||||
}
|
|
||||||
chassert(index >= result.start); /// We always check arguments consecutively
|
|
||||||
result.count = index + 1 - result.start;
|
|
||||||
if (!argument_is_named)
|
|
||||||
result.are_named = false;
|
|
||||||
}
|
|
||||||
|
|
||||||
void findFunctionSecretArguments()
|
|
||||||
{
|
|
||||||
const auto & name = function.getFunctionName();
|
|
||||||
|
|
||||||
if ((name == "mysql") || (name == "postgresql") || (name == "mongodb"))
|
|
||||||
{
|
|
||||||
/// mysql('host:port', 'database', 'table', 'user', 'password', ...)
|
|
||||||
/// postgresql('host:port', 'database', 'table', 'user', 'password', ...)
|
|
||||||
/// mongodb('host:port', 'database', 'collection', 'user', 'password', ...)
|
|
||||||
findMySQLFunctionSecretArguments();
|
|
||||||
}
|
|
||||||
else if ((name == "s3") || (name == "cosn") || (name == "oss") ||
|
|
||||||
(name == "deltaLake") || (name == "hudi") || (name == "iceberg"))
|
|
||||||
{
|
|
||||||
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
|
||||||
findS3FunctionSecretArguments(/* is_cluster_function= */ false);
|
|
||||||
}
|
|
||||||
else if (name == "s3Cluster")
|
|
||||||
{
|
|
||||||
/// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
|
||||||
findS3FunctionSecretArguments(/* is_cluster_function= */ true);
|
|
||||||
}
|
|
||||||
else if ((name == "remote") || (name == "remoteSecure"))
|
|
||||||
{
|
|
||||||
/// remote('addresses_expr', 'db', 'table', 'user', 'password', ...)
|
|
||||||
findRemoteFunctionSecretArguments();
|
|
||||||
}
|
|
||||||
else if ((name == "encrypt") || (name == "decrypt") ||
|
|
||||||
(name == "aes_encrypt_mysql") || (name == "aes_decrypt_mysql") ||
|
|
||||||
(name == "tryDecrypt"))
|
|
||||||
{
|
|
||||||
/// encrypt('mode', 'plaintext', 'key' [, iv, aad])
|
|
||||||
findEncryptionFunctionSecretArguments();
|
|
||||||
}
|
|
||||||
else if (name == "url")
|
|
||||||
{
|
|
||||||
findURLSecretArguments();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void findMySQLFunctionSecretArguments()
|
|
||||||
{
|
|
||||||
if (isNamedCollectionName(0))
|
|
||||||
{
|
|
||||||
/// mysql(named_collection, ..., password = 'password', ...)
|
|
||||||
findSecretNamedArgument("password", 1);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
/// mysql('host:port', 'database', 'table', 'user', 'password', ...)
|
|
||||||
markSecretArgument(4);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Returns the number of arguments excluding "headers" and "extra_credentials" (which should
|
|
||||||
/// always be at the end). Marks "headers" as secret, if found.
|
|
||||||
size_t excludeS3OrURLNestedMaps()
|
|
||||||
{
|
|
||||||
const auto & nodes = arguments.getNodes();
|
|
||||||
size_t count = nodes.size();
|
|
||||||
while (count > 0)
|
|
||||||
{
|
|
||||||
const FunctionNode * f = nodes.at(count - 1)->as<FunctionNode>();
|
|
||||||
if (!f)
|
|
||||||
break;
|
|
||||||
if (f->getFunctionName() == "headers")
|
|
||||||
result.nested_maps.push_back(f->getFunctionName());
|
|
||||||
else if (f->getFunctionName() != "extra_credentials")
|
|
||||||
break;
|
|
||||||
count -= 1;
|
|
||||||
}
|
|
||||||
return count;
|
|
||||||
}
|
|
||||||
|
|
||||||
void findS3FunctionSecretArguments(bool is_cluster_function)
|
|
||||||
{
|
|
||||||
/// s3Cluster('cluster_name', 'url', ...) has 'url' as its second argument.
|
|
||||||
size_t url_arg_idx = is_cluster_function ? 1 : 0;
|
|
||||||
|
|
||||||
if (!is_cluster_function && isNamedCollectionName(0))
|
|
||||||
{
|
|
||||||
/// s3(named_collection, ..., secret_access_key = 'secret_access_key', ...)
|
|
||||||
findSecretNamedArgument("secret_access_key", 1);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// We should check other arguments first because we don't need to do any replacement in case of
|
|
||||||
/// s3('url', NOSIGN, 'format' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
|
|
||||||
/// s3('url', 'format', 'structure' [, 'compression'] [, extra_credentials(..)] [, headers(..)])
|
|
||||||
size_t count = excludeS3OrURLNestedMaps();
|
|
||||||
if ((url_arg_idx + 3 <= count) && (count <= url_arg_idx + 4))
|
|
||||||
{
|
|
||||||
String second_arg;
|
|
||||||
if (tryGetStringFromArgument(url_arg_idx + 1, &second_arg))
|
|
||||||
{
|
|
||||||
if (boost::iequals(second_arg, "NOSIGN"))
|
|
||||||
return; /// The argument after 'url' is "NOSIGN".
|
|
||||||
|
|
||||||
if (second_arg == "auto" || KnownFormatNames::instance().exists(second_arg))
|
|
||||||
return; /// The argument after 'url' is a format: s3('url', 'format', ...)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// We're going to replace 'aws_secret_access_key' with '[HIDDEN]' for the following signatures:
|
|
||||||
/// s3('url', 'aws_access_key_id', 'aws_secret_access_key', ...)
|
|
||||||
/// s3Cluster('cluster_name', 'url', 'aws_access_key_id', 'aws_secret_access_key', 'format', 'compression')
|
|
||||||
if (url_arg_idx + 2 < count)
|
|
||||||
markSecretArgument(url_arg_idx + 2);
|
|
||||||
}
|
|
||||||
|
|
||||||
void findURLSecretArguments()
|
|
||||||
{
|
|
||||||
if (!isNamedCollectionName(0))
|
|
||||||
excludeS3OrURLNestedMaps();
|
|
||||||
}
|
|
||||||
|
|
||||||
bool tryGetStringFromArgument(size_t arg_idx, String * res, bool allow_identifier = true) const
|
|
||||||
{
|
|
||||||
if (arg_idx >= arguments.getNodes().size())
|
|
||||||
return false;
|
|
||||||
|
|
||||||
return tryGetStringFromArgument(arguments.getNodes()[arg_idx], res, allow_identifier);
|
|
||||||
}
|
|
||||||
|
|
||||||
static bool tryGetStringFromArgument(const QueryTreeNodePtr argument, String * res, bool allow_identifier = true)
|
|
||||||
{
|
{
|
||||||
if (const auto * literal = argument->as<ConstantNode>())
|
if (const auto * literal = argument->as<ConstantNode>())
|
||||||
{
|
{
|
||||||
@ -214,159 +46,46 @@ private:
|
|||||||
|
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
private:
|
||||||
|
const IQueryTreeNode * argument = nullptr;
|
||||||
|
};
|
||||||
|
|
||||||
void findRemoteFunctionSecretArguments()
|
class ArgumentsTreeNode : public Arguments
|
||||||
{
|
{
|
||||||
if (isNamedCollectionName(0))
|
public:
|
||||||
|
explicit ArgumentsTreeNode(const QueryTreeNodes * arguments_) : arguments(arguments_) {}
|
||||||
|
size_t size() const override { return arguments ? arguments->size() : 0; }
|
||||||
|
std::unique_ptr<Argument> at(size_t n) const override { return std::make_unique<ArgumentTreeNode>(arguments->at(n).get()); }
|
||||||
|
private:
|
||||||
|
const QueryTreeNodes * arguments = nullptr;
|
||||||
|
};
|
||||||
|
|
||||||
|
explicit FunctionTreeNode(const FunctionNode & function_) : function(&function_)
|
||||||
{
|
{
|
||||||
/// remote(named_collection, ..., password = 'password', ...)
|
if (const auto & nodes = function->getArguments().getNodes(); !nodes.empty())
|
||||||
findSecretNamedArgument("password", 1);
|
arguments = std::make_unique<ArgumentsTreeNode>(&nodes);
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
String name() const override { return function->getFunctionName(); }
|
||||||
|
private:
|
||||||
|
const FunctionNode * function = nullptr;
|
||||||
|
};
|
||||||
|
|
||||||
/// We're going to replace 'password' with '[HIDDEN'] for the following signatures:
|
|
||||||
/// remote('addresses_expr', db.table, 'user' [, 'password'] [, sharding_key])
|
|
||||||
/// remote('addresses_expr', 'db', 'table', 'user' [, 'password'] [, sharding_key])
|
|
||||||
/// remote('addresses_expr', table_function(), 'user' [, 'password'] [, sharding_key])
|
|
||||||
|
|
||||||
/// But we should check the number of arguments first because we don't need to do any replacements in case of
|
/// Finds arguments of a specified function which should not be displayed for most users for security reasons.
|
||||||
/// remote('addresses_expr', db.table)
|
/// That involves passwords and secret keys.
|
||||||
if (arguments.getNodes().size() < 3)
|
class FunctionSecretArgumentsFinderTreeNode : public FunctionSecretArgumentsFinder
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
explicit FunctionSecretArgumentsFinderTreeNode(const FunctionNode & function_)
|
||||||
|
: FunctionSecretArgumentsFinder(std::make_unique<FunctionTreeNode>(function_))
|
||||||
|
{
|
||||||
|
if (!function->hasArguments())
|
||||||
return;
|
return;
|
||||||
|
|
||||||
size_t arg_num = 1;
|
findOrdinaryFunctionSecretArguments();
|
||||||
|
|
||||||
/// Skip 1 or 2 arguments with table_function() or db.table or 'db', 'table'.
|
|
||||||
const auto * table_function = arguments.getNodes()[arg_num]->as<FunctionNode>();
|
|
||||||
if (table_function && KnownTableFunctionNames::instance().exists(table_function->getFunctionName()))
|
|
||||||
{
|
|
||||||
++arg_num;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
std::optional<String> database;
|
|
||||||
std::optional<QualifiedTableName> qualified_table_name;
|
|
||||||
if (!tryGetDatabaseNameOrQualifiedTableName(arg_num, database, qualified_table_name))
|
|
||||||
{
|
|
||||||
/// We couldn't evaluate the argument so we don't know whether it is 'db.table' or just 'db'.
|
|
||||||
/// Hence we can't figure out whether we should skip one argument 'user' or two arguments 'table', 'user'
|
|
||||||
/// before the argument 'password'. So it's safer to wipe two arguments just in case.
|
|
||||||
/// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
|
|
||||||
/// before wiping it (because the `password` argument is always a literal string).
|
|
||||||
if (tryGetStringFromArgument(arg_num + 2, nullptr, /* allow_identifier= */ false))
|
|
||||||
{
|
|
||||||
/// Wipe either `password` or `user`.
|
|
||||||
markSecretArgument(arg_num + 2);
|
|
||||||
}
|
|
||||||
if (tryGetStringFromArgument(arg_num + 3, nullptr, /* allow_identifier= */ false))
|
|
||||||
{
|
|
||||||
/// Wipe either `password` or `sharding_key`.
|
|
||||||
markSecretArgument(arg_num + 3);
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Skip the current argument (which is either a database name or a qualified table name).
|
FunctionSecretArgumentsFinder::Result getResult() const { return result; }
|
||||||
++arg_num;
|
|
||||||
if (database)
|
|
||||||
{
|
|
||||||
/// Skip the 'table' argument if the previous argument was a database name.
|
|
||||||
++arg_num;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Skip username.
|
|
||||||
++arg_num;
|
|
||||||
|
|
||||||
/// Do our replacement:
|
|
||||||
/// remote('addresses_expr', db.table, 'user', 'password', ...) -> remote('addresses_expr', db.table, 'user', '[HIDDEN]', ...)
|
|
||||||
/// The last argument can be also a `sharding_key`, so we need to check that argument is a literal string
|
|
||||||
/// before wiping it (because the `password` argument is always a literal string).
|
|
||||||
bool can_be_password = tryGetStringFromArgument(arg_num, nullptr, /* allow_identifier= */ false);
|
|
||||||
if (can_be_password)
|
|
||||||
markSecretArgument(arg_num);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Tries to get either a database name or a qualified table name from an argument.
|
|
||||||
/// Empty string is also allowed (it means the default database).
|
|
||||||
/// The function is used by findRemoteFunctionSecretArguments() to determine how many arguments to skip before a password.
|
|
||||||
bool tryGetDatabaseNameOrQualifiedTableName(
|
|
||||||
size_t arg_idx,
|
|
||||||
std::optional<String> & res_database,
|
|
||||||
std::optional<QualifiedTableName> & res_qualified_table_name) const
|
|
||||||
{
|
|
||||||
res_database.reset();
|
|
||||||
res_qualified_table_name.reset();
|
|
||||||
|
|
||||||
String str;
|
|
||||||
if (!tryGetStringFromArgument(arg_idx, &str, /* allow_identifier= */ true))
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (str.empty())
|
|
||||||
{
|
|
||||||
res_database = "";
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
auto qualified_table_name = QualifiedTableName::tryParseFromString(str);
|
|
||||||
if (!qualified_table_name)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
if (qualified_table_name->database.empty())
|
|
||||||
res_database = std::move(qualified_table_name->table);
|
|
||||||
else
|
|
||||||
res_qualified_table_name = std::move(qualified_table_name);
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
void findEncryptionFunctionSecretArguments()
|
|
||||||
{
|
|
||||||
if (arguments.getNodes().empty())
|
|
||||||
return;
|
|
||||||
|
|
||||||
/// We replace all arguments after 'mode' with '[HIDDEN]':
|
|
||||||
/// encrypt('mode', 'plaintext', 'key' [, iv, aad]) -> encrypt('mode', '[HIDDEN]')
|
|
||||||
result.start = 1;
|
|
||||||
result.count = arguments.getNodes().size() - 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
/// Whether a specified argument can be the name of a named collection?
|
|
||||||
bool isNamedCollectionName(size_t arg_idx) const
|
|
||||||
{
|
|
||||||
if (arguments.getNodes().size() <= arg_idx)
|
|
||||||
return false;
|
|
||||||
|
|
||||||
const auto * identifier = arguments.getNodes()[arg_idx]->as<IdentifierNode>();
|
|
||||||
return identifier != nullptr;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Looks for a secret argument with a specified name. This function looks for arguments in format `key=value` where the key is specified.
|
|
||||||
void findSecretNamedArgument(const std::string_view & key, size_t start = 0)
|
|
||||||
{
|
|
||||||
for (size_t i = start; i < arguments.getNodes().size(); ++i)
|
|
||||||
{
|
|
||||||
const auto & argument = arguments.getNodes()[i];
|
|
||||||
const auto * equals_func = argument->as<FunctionNode>();
|
|
||||||
if (!equals_func || (equals_func->getFunctionName() != "equals"))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
const auto * expr_list = equals_func->getArguments().as<ListNode>();
|
|
||||||
if (!expr_list)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
const auto & equal_args = expr_list->getNodes();
|
|
||||||
if (equal_args.size() != 2)
|
|
||||||
continue;
|
|
||||||
|
|
||||||
String found_key;
|
|
||||||
if (!tryGetStringFromArgument(equal_args[0], &found_key))
|
|
||||||
continue;
|
|
||||||
|
|
||||||
if (found_key == key)
|
|
||||||
markSecretArgument(i, /* argument_is_named= */ true);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -2564,8 +2564,8 @@ void checkFunctionNodeHasEmptyNullsAction(FunctionNode const & node)
|
|||||||
if (node.getNullsAction() != NullsAction::EMPTY)
|
if (node.getNullsAction() != NullsAction::EMPTY)
|
||||||
throw Exception(
|
throw Exception(
|
||||||
ErrorCodes::SYNTAX_ERROR,
|
ErrorCodes::SYNTAX_ERROR,
|
||||||
"Function with name '{}' cannot use {} NULLS",
|
"Function with name {} cannot use {} NULLS",
|
||||||
node.getFunctionName(),
|
backQuote(node.getFunctionName()),
|
||||||
node.getNullsAction() == NullsAction::IGNORE_NULLS ? "IGNORE" : "RESPECT");
|
node.getNullsAction() == NullsAction::IGNORE_NULLS ? "IGNORE" : "RESPECT");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -3228,16 +3228,16 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
|
|||||||
auto hints = NamePrompter<2>::getHints(function_name, possible_function_names);
|
auto hints = NamePrompter<2>::getHints(function_name, possible_function_names);
|
||||||
|
|
||||||
throw Exception(ErrorCodes::UNKNOWN_FUNCTION,
|
throw Exception(ErrorCodes::UNKNOWN_FUNCTION,
|
||||||
"Function with name '{}' does not exist. In scope {}{}",
|
"Function with name {} does not exist. In scope {}{}",
|
||||||
function_name,
|
backQuote(function_name),
|
||||||
scope.scope_node->formatASTForErrorMessage(),
|
scope.scope_node->formatASTForErrorMessage(),
|
||||||
getHintsErrorMessageSuffix(hints));
|
getHintsErrorMessageSuffix(hints));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!function_lambda_arguments_indexes.empty())
|
if (!function_lambda_arguments_indexes.empty())
|
||||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
||||||
"Aggregate function '{}' does not support lambda arguments",
|
"Aggregate function {} does not support lambda arguments",
|
||||||
function_name);
|
backQuote(function_name));
|
||||||
|
|
||||||
auto action = function_node_ptr->getNullsAction();
|
auto action = function_node_ptr->getNullsAction();
|
||||||
std::string aggregate_function_name = rewriteAggregateFunctionNameIfNeeded(function_name, action, scope.context);
|
std::string aggregate_function_name = rewriteAggregateFunctionNameIfNeeded(function_name, action, scope.context);
|
||||||
@ -3679,10 +3679,10 @@ ProjectionNames QueryAnalyzer::resolveExpressionNode(
|
|||||||
|
|
||||||
auto hints = IdentifierResolver::collectIdentifierTypoHints(unresolved_identifier, valid_identifiers);
|
auto hints = IdentifierResolver::collectIdentifierTypoHints(unresolved_identifier, valid_identifiers);
|
||||||
|
|
||||||
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Unknown {}{} identifier '{}' in scope {}{}",
|
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER, "Unknown {}{} identifier {} in scope {}{}",
|
||||||
toStringLowercase(IdentifierLookupContext::EXPRESSION),
|
toStringLowercase(IdentifierLookupContext::EXPRESSION),
|
||||||
message_clarification,
|
message_clarification,
|
||||||
unresolved_identifier.getFullName(),
|
backQuote(unresolved_identifier.getFullName()),
|
||||||
scope.scope_node->formatASTForErrorMessage(),
|
scope.scope_node->formatASTForErrorMessage(),
|
||||||
getHintsErrorMessageSuffix(hints));
|
getHintsErrorMessageSuffix(hints));
|
||||||
}
|
}
|
||||||
|
@ -1875,11 +1875,11 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
|
|||||||
|
|
||||||
if (const auto * create_user_query = parsed_query->as<ASTCreateUserQuery>())
|
if (const auto * create_user_query = parsed_query->as<ASTCreateUserQuery>())
|
||||||
{
|
{
|
||||||
if (!create_user_query->attach && create_user_query->auth_data)
|
if (!create_user_query->attach && !create_user_query->authentication_methods.empty())
|
||||||
{
|
{
|
||||||
if (const auto * auth_data = create_user_query->auth_data->as<ASTAuthenticationData>())
|
for (const auto & authentication_method : create_user_query->authentication_methods)
|
||||||
{
|
{
|
||||||
auto password = auth_data->getPassword();
|
auto password = authentication_method->getPassword();
|
||||||
|
|
||||||
if (password)
|
if (password)
|
||||||
client_context->getAccessControl().checkPasswordComplexityRules(*password);
|
client_context->getAccessControl().checkPasswordComplexityRules(*password);
|
||||||
|
@ -455,6 +455,9 @@ void Connection::sendAddendum()
|
|||||||
writeStringBinary(proto_recv_chunked, *out);
|
writeStringBinary(proto_recv_chunked, *out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (server_revision >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL)
|
||||||
|
writeVarUInt(DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION, *out);
|
||||||
|
|
||||||
out->next();
|
out->next();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -525,6 +528,8 @@ void Connection::receiveHello(const Poco::Timespan & handshake_timeout)
|
|||||||
readVarUInt(server_version_major, *in);
|
readVarUInt(server_version_major, *in);
|
||||||
readVarUInt(server_version_minor, *in);
|
readVarUInt(server_version_minor, *in);
|
||||||
readVarUInt(server_revision, *in);
|
readVarUInt(server_revision, *in);
|
||||||
|
if (server_revision >= DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL)
|
||||||
|
readVarUInt(server_parallel_replicas_protocol_version, *in);
|
||||||
if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE)
|
if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE)
|
||||||
readStringBinary(server_timezone, *in);
|
readStringBinary(server_timezone, *in);
|
||||||
if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
|
if (server_revision >= DBMS_MIN_REVISION_WITH_SERVER_DISPLAY_NAME)
|
||||||
@ -959,7 +964,7 @@ void Connection::sendReadTaskResponse(const String & response)
|
|||||||
void Connection::sendMergeTreeReadTaskResponse(const ParallelReadResponse & response)
|
void Connection::sendMergeTreeReadTaskResponse(const ParallelReadResponse & response)
|
||||||
{
|
{
|
||||||
writeVarUInt(Protocol::Client::MergeTreeReadTaskResponse, *out);
|
writeVarUInt(Protocol::Client::MergeTreeReadTaskResponse, *out);
|
||||||
response.serialize(*out);
|
response.serialize(*out, server_parallel_replicas_protocol_version);
|
||||||
out->finishChunk();
|
out->finishChunk();
|
||||||
out->next();
|
out->next();
|
||||||
}
|
}
|
||||||
@ -1413,7 +1418,7 @@ ParallelReadRequest Connection::receiveParallelReadRequest() const
|
|||||||
|
|
||||||
InitialAllRangesAnnouncement Connection::receiveInitialParallelReadAnnouncement() const
|
InitialAllRangesAnnouncement Connection::receiveInitialParallelReadAnnouncement() const
|
||||||
{
|
{
|
||||||
return InitialAllRangesAnnouncement::deserialize(*in);
|
return InitialAllRangesAnnouncement::deserialize(*in, server_parallel_replicas_protocol_version);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@ -210,6 +210,7 @@ private:
|
|||||||
UInt64 server_version_minor = 0;
|
UInt64 server_version_minor = 0;
|
||||||
UInt64 server_version_patch = 0;
|
UInt64 server_version_patch = 0;
|
||||||
UInt64 server_revision = 0;
|
UInt64 server_revision = 0;
|
||||||
|
UInt64 server_parallel_replicas_protocol_version = 0;
|
||||||
String server_timezone;
|
String server_timezone;
|
||||||
String server_display_name;
|
String server_display_name;
|
||||||
|
|
||||||
|
@ -816,6 +816,22 @@ void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// If it's not null we update hash with the type name and the actual value.
|
||||||
|
|
||||||
|
/// If value in this row is in shared variant, deserialize type and value and
|
||||||
|
/// update hash with it.
|
||||||
|
if (discr == getSharedVariantDiscriminator())
|
||||||
|
{
|
||||||
|
auto value = getSharedVariant().getDataAt(variant_col.offsetAt(n));
|
||||||
|
ReadBufferFromMemory buf(value.data, value.size);
|
||||||
|
auto type = decodeDataType(buf);
|
||||||
|
hash.update(type->getName());
|
||||||
|
auto tmp_column = type->createColumn();
|
||||||
|
type->getDefaultSerialization()->deserializeBinary(*tmp_column, buf, getFormatSettings());
|
||||||
|
tmp_column->updateHashWithValue(0, hash);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
hash.update(variant_info.variant_names[discr]);
|
hash.update(variant_info.variant_names[discr]);
|
||||||
variant_col.getVariantByGlobalDiscriminator(discr).updateHashWithValue(variant_col.offsetAt(n), hash);
|
variant_col.getVariantByGlobalDiscriminator(discr).updateHashWithValue(variant_col.offsetAt(n), hash);
|
||||||
}
|
}
|
||||||
|
@ -47,15 +47,21 @@ ColumnObject::ColumnObject(
|
|||||||
, statistics(statistics_)
|
, statistics(statistics_)
|
||||||
{
|
{
|
||||||
typed_paths.reserve(typed_paths_.size());
|
typed_paths.reserve(typed_paths_.size());
|
||||||
|
sorted_typed_paths.reserve(typed_paths_.size());
|
||||||
for (auto & [path, column] : typed_paths_)
|
for (auto & [path, column] : typed_paths_)
|
||||||
typed_paths[path] = std::move(column);
|
{
|
||||||
|
auto it = typed_paths.emplace(path, std::move(column)).first;
|
||||||
|
sorted_typed_paths.push_back(it->first);
|
||||||
|
}
|
||||||
|
std::sort(sorted_typed_paths.begin(), sorted_typed_paths.end());
|
||||||
|
|
||||||
dynamic_paths.reserve(dynamic_paths_.size());
|
dynamic_paths.reserve(dynamic_paths_.size());
|
||||||
dynamic_paths_ptrs.reserve(dynamic_paths_.size());
|
dynamic_paths_ptrs.reserve(dynamic_paths_.size());
|
||||||
for (auto & [path, column] : dynamic_paths_)
|
for (auto & [path, column] : dynamic_paths_)
|
||||||
{
|
{
|
||||||
dynamic_paths[path] = std::move(column);
|
auto it = dynamic_paths.emplace(path, std::move(column)).first;
|
||||||
dynamic_paths_ptrs[path] = assert_cast<ColumnDynamic *>(dynamic_paths[path].get());
|
dynamic_paths_ptrs[path] = assert_cast<ColumnDynamic *>(it->second.get());
|
||||||
|
sorted_dynamic_paths.insert(it->first);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -64,13 +70,17 @@ ColumnObject::ColumnObject(
|
|||||||
: max_dynamic_paths(max_dynamic_paths_), global_max_dynamic_paths(max_dynamic_paths_), max_dynamic_types(max_dynamic_types_)
|
: max_dynamic_paths(max_dynamic_paths_), global_max_dynamic_paths(max_dynamic_paths_), max_dynamic_types(max_dynamic_types_)
|
||||||
{
|
{
|
||||||
typed_paths.reserve(typed_paths_.size());
|
typed_paths.reserve(typed_paths_.size());
|
||||||
|
sorted_typed_paths.reserve(typed_paths_.size());
|
||||||
for (auto & [path, column] : typed_paths_)
|
for (auto & [path, column] : typed_paths_)
|
||||||
{
|
{
|
||||||
if (!column->empty())
|
if (!column->empty())
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected non-empty typed path column in ColumnObject constructor");
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected non-empty typed path column in ColumnObject constructor");
|
||||||
typed_paths[path] = std::move(column);
|
auto it = typed_paths.emplace(path, std::move(column)).first;
|
||||||
|
sorted_typed_paths.push_back(it->first);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
std::sort(sorted_typed_paths.begin(), sorted_typed_paths.end());
|
||||||
|
|
||||||
MutableColumns paths_and_values;
|
MutableColumns paths_and_values;
|
||||||
paths_and_values.emplace_back(ColumnString::create());
|
paths_and_values.emplace_back(ColumnString::create());
|
||||||
paths_and_values.emplace_back(ColumnString::create());
|
paths_and_values.emplace_back(ColumnString::create());
|
||||||
@ -129,13 +139,8 @@ std::string ColumnObject::getName() const
|
|||||||
ss << "Object(";
|
ss << "Object(";
|
||||||
ss << "max_dynamic_paths=" << global_max_dynamic_paths;
|
ss << "max_dynamic_paths=" << global_max_dynamic_paths;
|
||||||
ss << ", max_dynamic_types=" << max_dynamic_types;
|
ss << ", max_dynamic_types=" << max_dynamic_types;
|
||||||
std::vector<String> sorted_typed_paths;
|
|
||||||
sorted_typed_paths.reserve(typed_paths.size());
|
|
||||||
for (const auto & [path, column] : typed_paths)
|
|
||||||
sorted_typed_paths.push_back(path);
|
|
||||||
std::sort(sorted_typed_paths.begin(), sorted_typed_paths.end());
|
|
||||||
for (const auto & path : sorted_typed_paths)
|
for (const auto & path : sorted_typed_paths)
|
||||||
ss << ", " << path << " " << typed_paths.at(path)->getName();
|
ss << ", " << path << " " << typed_paths.find(path)->second->getName();
|
||||||
ss << ")";
|
ss << ")";
|
||||||
return ss.str();
|
return ss.str();
|
||||||
}
|
}
|
||||||
@ -260,6 +265,7 @@ ColumnDynamic * ColumnObject::tryToAddNewDynamicPath(std::string_view path)
|
|||||||
new_dynamic_column->insertManyDefaults(size());
|
new_dynamic_column->insertManyDefaults(size());
|
||||||
auto it = dynamic_paths.emplace(path, std::move(new_dynamic_column)).first;
|
auto it = dynamic_paths.emplace(path, std::move(new_dynamic_column)).first;
|
||||||
auto it_ptr = dynamic_paths_ptrs.emplace(path, assert_cast<ColumnDynamic *>(it->second.get())).first;
|
auto it_ptr = dynamic_paths_ptrs.emplace(path, assert_cast<ColumnDynamic *>(it->second.get())).first;
|
||||||
|
sorted_dynamic_paths.insert(it->first);
|
||||||
return it_ptr->second;
|
return it_ptr->second;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -288,8 +294,9 @@ void ColumnObject::setDynamicPaths(const std::vector<String> & paths)
|
|||||||
auto new_dynamic_column = ColumnDynamic::create(max_dynamic_types);
|
auto new_dynamic_column = ColumnDynamic::create(max_dynamic_types);
|
||||||
if (size)
|
if (size)
|
||||||
new_dynamic_column->insertManyDefaults(size);
|
new_dynamic_column->insertManyDefaults(size);
|
||||||
dynamic_paths[path] = std::move(new_dynamic_column);
|
auto it = dynamic_paths.emplace(path, std::move(new_dynamic_column)).first;
|
||||||
dynamic_paths_ptrs[path] = assert_cast<ColumnDynamic *>(dynamic_paths[path].get());
|
dynamic_paths_ptrs[path] = assert_cast<ColumnDynamic *>(it->second.get());
|
||||||
|
sorted_dynamic_paths.insert(it->first);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -658,39 +665,61 @@ void ColumnObject::popBack(size_t n)
|
|||||||
StringRef ColumnObject::serializeValueIntoArena(size_t n, Arena & arena, const char *& begin) const
|
StringRef ColumnObject::serializeValueIntoArena(size_t n, Arena & arena, const char *& begin) const
|
||||||
{
|
{
|
||||||
StringRef res(begin, 0);
|
StringRef res(begin, 0);
|
||||||
// Serialize all paths and values in binary format.
|
/// First serialize values from typed paths in sorted order. They are the same for all instances of this column.
|
||||||
|
for (auto path : sorted_typed_paths)
|
||||||
|
{
|
||||||
|
auto data_ref = typed_paths.find(path)->second->serializeValueIntoArena(n, arena, begin);
|
||||||
|
res.data = data_ref.data - res.size;
|
||||||
|
res.size += data_ref.size;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Second, serialize paths and values in bunary format from dynamic paths and shared data in sorted by path order.
|
||||||
|
/// Calculate total number of paths to serialize and write it.
|
||||||
const auto & shared_data_offsets = getSharedDataOffsets();
|
const auto & shared_data_offsets = getSharedDataOffsets();
|
||||||
size_t offset = shared_data_offsets[static_cast<ssize_t>(n) - 1];
|
size_t offset = shared_data_offsets[static_cast<ssize_t>(n) - 1];
|
||||||
size_t end = shared_data_offsets[static_cast<ssize_t>(n)];
|
size_t end = shared_data_offsets[static_cast<ssize_t>(n)];
|
||||||
size_t num_paths = typed_paths.size() + dynamic_paths.size() + (end - offset);
|
size_t num_paths = (end - offset);
|
||||||
|
/// Don't serialize Nulls from dynamic paths.
|
||||||
|
for (const auto & [_, column] : dynamic_paths)
|
||||||
|
num_paths += !column->isNullAt(n);
|
||||||
|
|
||||||
char * pos = arena.allocContinue(sizeof(size_t), begin);
|
char * pos = arena.allocContinue(sizeof(size_t), begin);
|
||||||
memcpy(pos, &num_paths, sizeof(size_t));
|
memcpy(pos, &num_paths, sizeof(size_t));
|
||||||
res.data = pos - res.size;
|
res.data = pos - res.size;
|
||||||
res.size += sizeof(size_t);
|
res.size += sizeof(size_t);
|
||||||
/// Serialize paths and values from typed paths.
|
|
||||||
for (const auto & [path, column] : typed_paths)
|
|
||||||
{
|
|
||||||
size_t path_size = path.size();
|
|
||||||
pos = arena.allocContinue(sizeof(size_t) + path_size, begin);
|
|
||||||
memcpy(pos, &path_size, sizeof(size_t));
|
|
||||||
memcpy(pos + sizeof(size_t), path.data(), path_size);
|
|
||||||
auto data_ref = column->serializeValueIntoArena(n, arena, begin);
|
|
||||||
res.data = data_ref.data - res.size - sizeof(size_t) - path_size;
|
|
||||||
res.size += data_ref.size + sizeof(size_t) + path_size;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Serialize paths and values from dynamic paths.
|
auto dynamic_paths_it = sorted_dynamic_paths.begin();
|
||||||
for (const auto & [path, column] : dynamic_paths)
|
|
||||||
{
|
|
||||||
WriteBufferFromOwnString buf;
|
|
||||||
getDynamicSerialization()->serializeBinary(*column, n, buf, getFormatSettings());
|
|
||||||
serializePathAndValueIntoArena(arena, begin, path, buf.str(), res);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Serialize paths and values from shared data.
|
|
||||||
auto [shared_data_paths, shared_data_values] = getSharedDataPathsAndValues();
|
auto [shared_data_paths, shared_data_values] = getSharedDataPathsAndValues();
|
||||||
for (size_t i = offset; i != end; ++i)
|
for (size_t i = offset; i != end; ++i)
|
||||||
serializePathAndValueIntoArena(arena, begin, shared_data_paths->getDataAt(i), shared_data_values->getDataAt(i), res);
|
{
|
||||||
|
auto path = shared_data_paths->getDataAt(i).toView();
|
||||||
|
/// Paths in shared data are sorted. Serialize all paths from dynamic paths that go before this path in sorted order.
|
||||||
|
while (dynamic_paths_it != sorted_dynamic_paths.end() && *dynamic_paths_it < path)
|
||||||
|
{
|
||||||
|
const auto * dynamic_column = dynamic_paths_ptrs.find(*dynamic_paths_it)->second;
|
||||||
|
/// Don't serialize Nulls.
|
||||||
|
if (!dynamic_column->isNullAt(n))
|
||||||
|
{
|
||||||
|
WriteBufferFromOwnString buf;
|
||||||
|
getDynamicSerialization()->serializeBinary(*dynamic_column, n, buf, getFormatSettings());
|
||||||
|
serializePathAndValueIntoArena(arena, begin, StringRef(*dynamic_paths_it), buf.str(), res);
|
||||||
|
}
|
||||||
|
++dynamic_paths_it;
|
||||||
|
}
|
||||||
|
serializePathAndValueIntoArena(arena, begin, StringRef(path), shared_data_values->getDataAt(i), res);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Serialize all remaining paths in dynamic paths.
|
||||||
|
for (; dynamic_paths_it != sorted_dynamic_paths.end(); ++dynamic_paths_it)
|
||||||
|
{
|
||||||
|
const auto * dynamic_column = dynamic_paths_ptrs.find(*dynamic_paths_it)->second;
|
||||||
|
if (!dynamic_column->isNullAt(n))
|
||||||
|
{
|
||||||
|
WriteBufferFromOwnString buf;
|
||||||
|
getDynamicSerialization()->serializeBinary(*dynamic_column, n, buf, getFormatSettings());
|
||||||
|
serializePathAndValueIntoArena(arena, begin, StringRef(*dynamic_paths_it), buf.str(), res);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
@ -711,27 +740,21 @@ void ColumnObject::serializePathAndValueIntoArena(DB::Arena & arena, const char
|
|||||||
const char * ColumnObject::deserializeAndInsertFromArena(const char * pos)
|
const char * ColumnObject::deserializeAndInsertFromArena(const char * pos)
|
||||||
{
|
{
|
||||||
size_t current_size = size();
|
size_t current_size = size();
|
||||||
/// Deserialize paths and values and insert them into typed paths, dynamic paths or shared data.
|
/// First deserialize typed paths. They come first.
|
||||||
/// Serialized paths could be unsorted, so we will have to sort all paths that will be inserted into shared data.
|
for (auto path : sorted_typed_paths)
|
||||||
std::vector<std::pair<std::string_view, std::string_view>> paths_and_values_for_shared_data;
|
pos = typed_paths.find(path)->second->deserializeAndInsertFromArena(pos);
|
||||||
|
|
||||||
|
/// Second deserialize all other paths and values and insert them into dynamic paths or shared data.
|
||||||
auto num_paths = unalignedLoad<size_t>(pos);
|
auto num_paths = unalignedLoad<size_t>(pos);
|
||||||
pos += sizeof(size_t);
|
pos += sizeof(size_t);
|
||||||
|
const auto [shared_data_paths, shared_data_values] = getSharedDataPathsAndValues();
|
||||||
for (size_t i = 0; i != num_paths; ++i)
|
for (size_t i = 0; i != num_paths; ++i)
|
||||||
{
|
{
|
||||||
auto path_size = unalignedLoad<size_t>(pos);
|
auto path_size = unalignedLoad<size_t>(pos);
|
||||||
pos += sizeof(size_t);
|
pos += sizeof(size_t);
|
||||||
std::string_view path(pos, path_size);
|
std::string_view path(pos, path_size);
|
||||||
pos += path_size;
|
pos += path_size;
|
||||||
/// Check if it's a typed path. In this case we should use
|
/// Deserialize binary value and try to insert it to dynamic paths or shared data.
|
||||||
/// deserializeAndInsertFromArena of corresponding column.
|
|
||||||
if (auto typed_it = typed_paths.find(path); typed_it != typed_paths.end())
|
|
||||||
{
|
|
||||||
pos = typed_it->second->deserializeAndInsertFromArena(pos);
|
|
||||||
}
|
|
||||||
/// If it's not a typed path, deserialize binary value and try to insert it
|
|
||||||
/// to dynamic paths or shared data.
|
|
||||||
else
|
|
||||||
{
|
|
||||||
auto value_size = unalignedLoad<size_t>(pos);
|
auto value_size = unalignedLoad<size_t>(pos);
|
||||||
pos += sizeof(size_t);
|
pos += sizeof(size_t);
|
||||||
std::string_view value(pos, value_size);
|
std::string_view value(pos, value_size);
|
||||||
@ -748,33 +771,18 @@ const char * ColumnObject::deserializeAndInsertFromArena(const char * pos)
|
|||||||
ReadBufferFromMemory buf(value.data(), value.size());
|
ReadBufferFromMemory buf(value.data(), value.size());
|
||||||
getDynamicSerialization()->deserializeBinary(*dynamic_path_column, buf, getFormatSettings());
|
getDynamicSerialization()->deserializeBinary(*dynamic_path_column, buf, getFormatSettings());
|
||||||
}
|
}
|
||||||
/// Limit on dynamic paths is reached, add this path to shared data later.
|
/// Limit on dynamic paths is reached, add this path to shared data.
|
||||||
|
/// Serialized paths are sorted, so we can insert right away.
|
||||||
else
|
else
|
||||||
{
|
|
||||||
paths_and_values_for_shared_data.emplace_back(path, value);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Sort and insert all paths from paths_and_values_for_shared_data into shared data.
|
|
||||||
std::sort(paths_and_values_for_shared_data.begin(), paths_and_values_for_shared_data.end());
|
|
||||||
const auto [shared_data_paths, shared_data_values] = getSharedDataPathsAndValues();
|
|
||||||
for (const auto & [path, value] : paths_and_values_for_shared_data)
|
|
||||||
{
|
{
|
||||||
shared_data_paths->insertData(path.data(), path.size());
|
shared_data_paths->insertData(path.data(), path.size());
|
||||||
shared_data_values->insertData(value.data(), value.size());
|
shared_data_values->insertData(value.data(), value.size());
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
getSharedDataOffsets().push_back(shared_data_paths->size());
|
getSharedDataOffsets().push_back(shared_data_paths->size());
|
||||||
|
|
||||||
/// Insert default value in all remaining typed and dynamic paths.
|
/// Insert default value in all remaining dynamic paths.
|
||||||
|
|
||||||
for (auto & [_, column] : typed_paths)
|
|
||||||
{
|
|
||||||
if (column->size() == current_size)
|
|
||||||
column->insertDefault();
|
|
||||||
}
|
|
||||||
|
|
||||||
for (auto & [_, column] : dynamic_paths_ptrs)
|
for (auto & [_, column] : dynamic_paths_ptrs)
|
||||||
{
|
{
|
||||||
if (column->size() == current_size)
|
if (column->size() == current_size)
|
||||||
@ -786,6 +794,11 @@ const char * ColumnObject::deserializeAndInsertFromArena(const char * pos)
|
|||||||
|
|
||||||
const char * ColumnObject::skipSerializedInArena(const char * pos) const
|
const char * ColumnObject::skipSerializedInArena(const char * pos) const
|
||||||
{
|
{
|
||||||
|
/// First, skip all values of typed paths;
|
||||||
|
for (auto path : sorted_typed_paths)
|
||||||
|
pos = typed_paths.find(path)->second->skipSerializedInArena(pos);
|
||||||
|
|
||||||
|
/// Second, skip all other paths and values.
|
||||||
auto num_paths = unalignedLoad<size_t>(pos);
|
auto num_paths = unalignedLoad<size_t>(pos);
|
||||||
pos += sizeof(size_t);
|
pos += sizeof(size_t);
|
||||||
for (size_t i = 0; i != num_paths; ++i)
|
for (size_t i = 0; i != num_paths; ++i)
|
||||||
@ -794,27 +807,60 @@ const char * ColumnObject::skipSerializedInArena(const char * pos) const
|
|||||||
pos += sizeof(size_t);
|
pos += sizeof(size_t);
|
||||||
std::string_view path(pos, path_size);
|
std::string_view path(pos, path_size);
|
||||||
pos += path_size;
|
pos += path_size;
|
||||||
if (auto typed_it = typed_paths.find(path); typed_it != typed_paths.end())
|
|
||||||
{
|
|
||||||
pos = typed_it->second->skipSerializedInArena(pos);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
auto value_size = unalignedLoad<size_t>(pos);
|
auto value_size = unalignedLoad<size_t>(pos);
|
||||||
pos += sizeof(size_t) + value_size;
|
pos += sizeof(size_t) + value_size;
|
||||||
}
|
}
|
||||||
}
|
|
||||||
|
|
||||||
return pos;
|
return pos;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ColumnObject::updateHashWithValue(size_t n, SipHash & hash) const
|
void ColumnObject::updateHashWithValue(size_t n, SipHash & hash) const
|
||||||
{
|
{
|
||||||
for (const auto & [_, column] : typed_paths)
|
for (auto path : sorted_typed_paths)
|
||||||
column->updateHashWithValue(n, hash);
|
typed_paths.find(path)->second->updateHashWithValue(n, hash);
|
||||||
for (const auto & [_, column] : dynamic_paths_ptrs)
|
|
||||||
column->updateHashWithValue(n, hash);
|
/// The hash of the object in row should not depend on the way we store paths (in dynamic paths or in shared data)
|
||||||
shared_data->updateHashWithValue(n, hash);
|
/// and should be the same for the same objects. To support it we update hash with path and its value (if not null) in
|
||||||
|
/// sorted by path order from both dynamic paths and shared data.
|
||||||
|
const auto [shared_data_paths, shared_data_values] = getSharedDataPathsAndValues();
|
||||||
|
const auto & shared_data_offsets = getSharedDataOffsets();
|
||||||
|
size_t start = shared_data_offsets[static_cast<ssize_t>(n) - 1];
|
||||||
|
size_t end = shared_data_offsets[static_cast<ssize_t>(n)];
|
||||||
|
auto dynamic_paths_it = sorted_dynamic_paths.begin();
|
||||||
|
for (size_t i = start; i != end; ++i)
|
||||||
|
{
|
||||||
|
auto path = shared_data_paths->getDataAt(i).toView();
|
||||||
|
/// Paths in shared data are sorted. Update hash with all paths from dynamic paths that go before this path in sorted order.
|
||||||
|
while (dynamic_paths_it != sorted_dynamic_paths.end() && *dynamic_paths_it < path)
|
||||||
|
{
|
||||||
|
const auto * dynamic_column = dynamic_paths_ptrs.find(*dynamic_paths_it)->second;
|
||||||
|
if (!dynamic_column->isNullAt(n))
|
||||||
|
{
|
||||||
|
hash.update(*dynamic_paths_it);
|
||||||
|
dynamic_column->updateHashWithValue(n, hash);
|
||||||
|
}
|
||||||
|
++dynamic_paths_it;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Deserialize value in temporary column to get its hash.
|
||||||
|
auto value = shared_data_values->getDataAt(i);
|
||||||
|
ReadBufferFromMemory buf(value.data, value.size);
|
||||||
|
auto tmp_column = ColumnDynamic::create();
|
||||||
|
getDynamicSerialization()->deserializeBinary(*tmp_column, buf, getFormatSettings());
|
||||||
|
hash.update(path);
|
||||||
|
tmp_column->updateHashWithValue(0, hash);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Iterate over all remaining paths in dynamic paths.
|
||||||
|
for (; dynamic_paths_it != sorted_dynamic_paths.end(); ++dynamic_paths_it)
|
||||||
|
{
|
||||||
|
const auto * dynamic_column = dynamic_paths_ptrs.find(*dynamic_paths_it)->second;
|
||||||
|
if (!dynamic_column->isNullAt(n))
|
||||||
|
{
|
||||||
|
hash.update(*dynamic_paths_it);
|
||||||
|
dynamic_column->updateHashWithValue(n, hash);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
WeakHash32 ColumnObject::getWeakHash32() const
|
WeakHash32 ColumnObject::getWeakHash32() const
|
||||||
@ -1310,6 +1356,7 @@ void ColumnObject::takeDynamicStructureFromSourceColumns(const DB::Columns & sou
|
|||||||
/// Reset current state.
|
/// Reset current state.
|
||||||
dynamic_paths.clear();
|
dynamic_paths.clear();
|
||||||
dynamic_paths_ptrs.clear();
|
dynamic_paths_ptrs.clear();
|
||||||
|
sorted_dynamic_paths.clear();
|
||||||
max_dynamic_paths = global_max_dynamic_paths;
|
max_dynamic_paths = global_max_dynamic_paths;
|
||||||
Statistics new_statistics(Statistics::Source::MERGE);
|
Statistics new_statistics(Statistics::Source::MERGE);
|
||||||
|
|
||||||
@ -1328,8 +1375,9 @@ void ColumnObject::takeDynamicStructureFromSourceColumns(const DB::Columns & sou
|
|||||||
{
|
{
|
||||||
if (dynamic_paths.size() < max_dynamic_paths)
|
if (dynamic_paths.size() < max_dynamic_paths)
|
||||||
{
|
{
|
||||||
dynamic_paths.emplace(path, ColumnDynamic::create(max_dynamic_types));
|
auto it = dynamic_paths.emplace(path, ColumnDynamic::create(max_dynamic_types)).first;
|
||||||
dynamic_paths_ptrs.emplace(path, assert_cast<ColumnDynamic *>(dynamic_paths.find(path)->second.get()));
|
dynamic_paths_ptrs.emplace(path, assert_cast<ColumnDynamic *>(it->second.get()));
|
||||||
|
sorted_dynamic_paths.insert(it->first);
|
||||||
}
|
}
|
||||||
/// Add all remaining paths into shared data statistics until we reach its max size;
|
/// Add all remaining paths into shared data statistics until we reach its max size;
|
||||||
else if (new_statistics.shared_data_paths_statistics.size() < Statistics::MAX_SHARED_DATA_STATISTICS_SIZE)
|
else if (new_statistics.shared_data_paths_statistics.size() < Statistics::MAX_SHARED_DATA_STATISTICS_SIZE)
|
||||||
@ -1343,8 +1391,9 @@ void ColumnObject::takeDynamicStructureFromSourceColumns(const DB::Columns & sou
|
|||||||
{
|
{
|
||||||
for (const auto & [path, _] : path_to_total_number_of_non_null_values)
|
for (const auto & [path, _] : path_to_total_number_of_non_null_values)
|
||||||
{
|
{
|
||||||
dynamic_paths[path] = ColumnDynamic::create(max_dynamic_types);
|
auto it = dynamic_paths.emplace(path, ColumnDynamic::create(max_dynamic_types)).first;
|
||||||
dynamic_paths_ptrs[path] = assert_cast<ColumnDynamic *>(dynamic_paths[path].get());
|
dynamic_paths_ptrs[path] = assert_cast<ColumnDynamic *>(it->second.get());
|
||||||
|
sorted_dynamic_paths.insert(it->first);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -238,10 +238,15 @@ private:
|
|||||||
/// Map path -> column for paths with explicitly specified types.
|
/// Map path -> column for paths with explicitly specified types.
|
||||||
/// This set of paths is constant and cannot be changed.
|
/// This set of paths is constant and cannot be changed.
|
||||||
PathToColumnMap typed_paths;
|
PathToColumnMap typed_paths;
|
||||||
|
/// Sorted list of typed paths. Used to avoid sorting paths every time in some methods.
|
||||||
|
std::vector<std::string_view> sorted_typed_paths;
|
||||||
/// Map path -> column for dynamically added paths. All columns
|
/// Map path -> column for dynamically added paths. All columns
|
||||||
/// here are Dynamic columns. This set of paths can be extended
|
/// here are Dynamic columns. This set of paths can be extended
|
||||||
/// during inerts into the column.
|
/// during inerts into the column.
|
||||||
PathToColumnMap dynamic_paths;
|
PathToColumnMap dynamic_paths;
|
||||||
|
/// Sorted list of dynamic paths. Used to avoid sorting paths every time in some methods.
|
||||||
|
std::set<std::string_view> sorted_dynamic_paths;
|
||||||
|
|
||||||
/// Store and use pointers to ColumnDynamic to avoid virtual calls.
|
/// Store and use pointers to ColumnDynamic to avoid virtual calls.
|
||||||
/// With hundreds of dynamic paths these virtual calls are noticeable.
|
/// With hundreds of dynamic paths these virtual calls are noticeable.
|
||||||
PathToDynamicColumnPtrMap dynamic_paths_ptrs;
|
PathToDynamicColumnPtrMap dynamic_paths_ptrs;
|
||||||
|
@ -64,6 +64,7 @@ static struct InitFiu
|
|||||||
REGULAR(lazy_pipe_fds_fail_close) \
|
REGULAR(lazy_pipe_fds_fail_close) \
|
||||||
PAUSEABLE(infinite_sleep) \
|
PAUSEABLE(infinite_sleep) \
|
||||||
PAUSEABLE(stop_moving_part_before_swap_with_active) \
|
PAUSEABLE(stop_moving_part_before_swap_with_active) \
|
||||||
|
REGULAR(slowdown_index_analysis) \
|
||||||
|
|
||||||
|
|
||||||
namespace FailPoints
|
namespace FailPoints
|
||||||
|
@ -376,6 +376,7 @@ The server successfully detected this situation and will download merged part fr
|
|||||||
M(ParallelReplicasReadAssignedMarks, "Sum across all replicas of how many of scheduled marks were assigned by consistent hash") \
|
M(ParallelReplicasReadAssignedMarks, "Sum across all replicas of how many of scheduled marks were assigned by consistent hash") \
|
||||||
M(ParallelReplicasReadUnassignedMarks, "Sum across all replicas of how many unassigned marks were scheduled") \
|
M(ParallelReplicasReadUnassignedMarks, "Sum across all replicas of how many unassigned marks were scheduled") \
|
||||||
M(ParallelReplicasReadAssignedForStealingMarks, "Sum across all replicas of how many of scheduled marks were assigned for stealing by consistent hash") \
|
M(ParallelReplicasReadAssignedForStealingMarks, "Sum across all replicas of how many of scheduled marks were assigned for stealing by consistent hash") \
|
||||||
|
M(ParallelReplicasReadMarks, "How many marks were read by the given replica") \
|
||||||
\
|
\
|
||||||
M(ParallelReplicasStealingByHashMicroseconds, "Time spent collecting segments meant for stealing by hash") \
|
M(ParallelReplicasStealingByHashMicroseconds, "Time spent collecting segments meant for stealing by hash") \
|
||||||
M(ParallelReplicasProcessingPartsMicroseconds, "Time spent processing data parts") \
|
M(ParallelReplicasProcessingPartsMicroseconds, "Time spent processing data parts") \
|
||||||
@ -529,6 +530,7 @@ The server successfully detected this situation and will download merged part fr
|
|||||||
M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache") \
|
M(CachedReadBufferReadFromCacheMicroseconds, "Time reading from filesystem cache") \
|
||||||
M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)") \
|
M(CachedReadBufferReadFromSourceBytes, "Bytes read from filesystem cache source (from remote fs, etc)") \
|
||||||
M(CachedReadBufferReadFromCacheBytes, "Bytes read from filesystem cache") \
|
M(CachedReadBufferReadFromCacheBytes, "Bytes read from filesystem cache") \
|
||||||
|
M(CachedReadBufferPredownloadedBytes, "Bytes read from filesystem cache source. Cache segments are read from left to right as a whole, it might be that we need to predownload some part of the segment irrelevant for the current task just to get to the needed data") \
|
||||||
M(CachedReadBufferCacheWriteBytes, "Bytes written from source (remote fs, etc) to filesystem cache") \
|
M(CachedReadBufferCacheWriteBytes, "Bytes written from source (remote fs, etc) to filesystem cache") \
|
||||||
M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into filesystem cache") \
|
M(CachedReadBufferCacheWriteMicroseconds, "Time spent writing data into filesystem cache") \
|
||||||
M(CachedReadBufferCreateBufferMicroseconds, "Prepare buffer time") \
|
M(CachedReadBufferCreateBufferMicroseconds, "Prepare buffer time") \
|
||||||
|
@ -181,12 +181,6 @@ void SetACLRequest::addRootPath(const String & root_path) { Coordination::addRoo
|
|||||||
void GetACLRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); }
|
void GetACLRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); }
|
||||||
void SyncRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); }
|
void SyncRequest::addRootPath(const String & root_path) { Coordination::addRootPath(path, root_path); }
|
||||||
|
|
||||||
void MultiRequest::addRootPath(const String & root_path)
|
|
||||||
{
|
|
||||||
for (auto & request : requests)
|
|
||||||
request->addRootPath(root_path);
|
|
||||||
}
|
|
||||||
|
|
||||||
void CreateResponse::removeRootPath(const String & root_path) { Coordination::removeRootPath(path_created, root_path); }
|
void CreateResponse::removeRootPath(const String & root_path) { Coordination::removeRootPath(path_created, root_path); }
|
||||||
void WatchResponse::removeRootPath(const String & root_path) { Coordination::removeRootPath(path, root_path); }
|
void WatchResponse::removeRootPath(const String & root_path) { Coordination::removeRootPath(path, root_path); }
|
||||||
|
|
||||||
|
@ -408,11 +408,17 @@ struct ReconfigResponse : virtual Response
|
|||||||
size_t bytesSize() const override { return value.size() + sizeof(stat); }
|
size_t bytesSize() const override { return value.size() + sizeof(stat); }
|
||||||
};
|
};
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
struct MultiRequest : virtual Request
|
struct MultiRequest : virtual Request
|
||||||
{
|
{
|
||||||
Requests requests;
|
std::vector<T> requests;
|
||||||
|
|
||||||
|
void addRootPath(const String & root_path) override
|
||||||
|
{
|
||||||
|
for (auto & request : requests)
|
||||||
|
request->addRootPath(root_path);
|
||||||
|
}
|
||||||
|
|
||||||
void addRootPath(const String & root_path) override;
|
|
||||||
String getPath() const override { return {}; }
|
String getPath() const override { return {}; }
|
||||||
|
|
||||||
size_t bytesSize() const override
|
size_t bytesSize() const override
|
||||||
|
@ -184,7 +184,7 @@ struct TestKeeperReconfigRequest final : ReconfigRequest, TestKeeperRequest
|
|||||||
std::pair<ResponsePtr, Undo> process(TestKeeper::Container & container, int64_t zxid) const override;
|
std::pair<ResponsePtr, Undo> process(TestKeeper::Container & container, int64_t zxid) const override;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct TestKeeperMultiRequest final : MultiRequest, TestKeeperRequest
|
struct TestKeeperMultiRequest final : MultiRequest<RequestPtr>, TestKeeperRequest
|
||||||
{
|
{
|
||||||
explicit TestKeeperMultiRequest(const Requests & generic_requests)
|
explicit TestKeeperMultiRequest(const Requests & generic_requests)
|
||||||
: TestKeeperMultiRequest(std::span(generic_requests))
|
: TestKeeperMultiRequest(std::span(generic_requests))
|
||||||
|
@ -18,14 +18,16 @@ using namespace DB;
|
|||||||
|
|
||||||
void ZooKeeperResponse::write(WriteBuffer & out) const
|
void ZooKeeperResponse::write(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
/// Excessive copy to calculate length.
|
auto response_size = Coordination::size(xid) + Coordination::size(zxid) + Coordination::size(error);
|
||||||
WriteBufferFromOwnString buf;
|
|
||||||
Coordination::write(xid, buf);
|
|
||||||
Coordination::write(zxid, buf);
|
|
||||||
Coordination::write(error, buf);
|
|
||||||
if (error == Error::ZOK)
|
if (error == Error::ZOK)
|
||||||
writeImpl(buf);
|
response_size += sizeImpl();
|
||||||
Coordination::write(buf.str(), out);
|
|
||||||
|
Coordination::write(static_cast<int32_t>(response_size), out);
|
||||||
|
Coordination::write(xid, out);
|
||||||
|
Coordination::write(zxid, out);
|
||||||
|
Coordination::write(error, out);
|
||||||
|
if (error == Error::ZOK)
|
||||||
|
writeImpl(out);
|
||||||
}
|
}
|
||||||
|
|
||||||
std::string ZooKeeperRequest::toString(bool short_format) const
|
std::string ZooKeeperRequest::toString(bool short_format) const
|
||||||
@ -41,12 +43,12 @@ std::string ZooKeeperRequest::toString(bool short_format) const
|
|||||||
|
|
||||||
void ZooKeeperRequest::write(WriteBuffer & out) const
|
void ZooKeeperRequest::write(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
/// Excessive copy to calculate length.
|
auto request_size = Coordination::size(xid) + Coordination::size(getOpNum()) + sizeImpl();
|
||||||
WriteBufferFromOwnString buf;
|
|
||||||
Coordination::write(xid, buf);
|
Coordination::write(static_cast<int32_t>(request_size), out);
|
||||||
Coordination::write(getOpNum(), buf);
|
Coordination::write(xid, out);
|
||||||
writeImpl(buf);
|
Coordination::write(getOpNum(), out);
|
||||||
Coordination::write(buf.str(), out);
|
writeImpl(out);
|
||||||
}
|
}
|
||||||
|
|
||||||
void ZooKeeperSyncRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperSyncRequest::writeImpl(WriteBuffer & out) const
|
||||||
@ -54,6 +56,11 @@ void ZooKeeperSyncRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSyncRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSyncRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperSyncRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -74,6 +81,11 @@ void ZooKeeperSyncResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSyncResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperReconfigRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperReconfigRequest::writeImpl(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
Coordination::write(joining, out);
|
Coordination::write(joining, out);
|
||||||
@ -82,6 +94,11 @@ void ZooKeeperReconfigRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(version, out);
|
Coordination::write(version, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperReconfigRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(joining) + Coordination::size(leaving) + Coordination::size(new_members) + Coordination::size(version);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperReconfigRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperReconfigRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(joining, in);
|
Coordination::read(joining, in);
|
||||||
@ -109,6 +126,11 @@ void ZooKeeperReconfigResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(stat, out);
|
Coordination::write(stat, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperReconfigResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(value) + Coordination::size(stat);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperWatchResponse::readImpl(ReadBuffer & in)
|
void ZooKeeperWatchResponse::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(type, in);
|
Coordination::read(type, in);
|
||||||
@ -123,6 +145,11 @@ void ZooKeeperWatchResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperWatchResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(type) + Coordination::size(state) + Coordination::size(path);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperWatchResponse::write(WriteBuffer & out) const
|
void ZooKeeperWatchResponse::write(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
if (error == Error::ZOK)
|
if (error == Error::ZOK)
|
||||||
@ -137,6 +164,11 @@ void ZooKeeperAuthRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(data, out);
|
Coordination::write(data, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperAuthRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(type) + Coordination::size(scheme) + Coordination::size(data);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperAuthRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperAuthRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(type, in);
|
Coordination::read(type, in);
|
||||||
@ -175,6 +207,12 @@ void ZooKeeperCreateRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(flags, out);
|
Coordination::write(flags, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperCreateRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
int32_t flags = 0;
|
||||||
|
return Coordination::size(path) + Coordination::size(data) + Coordination::size(acls) + Coordination::size(flags);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperCreateRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperCreateRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -211,12 +249,22 @@ void ZooKeeperCreateResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(path_created, out);
|
Coordination::write(path_created, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperCreateResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path_created);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperRemoveRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperRemoveRequest::writeImpl(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
Coordination::write(version, out);
|
Coordination::write(version, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperRemoveRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(version);
|
||||||
|
}
|
||||||
|
|
||||||
std::string ZooKeeperRemoveRequest::toStringImpl(bool /*short_format*/) const
|
std::string ZooKeeperRemoveRequest::toStringImpl(bool /*short_format*/) const
|
||||||
{
|
{
|
||||||
return fmt::format(
|
return fmt::format(
|
||||||
@ -244,6 +292,11 @@ void ZooKeeperRemoveRecursiveRequest::readImpl(ReadBuffer & in)
|
|||||||
Coordination::read(remove_nodes_limit, in);
|
Coordination::read(remove_nodes_limit, in);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperRemoveRecursiveRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(remove_nodes_limit);
|
||||||
|
}
|
||||||
|
|
||||||
std::string ZooKeeperRemoveRecursiveRequest::toStringImpl(bool /*short_format*/) const
|
std::string ZooKeeperRemoveRecursiveRequest::toStringImpl(bool /*short_format*/) const
|
||||||
{
|
{
|
||||||
return fmt::format(
|
return fmt::format(
|
||||||
@ -259,6 +312,11 @@ void ZooKeeperExistsRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(has_watch, out);
|
Coordination::write(has_watch, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperExistsRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(has_watch);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperExistsRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperExistsRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -280,12 +338,22 @@ void ZooKeeperExistsResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(stat, out);
|
Coordination::write(stat, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperExistsResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(stat);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperGetRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperGetRequest::writeImpl(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
Coordination::write(has_watch, out);
|
Coordination::write(has_watch, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperGetRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(has_watch);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperGetRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperGetRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -309,6 +377,11 @@ void ZooKeeperGetResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(stat, out);
|
Coordination::write(stat, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperGetResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(data) + Coordination::size(stat);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSetRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperSetRequest::writeImpl(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
@ -316,6 +389,11 @@ void ZooKeeperSetRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(version, out);
|
Coordination::write(version, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSetRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(data) + Coordination::size(version);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSetRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperSetRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -342,12 +420,22 @@ void ZooKeeperSetResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(stat, out);
|
Coordination::write(stat, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSetResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(stat);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperListRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperListRequest::writeImpl(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
Coordination::write(has_watch, out);
|
Coordination::write(has_watch, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperListRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(has_watch);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperListRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperListRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -366,6 +454,11 @@ void ZooKeeperFilteredListRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(static_cast<uint8_t>(list_request_type), out);
|
Coordination::write(static_cast<uint8_t>(list_request_type), out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperFilteredListRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(has_watch) + Coordination::size(static_cast<uint8_t>(list_request_type));
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperFilteredListRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperFilteredListRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -397,6 +490,11 @@ void ZooKeeperListResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(stat, out);
|
Coordination::write(stat, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperListResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(names) + Coordination::size(stat);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSimpleListResponse::readImpl(ReadBuffer & in)
|
void ZooKeeperSimpleListResponse::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(names, in);
|
Coordination::read(names, in);
|
||||||
@ -407,6 +505,11 @@ void ZooKeeperSimpleListResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(names, out);
|
Coordination::write(names, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSimpleListResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(names);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSetACLRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperSetACLRequest::writeImpl(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
@ -414,6 +517,11 @@ void ZooKeeperSetACLRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(version, out);
|
Coordination::write(version, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSetACLRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(acls) + Coordination::size(version);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSetACLRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperSetACLRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -431,6 +539,11 @@ void ZooKeeperSetACLResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(stat, out);
|
Coordination::write(stat, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSetACLResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(stat);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSetACLResponse::readImpl(ReadBuffer & in)
|
void ZooKeeperSetACLResponse::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(stat, in);
|
Coordination::read(stat, in);
|
||||||
@ -446,6 +559,11 @@ void ZooKeeperGetACLRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(path, out);
|
Coordination::write(path, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperGetACLRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path);
|
||||||
|
}
|
||||||
|
|
||||||
std::string ZooKeeperGetACLRequest::toStringImpl(bool /*short_format*/) const
|
std::string ZooKeeperGetACLRequest::toStringImpl(bool /*short_format*/) const
|
||||||
{
|
{
|
||||||
return fmt::format("path = {}", path);
|
return fmt::format("path = {}", path);
|
||||||
@ -457,6 +575,11 @@ void ZooKeeperGetACLResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(stat, out);
|
Coordination::write(stat, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperGetACLResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(acl) + Coordination::size(stat);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperGetACLResponse::readImpl(ReadBuffer & in)
|
void ZooKeeperGetACLResponse::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(acl, in);
|
Coordination::read(acl, in);
|
||||||
@ -469,6 +592,11 @@ void ZooKeeperCheckRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(version, out);
|
Coordination::write(version, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperCheckRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(path) + Coordination::size(version);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperCheckRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperCheckRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(path, in);
|
Coordination::read(path, in);
|
||||||
@ -494,6 +622,11 @@ void ZooKeeperErrorResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(error, out);
|
Coordination::write(error, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperErrorResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(error);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperMultiRequest::checkOperationType(OperationType type)
|
void ZooKeeperMultiRequest::checkOperationType(OperationType type)
|
||||||
{
|
{
|
||||||
chassert(!operation_type.has_value() || *operation_type == type);
|
chassert(!operation_type.has_value() || *operation_type == type);
|
||||||
@ -596,6 +729,27 @@ void ZooKeeperMultiRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(error, out);
|
Coordination::write(error, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperMultiRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
size_t total_size = 0;
|
||||||
|
for (const auto & request : requests)
|
||||||
|
{
|
||||||
|
const auto & zk_request = dynamic_cast<const ZooKeeperRequest &>(*request);
|
||||||
|
|
||||||
|
bool done = false;
|
||||||
|
int32_t error = -1;
|
||||||
|
|
||||||
|
total_size
|
||||||
|
+= Coordination::size(zk_request.getOpNum()) + Coordination::size(done) + Coordination::size(error) + zk_request.sizeImpl();
|
||||||
|
}
|
||||||
|
|
||||||
|
OpNum op_num = OpNum::Error;
|
||||||
|
bool done = true;
|
||||||
|
int32_t error = -1;
|
||||||
|
|
||||||
|
return total_size + Coordination::size(op_num) + Coordination::size(done) + Coordination::size(error);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperMultiRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperMultiRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
while (true)
|
while (true)
|
||||||
@ -729,31 +883,54 @@ void ZooKeeperMultiResponse::writeImpl(WriteBuffer & out) const
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
ZooKeeperResponsePtr ZooKeeperHeartbeatRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperHeartbeatResponse>()); }
|
size_t ZooKeeperMultiResponse::sizeImpl() const
|
||||||
ZooKeeperResponsePtr ZooKeeperSyncRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperSyncResponse>()); }
|
{
|
||||||
ZooKeeperResponsePtr ZooKeeperAuthRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperAuthResponse>()); }
|
size_t total_size = 0;
|
||||||
ZooKeeperResponsePtr ZooKeeperRemoveRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperRemoveResponse>()); }
|
for (const auto & response : responses)
|
||||||
ZooKeeperResponsePtr ZooKeeperRemoveRecursiveRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperRemoveRecursiveResponse>()); }
|
{
|
||||||
ZooKeeperResponsePtr ZooKeeperExistsRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperExistsResponse>()); }
|
const ZooKeeperResponse & zk_response = dynamic_cast<const ZooKeeperResponse &>(*response);
|
||||||
ZooKeeperResponsePtr ZooKeeperGetRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperGetResponse>()); }
|
OpNum op_num = zk_response.getOpNum();
|
||||||
ZooKeeperResponsePtr ZooKeeperSetRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperSetResponse>()); }
|
bool done = false;
|
||||||
ZooKeeperResponsePtr ZooKeeperReconfigRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperReconfigResponse>()); }
|
Error op_error = zk_response.error;
|
||||||
ZooKeeperResponsePtr ZooKeeperListRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperListResponse>()); }
|
|
||||||
ZooKeeperResponsePtr ZooKeeperSimpleListRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperSimpleListResponse>()); }
|
total_size += Coordination::size(op_num) + Coordination::size(done) + Coordination::size(op_error);
|
||||||
|
if (op_error == Error::ZOK || op_num == OpNum::Error)
|
||||||
|
total_size += zk_response.sizeImpl();
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Footer.
|
||||||
|
OpNum op_num = OpNum::Error;
|
||||||
|
bool done = true;
|
||||||
|
int32_t error_read = - 1;
|
||||||
|
|
||||||
|
return total_size + Coordination::size(op_num) + Coordination::size(done) + Coordination::size(error_read);
|
||||||
|
}
|
||||||
|
|
||||||
|
ZooKeeperResponsePtr ZooKeeperHeartbeatRequest::makeResponse() const { return std::make_shared<ZooKeeperHeartbeatResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperSyncRequest::makeResponse() const { return std::make_shared<ZooKeeperSyncResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperAuthRequest::makeResponse() const { return std::make_shared<ZooKeeperAuthResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperRemoveRequest::makeResponse() const { return std::make_shared<ZooKeeperRemoveResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperRemoveRecursiveRequest::makeResponse() const { return std::make_shared<ZooKeeperRemoveRecursiveResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperExistsRequest::makeResponse() const { return std::make_shared<ZooKeeperExistsResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperGetRequest::makeResponse() const { return std::make_shared<ZooKeeperGetResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperSetRequest::makeResponse() const { return std::make_shared<ZooKeeperSetResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperReconfigRequest::makeResponse() const { return std::make_shared<ZooKeeperReconfigResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperListRequest::makeResponse() const { return std::make_shared<ZooKeeperListResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperSimpleListRequest::makeResponse() const { return std::make_shared<ZooKeeperSimpleListResponse>(); }
|
||||||
|
|
||||||
ZooKeeperResponsePtr ZooKeeperCreateRequest::makeResponse() const
|
ZooKeeperResponsePtr ZooKeeperCreateRequest::makeResponse() const
|
||||||
{
|
{
|
||||||
if (not_exists)
|
if (not_exists)
|
||||||
return setTime(std::make_shared<ZooKeeperCreateIfNotExistsResponse>());
|
return std::make_shared<ZooKeeperCreateIfNotExistsResponse>();
|
||||||
return setTime(std::make_shared<ZooKeeperCreateResponse>());
|
return std::make_shared<ZooKeeperCreateResponse>();
|
||||||
}
|
}
|
||||||
|
|
||||||
ZooKeeperResponsePtr ZooKeeperCheckRequest::makeResponse() const
|
ZooKeeperResponsePtr ZooKeeperCheckRequest::makeResponse() const
|
||||||
{
|
{
|
||||||
if (not_exists)
|
if (not_exists)
|
||||||
return setTime(std::make_shared<ZooKeeperCheckNotExistsResponse>());
|
return std::make_shared<ZooKeeperCheckNotExistsResponse>();
|
||||||
|
|
||||||
return setTime(std::make_shared<ZooKeeperCheckResponse>());
|
return std::make_shared<ZooKeeperCheckResponse>();
|
||||||
}
|
}
|
||||||
|
|
||||||
ZooKeeperResponsePtr ZooKeeperMultiRequest::makeResponse() const
|
ZooKeeperResponsePtr ZooKeeperMultiRequest::makeResponse() const
|
||||||
@ -764,11 +941,12 @@ ZooKeeperResponsePtr ZooKeeperMultiRequest::makeResponse() const
|
|||||||
else
|
else
|
||||||
response = std::make_shared<ZooKeeperMultiReadResponse>(requests);
|
response = std::make_shared<ZooKeeperMultiReadResponse>(requests);
|
||||||
|
|
||||||
return setTime(std::move(response));
|
return std::move(response);
|
||||||
}
|
}
|
||||||
ZooKeeperResponsePtr ZooKeeperCloseRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperCloseResponse>()); }
|
|
||||||
ZooKeeperResponsePtr ZooKeeperSetACLRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperSetACLResponse>()); }
|
ZooKeeperResponsePtr ZooKeeperCloseRequest::makeResponse() const { return std::make_shared<ZooKeeperCloseResponse>(); }
|
||||||
ZooKeeperResponsePtr ZooKeeperGetACLRequest::makeResponse() const { return setTime(std::make_shared<ZooKeeperGetACLResponse>()); }
|
ZooKeeperResponsePtr ZooKeeperSetACLRequest::makeResponse() const { return std::make_shared<ZooKeeperSetACLResponse>(); }
|
||||||
|
ZooKeeperResponsePtr ZooKeeperGetACLRequest::makeResponse() const { return std::make_shared<ZooKeeperGetACLResponse>(); }
|
||||||
|
|
||||||
void ZooKeeperSessionIDRequest::writeImpl(WriteBuffer & out) const
|
void ZooKeeperSessionIDRequest::writeImpl(WriteBuffer & out) const
|
||||||
{
|
{
|
||||||
@ -777,6 +955,11 @@ void ZooKeeperSessionIDRequest::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(server_id, out);
|
Coordination::write(server_id, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSessionIDRequest::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(internal_id) + Coordination::size(session_timeout_ms) + Coordination::size(server_id);
|
||||||
|
}
|
||||||
|
|
||||||
void ZooKeeperSessionIDRequest::readImpl(ReadBuffer & in)
|
void ZooKeeperSessionIDRequest::readImpl(ReadBuffer & in)
|
||||||
{
|
{
|
||||||
Coordination::read(internal_id, in);
|
Coordination::read(internal_id, in);
|
||||||
@ -803,6 +986,11 @@ void ZooKeeperSessionIDResponse::writeImpl(WriteBuffer & out) const
|
|||||||
Coordination::write(server_id, out);
|
Coordination::write(server_id, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t ZooKeeperSessionIDResponse::sizeImpl() const
|
||||||
|
{
|
||||||
|
return Coordination::size(internal_id) + Coordination::size(session_id) + Coordination::size(server_id);
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
void ZooKeeperRequest::createLogElements(LogElements & elems) const
|
void ZooKeeperRequest::createLogElements(LogElements & elems) const
|
||||||
{
|
{
|
||||||
@ -960,40 +1148,6 @@ std::shared_ptr<ZooKeeperRequest> ZooKeeperRequest::read(ReadBuffer & in)
|
|||||||
return request;
|
return request;
|
||||||
}
|
}
|
||||||
|
|
||||||
ZooKeeperRequest::~ZooKeeperRequest()
|
|
||||||
{
|
|
||||||
if (!request_created_time_ns)
|
|
||||||
return;
|
|
||||||
UInt64 elapsed_ns = clock_gettime_ns() - request_created_time_ns;
|
|
||||||
constexpr UInt64 max_request_time_ns = 1000000000ULL; /// 1 sec
|
|
||||||
if (max_request_time_ns < elapsed_ns)
|
|
||||||
{
|
|
||||||
LOG_TEST(getLogger(__PRETTY_FUNCTION__), "Processing of request xid={} took {} ms", xid, elapsed_ns / 1000000UL);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
ZooKeeperResponsePtr ZooKeeperRequest::setTime(ZooKeeperResponsePtr response) const
|
|
||||||
{
|
|
||||||
if (request_created_time_ns)
|
|
||||||
{
|
|
||||||
response->response_created_time_ns = clock_gettime_ns();
|
|
||||||
}
|
|
||||||
return response;
|
|
||||||
}
|
|
||||||
|
|
||||||
ZooKeeperResponse::~ZooKeeperResponse()
|
|
||||||
{
|
|
||||||
if (!response_created_time_ns)
|
|
||||||
return;
|
|
||||||
UInt64 elapsed_ns = clock_gettime_ns() - response_created_time_ns;
|
|
||||||
constexpr UInt64 max_request_time_ns = 1000000000ULL; /// 1 sec
|
|
||||||
if (max_request_time_ns < elapsed_ns)
|
|
||||||
{
|
|
||||||
LOG_TEST(getLogger(__PRETTY_FUNCTION__), "Processing of response xid={} took {} ms", xid, elapsed_ns / 1000000UL);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
ZooKeeperRequestPtr ZooKeeperRequestFactory::get(OpNum op_num) const
|
ZooKeeperRequestPtr ZooKeeperRequestFactory::get(OpNum op_num) const
|
||||||
{
|
{
|
||||||
auto it = op_num_to_request.find(op_num);
|
auto it = op_num_to_request.find(op_num);
|
||||||
@ -1015,7 +1169,6 @@ void registerZooKeeperRequest(ZooKeeperRequestFactory & factory)
|
|||||||
factory.registerRequest(num, []
|
factory.registerRequest(num, []
|
||||||
{
|
{
|
||||||
auto res = std::make_shared<RequestT>();
|
auto res = std::make_shared<RequestT>();
|
||||||
res->request_created_time_ns = clock_gettime_ns();
|
|
||||||
|
|
||||||
if constexpr (num == OpNum::MultiRead)
|
if constexpr (num == OpNum::MultiRead)
|
||||||
res->operation_type = ZooKeeperMultiRequest::OperationType::Read;
|
res->operation_type = ZooKeeperMultiRequest::OperationType::Read;
|
||||||
|
@ -7,13 +7,11 @@
|
|||||||
#include <boost/noncopyable.hpp>
|
#include <boost/noncopyable.hpp>
|
||||||
#include <IO/ReadBuffer.h>
|
#include <IO/ReadBuffer.h>
|
||||||
#include <IO/WriteBuffer.h>
|
#include <IO/WriteBuffer.h>
|
||||||
#include <unordered_map>
|
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <memory>
|
#include <memory>
|
||||||
#include <cstdint>
|
#include <cstdint>
|
||||||
#include <optional>
|
#include <optional>
|
||||||
#include <functional>
|
#include <functional>
|
||||||
#include <span>
|
|
||||||
|
|
||||||
|
|
||||||
namespace Coordination
|
namespace Coordination
|
||||||
@ -25,13 +23,11 @@ struct ZooKeeperResponse : virtual Response
|
|||||||
{
|
{
|
||||||
XID xid = 0;
|
XID xid = 0;
|
||||||
|
|
||||||
UInt64 response_created_time_ns = 0;
|
|
||||||
|
|
||||||
ZooKeeperResponse() = default;
|
ZooKeeperResponse() = default;
|
||||||
ZooKeeperResponse(const ZooKeeperResponse &) = default;
|
ZooKeeperResponse(const ZooKeeperResponse &) = default;
|
||||||
~ZooKeeperResponse() override;
|
|
||||||
virtual void readImpl(ReadBuffer &) = 0;
|
virtual void readImpl(ReadBuffer &) = 0;
|
||||||
virtual void writeImpl(WriteBuffer &) const = 0;
|
virtual void writeImpl(WriteBuffer &) const = 0;
|
||||||
|
virtual size_t sizeImpl() const = 0;
|
||||||
virtual void write(WriteBuffer & out) const;
|
virtual void write(WriteBuffer & out) const;
|
||||||
virtual OpNum getOpNum() const = 0;
|
virtual OpNum getOpNum() const = 0;
|
||||||
virtual void fillLogElements(LogElements & elems, size_t idx) const;
|
virtual void fillLogElements(LogElements & elems, size_t idx) const;
|
||||||
@ -51,13 +47,11 @@ struct ZooKeeperRequest : virtual Request
|
|||||||
|
|
||||||
bool restored_from_zookeeper_log = false;
|
bool restored_from_zookeeper_log = false;
|
||||||
|
|
||||||
UInt64 request_created_time_ns = 0;
|
|
||||||
UInt64 thread_id = 0;
|
UInt64 thread_id = 0;
|
||||||
String query_id;
|
String query_id;
|
||||||
|
|
||||||
ZooKeeperRequest() = default;
|
ZooKeeperRequest() = default;
|
||||||
ZooKeeperRequest(const ZooKeeperRequest &) = default;
|
ZooKeeperRequest(const ZooKeeperRequest &) = default;
|
||||||
~ZooKeeperRequest() override;
|
|
||||||
|
|
||||||
virtual OpNum getOpNum() const = 0;
|
virtual OpNum getOpNum() const = 0;
|
||||||
|
|
||||||
@ -66,6 +60,7 @@ struct ZooKeeperRequest : virtual Request
|
|||||||
std::string toString(bool short_format = false) const;
|
std::string toString(bool short_format = false) const;
|
||||||
|
|
||||||
virtual void writeImpl(WriteBuffer &) const = 0;
|
virtual void writeImpl(WriteBuffer &) const = 0;
|
||||||
|
virtual size_t sizeImpl() const = 0;
|
||||||
virtual void readImpl(ReadBuffer &) = 0;
|
virtual void readImpl(ReadBuffer &) = 0;
|
||||||
|
|
||||||
virtual std::string toStringImpl(bool /*short_format*/) const { return ""; }
|
virtual std::string toStringImpl(bool /*short_format*/) const { return ""; }
|
||||||
@ -73,7 +68,6 @@ struct ZooKeeperRequest : virtual Request
|
|||||||
static std::shared_ptr<ZooKeeperRequest> read(ReadBuffer & in);
|
static std::shared_ptr<ZooKeeperRequest> read(ReadBuffer & in);
|
||||||
|
|
||||||
virtual ZooKeeperResponsePtr makeResponse() const = 0;
|
virtual ZooKeeperResponsePtr makeResponse() const = 0;
|
||||||
ZooKeeperResponsePtr setTime(ZooKeeperResponsePtr response) const;
|
|
||||||
virtual bool isReadRequest() const = 0;
|
virtual bool isReadRequest() const = 0;
|
||||||
|
|
||||||
virtual void createLogElements(LogElements & elems) const;
|
virtual void createLogElements(LogElements & elems) const;
|
||||||
@ -86,6 +80,7 @@ struct ZooKeeperHeartbeatRequest final : ZooKeeperRequest
|
|||||||
String getPath() const override { return {}; }
|
String getPath() const override { return {}; }
|
||||||
OpNum getOpNum() const override { return OpNum::Heartbeat; }
|
OpNum getOpNum() const override { return OpNum::Heartbeat; }
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
void readImpl(ReadBuffer &) override {}
|
void readImpl(ReadBuffer &) override {}
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
bool isReadRequest() const override { return false; }
|
bool isReadRequest() const override { return false; }
|
||||||
@ -97,6 +92,7 @@ struct ZooKeeperSyncRequest final : ZooKeeperRequest
|
|||||||
String getPath() const override { return path; }
|
String getPath() const override { return path; }
|
||||||
OpNum getOpNum() const override { return OpNum::Sync; }
|
OpNum getOpNum() const override { return OpNum::Sync; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -109,6 +105,7 @@ struct ZooKeeperSyncResponse final : SyncResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::Sync; }
|
OpNum getOpNum() const override { return OpNum::Sync; }
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -122,6 +119,7 @@ struct ZooKeeperReconfigRequest final : ZooKeeperRequest
|
|||||||
String getPath() const override { return keeper_config_path; }
|
String getPath() const override { return keeper_config_path; }
|
||||||
OpNum getOpNum() const override { return OpNum::Reconfig; }
|
OpNum getOpNum() const override { return OpNum::Reconfig; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -138,6 +136,7 @@ struct ZooKeeperReconfigResponse final : ReconfigResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::Reconfig; }
|
OpNum getOpNum() const override { return OpNum::Reconfig; }
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -145,6 +144,7 @@ struct ZooKeeperHeartbeatResponse final : ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer &) override {}
|
void readImpl(ReadBuffer &) override {}
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
OpNum getOpNum() const override { return OpNum::Heartbeat; }
|
OpNum getOpNum() const override { return OpNum::Heartbeat; }
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -153,6 +153,7 @@ struct ZooKeeperWatchResponse final : WatchResponse, ZooKeeperResponse
|
|||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
|
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
|
|
||||||
void write(WriteBuffer & out) const override;
|
void write(WriteBuffer & out) const override;
|
||||||
|
|
||||||
@ -175,6 +176,7 @@ struct ZooKeeperAuthRequest final : ZooKeeperRequest
|
|||||||
String getPath() const override { return {}; }
|
String getPath() const override { return {}; }
|
||||||
OpNum getOpNum() const override { return OpNum::Auth; }
|
OpNum getOpNum() const override { return OpNum::Auth; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -189,6 +191,7 @@ struct ZooKeeperAuthResponse final : ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer &) override {}
|
void readImpl(ReadBuffer &) override {}
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Auth; }
|
OpNum getOpNum() const override { return OpNum::Auth; }
|
||||||
|
|
||||||
@ -200,6 +203,7 @@ struct ZooKeeperCloseRequest final : ZooKeeperRequest
|
|||||||
String getPath() const override { return {}; }
|
String getPath() const override { return {}; }
|
||||||
OpNum getOpNum() const override { return OpNum::Close; }
|
OpNum getOpNum() const override { return OpNum::Close; }
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
void readImpl(ReadBuffer &) override {}
|
void readImpl(ReadBuffer &) override {}
|
||||||
|
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -214,6 +218,7 @@ struct ZooKeeperCloseResponse final : ZooKeeperResponse
|
|||||||
}
|
}
|
||||||
|
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Close; }
|
OpNum getOpNum() const override { return OpNum::Close; }
|
||||||
};
|
};
|
||||||
@ -228,6 +233,7 @@ struct ZooKeeperCreateRequest final : public CreateRequest, ZooKeeperRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return not_exists ? OpNum::CreateIfNotExists : OpNum::Create; }
|
OpNum getOpNum() const override { return not_exists ? OpNum::CreateIfNotExists : OpNum::Create; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -244,6 +250,7 @@ struct ZooKeeperCreateResponse : CreateResponse, ZooKeeperResponse
|
|||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
|
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Create; }
|
OpNum getOpNum() const override { return OpNum::Create; }
|
||||||
|
|
||||||
@ -265,6 +272,7 @@ struct ZooKeeperRemoveRequest final : RemoveRequest, ZooKeeperRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Remove; }
|
OpNum getOpNum() const override { return OpNum::Remove; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -280,6 +288,7 @@ struct ZooKeeperRemoveResponse final : RemoveResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer &) override {}
|
void readImpl(ReadBuffer &) override {}
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
OpNum getOpNum() const override { return OpNum::Remove; }
|
OpNum getOpNum() const override { return OpNum::Remove; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return RemoveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return RemoveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -293,6 +302,7 @@ struct ZooKeeperRemoveRecursiveRequest final : RemoveRecursiveRequest, ZooKeeper
|
|||||||
OpNum getOpNum() const override { return OpNum::RemoveRecursive; }
|
OpNum getOpNum() const override { return OpNum::RemoveRecursive; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -305,6 +315,7 @@ struct ZooKeeperRemoveRecursiveResponse : RemoveRecursiveResponse, ZooKeeperResp
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer &) override {}
|
void readImpl(ReadBuffer &) override {}
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
OpNum getOpNum() const override { return OpNum::RemoveRecursive; }
|
OpNum getOpNum() const override { return OpNum::RemoveRecursive; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return RemoveRecursiveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return RemoveRecursiveResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -317,6 +328,7 @@ struct ZooKeeperExistsRequest final : ExistsRequest, ZooKeeperRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Exists; }
|
OpNum getOpNum() const override { return OpNum::Exists; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -330,6 +342,7 @@ struct ZooKeeperExistsResponse final : ExistsResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::Exists; }
|
OpNum getOpNum() const override { return OpNum::Exists; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return ExistsResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return ExistsResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -344,6 +357,7 @@ struct ZooKeeperGetRequest final : GetRequest, ZooKeeperRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Get; }
|
OpNum getOpNum() const override { return OpNum::Get; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -357,6 +371,7 @@ struct ZooKeeperGetResponse final : GetResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::Get; }
|
OpNum getOpNum() const override { return OpNum::Get; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return GetResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return GetResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -371,6 +386,7 @@ struct ZooKeeperSetRequest final : SetRequest, ZooKeeperRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Set; }
|
OpNum getOpNum() const override { return OpNum::Set; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -385,6 +401,7 @@ struct ZooKeeperSetResponse final : SetResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::Set; }
|
OpNum getOpNum() const override { return OpNum::Set; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return SetResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return SetResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -399,6 +416,7 @@ struct ZooKeeperListRequest : ListRequest, ZooKeeperRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::List; }
|
OpNum getOpNum() const override { return OpNum::List; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -419,6 +437,7 @@ struct ZooKeeperFilteredListRequest final : ZooKeeperListRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::FilteredList; }
|
OpNum getOpNum() const override { return OpNum::FilteredList; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -429,6 +448,7 @@ struct ZooKeeperListResponse : ListResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::List; }
|
OpNum getOpNum() const override { return OpNum::List; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return ListResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return ListResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -440,6 +460,7 @@ struct ZooKeeperSimpleListResponse final : ZooKeeperListResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::SimpleList; }
|
OpNum getOpNum() const override { return OpNum::SimpleList; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return ZooKeeperListResponse::bytesSize() - sizeof(stat); }
|
size_t bytesSize() const override { return ZooKeeperListResponse::bytesSize() - sizeof(stat); }
|
||||||
@ -452,6 +473,7 @@ struct ZooKeeperCheckRequest : CheckRequest, ZooKeeperRequest
|
|||||||
|
|
||||||
OpNum getOpNum() const override { return not_exists ? OpNum::CheckNotExists : OpNum::Check; }
|
OpNum getOpNum() const override { return not_exists ? OpNum::CheckNotExists : OpNum::Check; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -467,6 +489,7 @@ struct ZooKeeperCheckResponse : CheckResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer &) override {}
|
void readImpl(ReadBuffer &) override {}
|
||||||
void writeImpl(WriteBuffer &) const override {}
|
void writeImpl(WriteBuffer &) const override {}
|
||||||
|
size_t sizeImpl() const override { return 0; }
|
||||||
OpNum getOpNum() const override { return OpNum::Check; }
|
OpNum getOpNum() const override { return OpNum::Check; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return CheckResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return CheckResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -483,6 +506,7 @@ struct ZooKeeperErrorResponse final : ErrorResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
|
|
||||||
OpNum getOpNum() const override { return OpNum::Error; }
|
OpNum getOpNum() const override { return OpNum::Error; }
|
||||||
|
|
||||||
@ -493,6 +517,7 @@ struct ZooKeeperSetACLRequest final : SetACLRequest, ZooKeeperRequest
|
|||||||
{
|
{
|
||||||
OpNum getOpNum() const override { return OpNum::SetACL; }
|
OpNum getOpNum() const override { return OpNum::SetACL; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -505,6 +530,7 @@ struct ZooKeeperSetACLResponse final : SetACLResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::SetACL; }
|
OpNum getOpNum() const override { return OpNum::SetACL; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return SetACLResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return SetACLResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
@ -514,6 +540,7 @@ struct ZooKeeperGetACLRequest final : GetACLRequest, ZooKeeperRequest
|
|||||||
{
|
{
|
||||||
OpNum getOpNum() const override { return OpNum::GetACL; }
|
OpNum getOpNum() const override { return OpNum::GetACL; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
ZooKeeperResponsePtr makeResponse() const override;
|
ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -526,12 +553,13 @@ struct ZooKeeperGetACLResponse final : GetACLResponse, ZooKeeperResponse
|
|||||||
{
|
{
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
OpNum getOpNum() const override { return OpNum::GetACL; }
|
OpNum getOpNum() const override { return OpNum::GetACL; }
|
||||||
|
|
||||||
size_t bytesSize() const override { return GetACLResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return GetACLResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
};
|
};
|
||||||
|
|
||||||
struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest
|
struct ZooKeeperMultiRequest final : MultiRequest<ZooKeeperRequestPtr>, ZooKeeperRequest
|
||||||
{
|
{
|
||||||
OpNum getOpNum() const override;
|
OpNum getOpNum() const override;
|
||||||
ZooKeeperMultiRequest() = default;
|
ZooKeeperMultiRequest() = default;
|
||||||
@ -540,6 +568,7 @@ struct ZooKeeperMultiRequest final : MultiRequest, ZooKeeperRequest
|
|||||||
ZooKeeperMultiRequest(std::span<const Coordination::RequestPtr> generic_requests, const ACLs & default_acls);
|
ZooKeeperMultiRequest(std::span<const Coordination::RequestPtr> generic_requests, const ACLs & default_acls);
|
||||||
|
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
std::string toStringImpl(bool short_format) const override;
|
std::string toStringImpl(bool short_format) const override;
|
||||||
|
|
||||||
@ -563,12 +592,14 @@ private:
|
|||||||
|
|
||||||
struct ZooKeeperMultiResponse : MultiResponse, ZooKeeperResponse
|
struct ZooKeeperMultiResponse : MultiResponse, ZooKeeperResponse
|
||||||
{
|
{
|
||||||
explicit ZooKeeperMultiResponse(const Requests & requests)
|
ZooKeeperMultiResponse() = default;
|
||||||
|
|
||||||
|
explicit ZooKeeperMultiResponse(const std::vector<ZooKeeperRequestPtr> & requests)
|
||||||
{
|
{
|
||||||
responses.reserve(requests.size());
|
responses.reserve(requests.size());
|
||||||
|
|
||||||
for (const auto & request : requests)
|
for (const auto & request : requests)
|
||||||
responses.emplace_back(dynamic_cast<const ZooKeeperRequest &>(*request).makeResponse());
|
responses.emplace_back(request->makeResponse());
|
||||||
}
|
}
|
||||||
|
|
||||||
explicit ZooKeeperMultiResponse(const Responses & responses_)
|
explicit ZooKeeperMultiResponse(const Responses & responses_)
|
||||||
@ -579,6 +610,7 @@ struct ZooKeeperMultiResponse : MultiResponse, ZooKeeperResponse
|
|||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
|
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
|
|
||||||
size_t bytesSize() const override { return MultiResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
size_t bytesSize() const override { return MultiResponse::bytesSize() + sizeof(xid) + sizeof(zxid); }
|
||||||
|
|
||||||
@ -609,6 +641,7 @@ struct ZooKeeperSessionIDRequest final : ZooKeeperRequest
|
|||||||
Coordination::OpNum getOpNum() const override { return OpNum::SessionID; }
|
Coordination::OpNum getOpNum() const override { return OpNum::SessionID; }
|
||||||
String getPath() const override { return {}; }
|
String getPath() const override { return {}; }
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
|
|
||||||
Coordination::ZooKeeperResponsePtr makeResponse() const override;
|
Coordination::ZooKeeperResponsePtr makeResponse() const override;
|
||||||
@ -627,6 +660,7 @@ struct ZooKeeperSessionIDResponse final : ZooKeeperResponse
|
|||||||
void readImpl(ReadBuffer & in) override;
|
void readImpl(ReadBuffer & in) override;
|
||||||
|
|
||||||
void writeImpl(WriteBuffer & out) const override;
|
void writeImpl(WriteBuffer & out) const override;
|
||||||
|
size_t sizeImpl() const override;
|
||||||
|
|
||||||
Coordination::OpNum getOpNum() const override { return OpNum::SessionID; }
|
Coordination::OpNum getOpNum() const override { return OpNum::SessionID; }
|
||||||
};
|
};
|
||||||
|
@ -42,6 +42,32 @@ void write(const Error & x, WriteBuffer & out)
|
|||||||
write(static_cast<int32_t>(x), out);
|
write(static_cast<int32_t>(x), out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
size_t size(OpNum x)
|
||||||
|
{
|
||||||
|
return size(static_cast<int32_t>(x));
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t size(const std::string & s)
|
||||||
|
{
|
||||||
|
return size(static_cast<int32_t>(s.size())) + s.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t size(const ACL & acl)
|
||||||
|
{
|
||||||
|
return size(acl.permissions) + size(acl.scheme) + size(acl.id);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t size(const Stat & stat)
|
||||||
|
{
|
||||||
|
return size(stat.czxid) + size(stat.mzxid) + size(stat.ctime) + size(stat.mtime) + size(stat.version) + size(stat.cversion)
|
||||||
|
+ size(stat.aversion) + size(stat.ephemeralOwner) + size(stat.dataLength) + size(stat.numChildren) + size(stat.pzxid);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t size(const Error & x)
|
||||||
|
{
|
||||||
|
return size(static_cast<int32_t>(x));
|
||||||
|
}
|
||||||
|
|
||||||
void read(OpNum & x, ReadBuffer & in)
|
void read(OpNum & x, ReadBuffer & in)
|
||||||
{
|
{
|
||||||
int32_t raw_op_num;
|
int32_t raw_op_num;
|
||||||
|
@ -43,6 +43,36 @@ void write(const std::vector<T> & arr, WriteBuffer & out)
|
|||||||
write(elem, out);
|
write(elem, out);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
requires is_arithmetic_v<T>
|
||||||
|
size_t size(T x)
|
||||||
|
{
|
||||||
|
return sizeof(x);
|
||||||
|
}
|
||||||
|
|
||||||
|
size_t size(OpNum x);
|
||||||
|
size_t size(const std::string & s);
|
||||||
|
size_t size(const ACL & acl);
|
||||||
|
size_t size(const Stat & stat);
|
||||||
|
size_t size(const Error & x);
|
||||||
|
|
||||||
|
template <size_t N>
|
||||||
|
size_t size(const std::array<char, N>)
|
||||||
|
{
|
||||||
|
return size(static_cast<int32_t>(N)) + N;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename T>
|
||||||
|
size_t size(const std::vector<T> & arr)
|
||||||
|
{
|
||||||
|
size_t total_size = size(static_cast<int32_t>(arr.size()));
|
||||||
|
for (const auto & elem : arr)
|
||||||
|
total_size += size(elem);
|
||||||
|
|
||||||
|
return total_size;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
template <typename T>
|
template <typename T>
|
||||||
requires is_arithmetic_v<T>
|
requires is_arithmetic_v<T>
|
||||||
void read(T & x, ReadBuffer & in)
|
void read(T & x, ReadBuffer & in)
|
||||||
|
@ -1,4 +1,4 @@
|
|||||||
clickhouse_add_executable(integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp)
|
clickhouse_add_executable(integer_hash_tables_and_hashes integer_hash_tables_and_hashes.cpp orc_string_dictionary.cpp)
|
||||||
target_link_libraries (integer_hash_tables_and_hashes PRIVATE
|
target_link_libraries (integer_hash_tables_and_hashes PRIVATE
|
||||||
ch_contrib::gbenchmark_all
|
ch_contrib::gbenchmark_all
|
||||||
dbms
|
dbms
|
||||||
@ -7,3 +7,8 @@ target_link_libraries (integer_hash_tables_and_hashes PRIVATE
|
|||||||
ch_contrib::wyhash
|
ch_contrib::wyhash
|
||||||
ch_contrib::farmhash
|
ch_contrib::farmhash
|
||||||
ch_contrib::xxHash)
|
ch_contrib::xxHash)
|
||||||
|
|
||||||
|
clickhouse_add_executable(orc_string_dictionary orc_string_dictionary.cpp)
|
||||||
|
target_link_libraries (orc_string_dictionary PRIVATE
|
||||||
|
ch_contrib::gbenchmark_all
|
||||||
|
dbms)
|
||||||
|
311
src/Common/benchmarks/orc_string_dictionary.cpp
Normal file
311
src/Common/benchmarks/orc_string_dictionary.cpp
Normal file
@ -0,0 +1,311 @@
|
|||||||
|
#include <cstdlib>
|
||||||
|
#include <base/defines.h>
|
||||||
|
#include <benchmark/benchmark.h>
|
||||||
|
|
||||||
|
class OldSortedStringDictionary
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
struct DictEntry
|
||||||
|
{
|
||||||
|
DictEntry(const char * str, size_t len) : data(str), length(len) { }
|
||||||
|
const char * data;
|
||||||
|
size_t length;
|
||||||
|
};
|
||||||
|
|
||||||
|
OldSortedStringDictionary() : totalLength(0) { }
|
||||||
|
|
||||||
|
// insert a new string into dictionary, return its insertion order
|
||||||
|
size_t insert(const char * str, size_t len);
|
||||||
|
|
||||||
|
// reorder input index buffer from insertion order to dictionary order
|
||||||
|
void reorder(std::vector<int64_t> & idxBuffer) const;
|
||||||
|
|
||||||
|
// get dict entries in insertion order
|
||||||
|
void getEntriesInInsertionOrder(std::vector<const DictEntry *> &) const;
|
||||||
|
|
||||||
|
size_t size() const;
|
||||||
|
|
||||||
|
// return total length of strings in the dictionary
|
||||||
|
uint64_t length() const;
|
||||||
|
|
||||||
|
void clear();
|
||||||
|
|
||||||
|
// store indexes of insertion order in the dictionary for not-null rows
|
||||||
|
std::vector<int64_t> idxInDictBuffer;
|
||||||
|
|
||||||
|
private:
|
||||||
|
struct LessThan
|
||||||
|
{
|
||||||
|
bool operator()(const DictEntry & left, const DictEntry & right) const
|
||||||
|
{
|
||||||
|
int ret = memcmp(left.data, right.data, std::min(left.length, right.length));
|
||||||
|
if (ret != 0)
|
||||||
|
{
|
||||||
|
return ret < 0;
|
||||||
|
}
|
||||||
|
return left.length < right.length;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
std::map<DictEntry, size_t, LessThan> dict;
|
||||||
|
std::vector<std::vector<char>> data;
|
||||||
|
uint64_t totalLength;
|
||||||
|
};
|
||||||
|
|
||||||
|
// insert a new string into dictionary, return its insertion order
|
||||||
|
size_t OldSortedStringDictionary::insert(const char * str, size_t len)
|
||||||
|
{
|
||||||
|
auto ret = dict.insert({DictEntry(str, len), dict.size()});
|
||||||
|
if (ret.second)
|
||||||
|
{
|
||||||
|
// make a copy to internal storage
|
||||||
|
data.push_back(std::vector<char>(len));
|
||||||
|
memcpy(data.back().data(), str, len);
|
||||||
|
// update dictionary entry to link pointer to internal storage
|
||||||
|
DictEntry * entry = const_cast<DictEntry *>(&(ret.first->first));
|
||||||
|
entry->data = data.back().data();
|
||||||
|
totalLength += len;
|
||||||
|
}
|
||||||
|
return ret.first->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reorder input index buffer from insertion order to dictionary order
|
||||||
|
*
|
||||||
|
* We require this function because string values are buffered by indexes
|
||||||
|
* in their insertion order. Until the entire dictionary is complete can
|
||||||
|
* we get their sorted indexes in the dictionary in that ORC specification
|
||||||
|
* demands dictionary should be ordered. Therefore this function transforms
|
||||||
|
* the indexes from insertion order to dictionary value order for final
|
||||||
|
* output.
|
||||||
|
*/
|
||||||
|
void OldSortedStringDictionary::reorder(std::vector<int64_t> & idxBuffer) const
|
||||||
|
{
|
||||||
|
// iterate the dictionary to get mapping from insertion order to value order
|
||||||
|
std::vector<size_t> mapping(dict.size());
|
||||||
|
size_t dictIdx = 0;
|
||||||
|
for (auto it = dict.cbegin(); it != dict.cend(); ++it)
|
||||||
|
{
|
||||||
|
mapping[it->second] = dictIdx++;
|
||||||
|
}
|
||||||
|
|
||||||
|
// do the transformation
|
||||||
|
for (size_t i = 0; i != idxBuffer.size(); ++i)
|
||||||
|
{
|
||||||
|
idxBuffer[i] = static_cast<int64_t>(mapping[static_cast<size_t>(idxBuffer[i])]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get dict entries in insertion order
|
||||||
|
void OldSortedStringDictionary::getEntriesInInsertionOrder(std::vector<const DictEntry *> & entries) const
|
||||||
|
{
|
||||||
|
entries.resize(dict.size());
|
||||||
|
for (auto it = dict.cbegin(); it != dict.cend(); ++it)
|
||||||
|
{
|
||||||
|
entries[it->second] = &(it->first);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// return count of entries
|
||||||
|
size_t OldSortedStringDictionary::size() const
|
||||||
|
{
|
||||||
|
return dict.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
// return total length of strings in the dictionary
|
||||||
|
uint64_t OldSortedStringDictionary::length() const
|
||||||
|
{
|
||||||
|
return totalLength;
|
||||||
|
}
|
||||||
|
|
||||||
|
void OldSortedStringDictionary::clear()
|
||||||
|
{
|
||||||
|
totalLength = 0;
|
||||||
|
data.clear();
|
||||||
|
dict.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Implementation of increasing sorted string dictionary
|
||||||
|
*/
|
||||||
|
class NewSortedStringDictionary
|
||||||
|
{
|
||||||
|
public:
|
||||||
|
struct DictEntry
|
||||||
|
{
|
||||||
|
DictEntry(const char * str, size_t len) : data(str), length(len) { }
|
||||||
|
const char * data;
|
||||||
|
size_t length;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct DictEntryWithIndex
|
||||||
|
{
|
||||||
|
DictEntryWithIndex(const char * str, size_t len, size_t index_) : entry(str, len), index(index_) { }
|
||||||
|
DictEntry entry;
|
||||||
|
size_t index;
|
||||||
|
};
|
||||||
|
|
||||||
|
NewSortedStringDictionary() : totalLength_(0) { }
|
||||||
|
|
||||||
|
// insert a new string into dictionary, return its insertion order
|
||||||
|
size_t insert(const char * str, size_t len);
|
||||||
|
|
||||||
|
// reorder input index buffer from insertion order to dictionary order
|
||||||
|
void reorder(std::vector<int64_t> & idxBuffer) const;
|
||||||
|
|
||||||
|
// get dict entries in insertion order
|
||||||
|
void getEntriesInInsertionOrder(std::vector<const DictEntry *> &) const;
|
||||||
|
|
||||||
|
// return count of entries
|
||||||
|
size_t size() const;
|
||||||
|
|
||||||
|
// return total length of strings in the dictionary
|
||||||
|
uint64_t length() const;
|
||||||
|
|
||||||
|
void clear();
|
||||||
|
|
||||||
|
// store indexes of insertion order in the dictionary for not-null rows
|
||||||
|
std::vector<int64_t> idxInDictBuffer;
|
||||||
|
|
||||||
|
private:
|
||||||
|
struct LessThan
|
||||||
|
{
|
||||||
|
bool operator()(const DictEntryWithIndex & l, const DictEntryWithIndex & r)
|
||||||
|
{
|
||||||
|
const auto & left = l.entry;
|
||||||
|
const auto & right = r.entry;
|
||||||
|
int ret = memcmp(left.data, right.data, std::min(left.length, right.length));
|
||||||
|
if (ret != 0)
|
||||||
|
{
|
||||||
|
return ret < 0;
|
||||||
|
}
|
||||||
|
return left.length < right.length;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
mutable std::vector<DictEntryWithIndex> flatDict_;
|
||||||
|
std::unordered_map<std::string, size_t> keyToIndex;
|
||||||
|
uint64_t totalLength_;
|
||||||
|
};
|
||||||
|
|
||||||
|
// insert a new string into dictionary, return its insertion order
|
||||||
|
size_t NewSortedStringDictionary::insert(const char * str, size_t len)
|
||||||
|
{
|
||||||
|
size_t index = flatDict_.size();
|
||||||
|
auto ret = keyToIndex.emplace(std::string(str, len), index);
|
||||||
|
if (ret.second)
|
||||||
|
{
|
||||||
|
flatDict_.emplace_back(ret.first->first.data(), ret.first->first.size(), index);
|
||||||
|
totalLength_ += len;
|
||||||
|
}
|
||||||
|
return ret.first->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Reorder input index buffer from insertion order to dictionary order
|
||||||
|
*
|
||||||
|
* We require this function because string values are buffered by indexes
|
||||||
|
* in their insertion order. Until the entire dictionary is complete can
|
||||||
|
* we get their sorted indexes in the dictionary in that ORC specification
|
||||||
|
* demands dictionary should be ordered. Therefore this function transforms
|
||||||
|
* the indexes from insertion order to dictionary value order for final
|
||||||
|
* output.
|
||||||
|
*/
|
||||||
|
void NewSortedStringDictionary::reorder(std::vector<int64_t> & idxBuffer) const
|
||||||
|
{
|
||||||
|
// iterate the dictionary to get mapping from insertion order to value order
|
||||||
|
std::vector<size_t> mapping(flatDict_.size());
|
||||||
|
for (size_t i = 0; i < flatDict_.size(); ++i)
|
||||||
|
{
|
||||||
|
mapping[flatDict_[i].index] = i;
|
||||||
|
}
|
||||||
|
|
||||||
|
// do the transformation
|
||||||
|
for (size_t i = 0; i != idxBuffer.size(); ++i)
|
||||||
|
{
|
||||||
|
idxBuffer[i] = static_cast<int64_t>(mapping[static_cast<size_t>(idxBuffer[i])]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// get dict entries in insertion order
|
||||||
|
void NewSortedStringDictionary::getEntriesInInsertionOrder(std::vector<const DictEntry *> & entries) const
|
||||||
|
{
|
||||||
|
std::sort(
|
||||||
|
flatDict_.begin(),
|
||||||
|
flatDict_.end(),
|
||||||
|
[](const DictEntryWithIndex & left, const DictEntryWithIndex & right) { return left.index < right.index; });
|
||||||
|
|
||||||
|
entries.resize(flatDict_.size());
|
||||||
|
for (size_t i = 0; i < flatDict_.size(); ++i)
|
||||||
|
{
|
||||||
|
entries[i] = &(flatDict_[i].entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// return count of entries
|
||||||
|
size_t NewSortedStringDictionary::size() const
|
||||||
|
{
|
||||||
|
return flatDict_.size();
|
||||||
|
}
|
||||||
|
|
||||||
|
// return total length of strings in the dictionary
|
||||||
|
uint64_t NewSortedStringDictionary::length() const
|
||||||
|
{
|
||||||
|
return totalLength_;
|
||||||
|
}
|
||||||
|
|
||||||
|
void NewSortedStringDictionary::clear()
|
||||||
|
{
|
||||||
|
totalLength_ = 0;
|
||||||
|
keyToIndex.clear();
|
||||||
|
flatDict_.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
template <size_t cardinality>
|
||||||
|
static std::vector<std::string> mockStrings()
|
||||||
|
{
|
||||||
|
std::vector<std::string> res(1000000);
|
||||||
|
for (auto & s : res)
|
||||||
|
{
|
||||||
|
s = "test string dictionary " + std::to_string(rand() % cardinality);
|
||||||
|
}
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename DictionaryImpl>
|
||||||
|
static NO_INLINE std::unique_ptr<DictionaryImpl> createAndWriteStringDictionary(const std::vector<std::string> & strs)
|
||||||
|
{
|
||||||
|
auto dict = std::make_unique<DictionaryImpl>();
|
||||||
|
for (const auto & str : strs)
|
||||||
|
{
|
||||||
|
auto index = dict->insert(str.data(), str.size());
|
||||||
|
dict->idxInDictBuffer.push_back(index);
|
||||||
|
}
|
||||||
|
dict->reorder(dict->idxInDictBuffer);
|
||||||
|
|
||||||
|
return dict;
|
||||||
|
}
|
||||||
|
|
||||||
|
template <typename DictionaryImpl, size_t cardinality>
|
||||||
|
static void BM_writeStringDictionary(benchmark::State & state)
|
||||||
|
{
|
||||||
|
auto strs = mockStrings<cardinality>();
|
||||||
|
for (auto _ : state)
|
||||||
|
{
|
||||||
|
auto dict = createAndWriteStringDictionary<DictionaryImpl>(strs);
|
||||||
|
benchmark::DoNotOptimize(dict);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 10);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 10);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 100);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 100);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 1000);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 1000);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 10000);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 10000);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, OldSortedStringDictionary, 100000);
|
||||||
|
BENCHMARK_TEMPLATE(BM_writeStringDictionary, NewSortedStringDictionary, 100000);
|
||||||
|
|
@ -45,6 +45,7 @@ uint64_t ACLMap::convertACLs(const Coordination::ACLs & acls)
|
|||||||
if (acls.empty())
|
if (acls.empty())
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
|
std::lock_guard lock(map_mutex);
|
||||||
if (acl_to_num.contains(acls))
|
if (acl_to_num.contains(acls))
|
||||||
return acl_to_num[acls];
|
return acl_to_num[acls];
|
||||||
|
|
||||||
@ -62,6 +63,7 @@ Coordination::ACLs ACLMap::convertNumber(uint64_t acls_id) const
|
|||||||
if (acls_id == 0)
|
if (acls_id == 0)
|
||||||
return Coordination::ACLs{};
|
return Coordination::ACLs{};
|
||||||
|
|
||||||
|
std::lock_guard lock(map_mutex);
|
||||||
if (!num_to_acl.contains(acls_id))
|
if (!num_to_acl.contains(acls_id))
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown ACL id {}. It's a bug", acls_id);
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown ACL id {}. It's a bug", acls_id);
|
||||||
|
|
||||||
@ -70,6 +72,7 @@ Coordination::ACLs ACLMap::convertNumber(uint64_t acls_id) const
|
|||||||
|
|
||||||
void ACLMap::addMapping(uint64_t acls_id, const Coordination::ACLs & acls)
|
void ACLMap::addMapping(uint64_t acls_id, const Coordination::ACLs & acls)
|
||||||
{
|
{
|
||||||
|
std::lock_guard lock(map_mutex);
|
||||||
num_to_acl[acls_id] = acls;
|
num_to_acl[acls_id] = acls;
|
||||||
acl_to_num[acls] = acls_id;
|
acl_to_num[acls] = acls_id;
|
||||||
max_acl_id = std::max(acls_id + 1, max_acl_id); /// max_acl_id pointer next slot
|
max_acl_id = std::max(acls_id + 1, max_acl_id); /// max_acl_id pointer next slot
|
||||||
@ -77,11 +80,13 @@ void ACLMap::addMapping(uint64_t acls_id, const Coordination::ACLs & acls)
|
|||||||
|
|
||||||
void ACLMap::addUsage(uint64_t acl_id)
|
void ACLMap::addUsage(uint64_t acl_id)
|
||||||
{
|
{
|
||||||
|
std::lock_guard lock(map_mutex);
|
||||||
usage_counter[acl_id]++;
|
usage_counter[acl_id]++;
|
||||||
}
|
}
|
||||||
|
|
||||||
void ACLMap::removeUsage(uint64_t acl_id)
|
void ACLMap::removeUsage(uint64_t acl_id)
|
||||||
{
|
{
|
||||||
|
std::lock_guard lock(map_mutex);
|
||||||
if (!usage_counter.contains(acl_id))
|
if (!usage_counter.contains(acl_id))
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
@ -32,6 +32,8 @@ private:
|
|||||||
NumToACLMap num_to_acl;
|
NumToACLMap num_to_acl;
|
||||||
UsageCounter usage_counter;
|
UsageCounter usage_counter;
|
||||||
uint64_t max_acl_id{1};
|
uint64_t max_acl_id{1};
|
||||||
|
|
||||||
|
mutable std::mutex map_mutex;
|
||||||
public:
|
public:
|
||||||
|
|
||||||
/// Convert ACL to number. If it's new ACL than adds it to map
|
/// Convert ACL to number. If it's new ACL than adds it to map
|
||||||
|
@ -301,11 +301,13 @@ String MonitorCommand::run()
|
|||||||
|
|
||||||
print(ret, "server_state", keeper_info.getRole());
|
print(ret, "server_state", keeper_info.getRole());
|
||||||
|
|
||||||
print(ret, "znode_count", state_machine.getNodesCount());
|
const auto & storage_stats = state_machine.getStorageStats();
|
||||||
print(ret, "watch_count", state_machine.getTotalWatchesCount());
|
|
||||||
print(ret, "ephemerals_count", state_machine.getTotalEphemeralNodesCount());
|
print(ret, "znode_count", storage_stats.nodes_count.load(std::memory_order_relaxed));
|
||||||
print(ret, "approximate_data_size", state_machine.getApproximateDataSize());
|
print(ret, "watch_count", storage_stats.total_watches_count.load(std::memory_order_relaxed));
|
||||||
print(ret, "key_arena_size", state_machine.getKeyArenaSize());
|
print(ret, "ephemerals_count", storage_stats.total_emphemeral_nodes_count.load(std::memory_order_relaxed));
|
||||||
|
print(ret, "approximate_data_size", storage_stats.approximate_data_size.load(std::memory_order_relaxed));
|
||||||
|
print(ret, "key_arena_size", 0);
|
||||||
print(ret, "latest_snapshot_size", state_machine.getLatestSnapshotSize());
|
print(ret, "latest_snapshot_size", state_machine.getLatestSnapshotSize());
|
||||||
|
|
||||||
#if defined(OS_LINUX) || defined(OS_DARWIN)
|
#if defined(OS_LINUX) || defined(OS_DARWIN)
|
||||||
@ -387,6 +389,7 @@ String ServerStatCommand::run()
|
|||||||
|
|
||||||
auto & stats = keeper_dispatcher.getKeeperConnectionStats();
|
auto & stats = keeper_dispatcher.getKeeperConnectionStats();
|
||||||
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
|
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
|
||||||
|
const auto & storage_stats = keeper_dispatcher.getStateMachine().getStorageStats();
|
||||||
|
|
||||||
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
|
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
|
||||||
|
|
||||||
@ -398,9 +401,9 @@ String ServerStatCommand::run()
|
|||||||
write("Sent", toString(stats.getPacketsSent()));
|
write("Sent", toString(stats.getPacketsSent()));
|
||||||
write("Connections", toString(keeper_info.alive_connections_count));
|
write("Connections", toString(keeper_info.alive_connections_count));
|
||||||
write("Outstanding", toString(keeper_info.outstanding_requests_count));
|
write("Outstanding", toString(keeper_info.outstanding_requests_count));
|
||||||
write("Zxid", formatZxid(keeper_info.last_zxid));
|
write("Zxid", formatZxid(storage_stats.last_zxid.load(std::memory_order_relaxed)));
|
||||||
write("Mode", keeper_info.getRole());
|
write("Mode", keeper_info.getRole());
|
||||||
write("Node count", toString(keeper_info.total_nodes_count));
|
write("Node count", toString(storage_stats.nodes_count.load(std::memory_order_relaxed)));
|
||||||
|
|
||||||
return buf.str();
|
return buf.str();
|
||||||
}
|
}
|
||||||
@ -416,6 +419,7 @@ String StatCommand::run()
|
|||||||
|
|
||||||
auto & stats = keeper_dispatcher.getKeeperConnectionStats();
|
auto & stats = keeper_dispatcher.getKeeperConnectionStats();
|
||||||
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
|
Keeper4LWInfo keeper_info = keeper_dispatcher.getKeeper4LWInfo();
|
||||||
|
const auto & storage_stats = keeper_dispatcher.getStateMachine().getStorageStats();
|
||||||
|
|
||||||
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
|
write("ClickHouse Keeper version", String(VERSION_DESCRIBE) + "-" + VERSION_GITHASH);
|
||||||
|
|
||||||
@ -431,9 +435,9 @@ String StatCommand::run()
|
|||||||
write("Sent", toString(stats.getPacketsSent()));
|
write("Sent", toString(stats.getPacketsSent()));
|
||||||
write("Connections", toString(keeper_info.alive_connections_count));
|
write("Connections", toString(keeper_info.alive_connections_count));
|
||||||
write("Outstanding", toString(keeper_info.outstanding_requests_count));
|
write("Outstanding", toString(keeper_info.outstanding_requests_count));
|
||||||
write("Zxid", formatZxid(keeper_info.last_zxid));
|
write("Zxid", formatZxid(storage_stats.last_zxid.load(std::memory_order_relaxed)));
|
||||||
write("Mode", keeper_info.getRole());
|
write("Mode", keeper_info.getRole());
|
||||||
write("Node count", toString(keeper_info.total_nodes_count));
|
write("Node count", toString(storage_stats.nodes_count.load(std::memory_order_relaxed)));
|
||||||
|
|
||||||
return buf.str();
|
return buf.str();
|
||||||
}
|
}
|
||||||
|
@ -1,7 +1,5 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <string>
|
|
||||||
|
|
||||||
#include <base/types.h>
|
#include <base/types.h>
|
||||||
#include <Common/Exception.h>
|
#include <Common/Exception.h>
|
||||||
|
|
||||||
@ -30,9 +28,6 @@ struct Keeper4LWInfo
|
|||||||
uint64_t follower_count;
|
uint64_t follower_count;
|
||||||
uint64_t synced_follower_count;
|
uint64_t synced_follower_count;
|
||||||
|
|
||||||
uint64_t total_nodes_count;
|
|
||||||
int64_t last_zxid;
|
|
||||||
|
|
||||||
String getRole() const
|
String getRole() const
|
||||||
{
|
{
|
||||||
if (is_standalone)
|
if (is_standalone)
|
||||||
|
@ -38,15 +38,16 @@ void updateKeeperInformation(KeeperDispatcher & keeper_dispatcher, AsynchronousM
|
|||||||
is_follower = static_cast<size_t>(keeper_info.is_follower);
|
is_follower = static_cast<size_t>(keeper_info.is_follower);
|
||||||
is_exceeding_mem_soft_limit = static_cast<size_t>(keeper_info.is_exceeding_mem_soft_limit);
|
is_exceeding_mem_soft_limit = static_cast<size_t>(keeper_info.is_exceeding_mem_soft_limit);
|
||||||
|
|
||||||
zxid = keeper_info.last_zxid;
|
|
||||||
const auto & state_machine = keeper_dispatcher.getStateMachine();
|
const auto & state_machine = keeper_dispatcher.getStateMachine();
|
||||||
znode_count = state_machine.getNodesCount();
|
const auto & storage_stats = state_machine.getStorageStats();
|
||||||
watch_count = state_machine.getTotalWatchesCount();
|
zxid = storage_stats.last_zxid.load(std::memory_order_relaxed);
|
||||||
ephemerals_count = state_machine.getTotalEphemeralNodesCount();
|
znode_count = storage_stats.nodes_count.load(std::memory_order_relaxed);
|
||||||
approximate_data_size = state_machine.getApproximateDataSize();
|
watch_count = storage_stats.total_watches_count.load(std::memory_order_relaxed);
|
||||||
key_arena_size = state_machine.getKeyArenaSize();
|
ephemerals_count = storage_stats.total_emphemeral_nodes_count.load(std::memory_order_relaxed);
|
||||||
session_with_watches = state_machine.getSessionsWithWatchesCount();
|
approximate_data_size = storage_stats.approximate_data_size.load(std::memory_order_relaxed);
|
||||||
paths_watched = state_machine.getWatchedPathsCount();
|
key_arena_size = 0;
|
||||||
|
session_with_watches = storage_stats.sessions_with_watches_count.load(std::memory_order_relaxed);
|
||||||
|
paths_watched = storage_stats.watched_paths_count.load(std::memory_order_relaxed);
|
||||||
|
|
||||||
# if defined(__linux__) || defined(__APPLE__)
|
# if defined(__linux__) || defined(__APPLE__)
|
||||||
open_file_descriptor_count = getCurrentProcessFDCount();
|
open_file_descriptor_count = getCurrentProcessFDCount();
|
||||||
|
@ -305,7 +305,7 @@ void KeeperDispatcher::requestThread()
|
|||||||
if (has_read_request)
|
if (has_read_request)
|
||||||
{
|
{
|
||||||
if (server->isLeaderAlive())
|
if (server->isLeaderAlive())
|
||||||
server->putLocalReadRequest(request);
|
server->putLocalReadRequest({request});
|
||||||
else
|
else
|
||||||
addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS);
|
addErrorResponses({request}, Coordination::Error::ZCONNECTIONLOSS);
|
||||||
}
|
}
|
||||||
|
@ -28,6 +28,16 @@
|
|||||||
#include <Common/getMultipleKeysFromConfig.h>
|
#include <Common/getMultipleKeysFromConfig.h>
|
||||||
#include <Common/getNumberOfPhysicalCPUCores.h>
|
#include <Common/getNumberOfPhysicalCPUCores.h>
|
||||||
|
|
||||||
|
#if USE_SSL
|
||||||
|
# include <Server/CertificateReloader.h>
|
||||||
|
# include <openssl/ssl.h>
|
||||||
|
# include <Poco/Crypto/EVPPKey.h>
|
||||||
|
# include <Poco/Net/Context.h>
|
||||||
|
# include <Poco/Net/SSLManager.h>
|
||||||
|
# include <Poco/Net/Utility.h>
|
||||||
|
# include <Poco/StringTokenizer.h>
|
||||||
|
#endif
|
||||||
|
|
||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <mutex>
|
#include <mutex>
|
||||||
#include <string>
|
#include <string>
|
||||||
@ -48,6 +58,7 @@ namespace ErrorCodes
|
|||||||
extern const int SUPPORT_IS_DISABLED;
|
extern const int SUPPORT_IS_DISABLED;
|
||||||
extern const int LOGICAL_ERROR;
|
extern const int LOGICAL_ERROR;
|
||||||
extern const int INVALID_CONFIG_PARAMETER;
|
extern const int INVALID_CONFIG_PARAMETER;
|
||||||
|
extern const int BAD_ARGUMENTS;
|
||||||
}
|
}
|
||||||
|
|
||||||
using namespace std::chrono_literals;
|
using namespace std::chrono_literals;
|
||||||
@ -56,6 +67,16 @@ namespace
|
|||||||
{
|
{
|
||||||
|
|
||||||
#if USE_SSL
|
#if USE_SSL
|
||||||
|
|
||||||
|
int callSetCertificate(SSL * ssl, void * arg)
|
||||||
|
{
|
||||||
|
if (!arg)
|
||||||
|
return -1;
|
||||||
|
|
||||||
|
const CertificateReloader::Data * data = reinterpret_cast<CertificateReloader::Data *>(arg);
|
||||||
|
return setCertificateCallback(ssl, data, getLogger("SSLContext"));
|
||||||
|
}
|
||||||
|
|
||||||
void setSSLParams(nuraft::asio_service::options & asio_opts)
|
void setSSLParams(nuraft::asio_service::options & asio_opts)
|
||||||
{
|
{
|
||||||
const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
|
const Poco::Util::LayeredConfiguration & config = Poco::Util::Application::instance().config();
|
||||||
@ -69,18 +90,55 @@ void setSSLParams(nuraft::asio_service::options & asio_opts)
|
|||||||
if (!config.has(private_key_file_property))
|
if (!config.has(private_key_file_property))
|
||||||
throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set.");
|
throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Server private key file is not set.");
|
||||||
|
|
||||||
asio_opts.enable_ssl_ = true;
|
Poco::Net::Context::Params params;
|
||||||
asio_opts.server_cert_file_ = config.getString(certificate_file_property);
|
params.certificateFile = config.getString(certificate_file_property);
|
||||||
asio_opts.server_key_file_ = config.getString(private_key_file_property);
|
if (params.certificateFile.empty())
|
||||||
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server certificate file in config '{}' is empty", certificate_file_property);
|
||||||
|
|
||||||
|
params.privateKeyFile = config.getString(private_key_file_property);
|
||||||
|
if (params.privateKeyFile.empty())
|
||||||
|
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Server key file in config '{}' is empty", private_key_file_property);
|
||||||
|
|
||||||
|
auto pass_phrase = config.getString("openSSL.server.privateKeyPassphraseHandler.options.password", "");
|
||||||
|
auto certificate_data = std::make_shared<CertificateReloader::Data>(params.certificateFile, params.privateKeyFile, pass_phrase);
|
||||||
|
|
||||||
if (config.has(root_ca_file_property))
|
if (config.has(root_ca_file_property))
|
||||||
asio_opts.root_cert_file_ = config.getString(root_ca_file_property);
|
params.caLocation = config.getString(root_ca_file_property);
|
||||||
|
|
||||||
if (config.getBool("openSSL.server.loadDefaultCAFile", false))
|
params.loadDefaultCAs = config.getBool("openSSL.server.loadDefaultCAFile", false);
|
||||||
asio_opts.load_default_ca_file_ = true;
|
params.verificationMode = Poco::Net::Utility::convertVerificationMode(config.getString("openSSL.server.verificationMode", "none"));
|
||||||
|
|
||||||
if (config.getString("openSSL.server.verificationMode", "none") == "none")
|
std::string disabled_protocols_list = config.getString("openSSL.server.disableProtocols", "");
|
||||||
asio_opts.skip_verification_ = true;
|
Poco::StringTokenizer dp_tok(disabled_protocols_list, ";,", Poco::StringTokenizer::TOK_TRIM | Poco::StringTokenizer::TOK_IGNORE_EMPTY);
|
||||||
|
int disabled_protocols = 0;
|
||||||
|
for (const auto & token : dp_tok)
|
||||||
|
{
|
||||||
|
if (token == "sslv2")
|
||||||
|
disabled_protocols |= Poco::Net::Context::PROTO_SSLV2;
|
||||||
|
else if (token == "sslv3")
|
||||||
|
disabled_protocols |= Poco::Net::Context::PROTO_SSLV3;
|
||||||
|
else if (token == "tlsv1")
|
||||||
|
disabled_protocols |= Poco::Net::Context::PROTO_TLSV1;
|
||||||
|
else if (token == "tlsv1_1")
|
||||||
|
disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_1;
|
||||||
|
else if (token == "tlsv1_2")
|
||||||
|
disabled_protocols |= Poco::Net::Context::PROTO_TLSV1_2;
|
||||||
|
}
|
||||||
|
|
||||||
|
asio_opts.ssl_context_provider_server_ = [params, certificate_data, disabled_protocols]
|
||||||
|
{
|
||||||
|
Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_SERVER_USE, params);
|
||||||
|
context.disableProtocols(disabled_protocols);
|
||||||
|
SSL_CTX * ssl_ctx = context.takeSslContext();
|
||||||
|
SSL_CTX_set_cert_cb(ssl_ctx, callSetCertificate, reinterpret_cast<void *>(certificate_data.get()));
|
||||||
|
return ssl_ctx;
|
||||||
|
};
|
||||||
|
|
||||||
|
asio_opts.ssl_context_provider_client_ = [ctx_params = std::move(params)]
|
||||||
|
{
|
||||||
|
Poco::Net::Context context(Poco::Net::Context::Usage::TLSV1_2_CLIENT_USE, ctx_params);
|
||||||
|
return context.takeSslContext();
|
||||||
|
};
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
@ -1149,8 +1207,6 @@ Keeper4LWInfo KeeperServer::getPartiallyFilled4LWInfo() const
|
|||||||
result.synced_follower_count = getSyncedFollowerCount();
|
result.synced_follower_count = getSyncedFollowerCount();
|
||||||
}
|
}
|
||||||
result.is_exceeding_mem_soft_limit = isExceedingMemorySoftLimit();
|
result.is_exceeding_mem_soft_limit = isExceedingMemorySoftLimit();
|
||||||
result.total_nodes_count = getKeeperStateMachine()->getNodesCount();
|
|
||||||
result.last_zxid = getKeeperStateMachine()->getLastProcessedZxid();
|
|
||||||
return result;
|
return result;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -78,20 +78,20 @@ namespace
|
|||||||
writeBinary(false, out);
|
writeBinary(false, out);
|
||||||
|
|
||||||
/// Serialize stat
|
/// Serialize stat
|
||||||
writeBinary(node.czxid, out);
|
writeBinary(node.stats.czxid, out);
|
||||||
writeBinary(node.mzxid, out);
|
writeBinary(node.stats.mzxid, out);
|
||||||
writeBinary(node.ctime(), out);
|
writeBinary(node.stats.ctime(), out);
|
||||||
writeBinary(node.mtime, out);
|
writeBinary(node.stats.mtime, out);
|
||||||
writeBinary(node.version, out);
|
writeBinary(node.stats.version, out);
|
||||||
writeBinary(node.cversion, out);
|
writeBinary(node.stats.cversion, out);
|
||||||
writeBinary(node.aversion, out);
|
writeBinary(node.stats.aversion, out);
|
||||||
writeBinary(node.ephemeralOwner(), out);
|
writeBinary(node.stats.ephemeralOwner(), out);
|
||||||
if (version < SnapshotVersion::V6)
|
if (version < SnapshotVersion::V6)
|
||||||
writeBinary(static_cast<int32_t>(node.getData().size()), out);
|
writeBinary(static_cast<int32_t>(node.stats.data_size), out);
|
||||||
writeBinary(node.numChildren(), out);
|
writeBinary(node.stats.numChildren(), out);
|
||||||
writeBinary(node.pzxid, out);
|
writeBinary(node.stats.pzxid, out);
|
||||||
|
|
||||||
writeBinary(node.seqNum(), out);
|
writeBinary(node.stats.seqNum(), out);
|
||||||
|
|
||||||
if (version >= SnapshotVersion::V4 && version <= SnapshotVersion::V5)
|
if (version >= SnapshotVersion::V4 && version <= SnapshotVersion::V5)
|
||||||
writeBinary(node.sizeInBytes(), out);
|
writeBinary(node.sizeInBytes(), out);
|
||||||
@ -100,11 +100,11 @@ namespace
|
|||||||
template<typename Node>
|
template<typename Node>
|
||||||
void readNode(Node & node, ReadBuffer & in, SnapshotVersion version, ACLMap & acl_map)
|
void readNode(Node & node, ReadBuffer & in, SnapshotVersion version, ACLMap & acl_map)
|
||||||
{
|
{
|
||||||
readVarUInt(node.data_size, in);
|
readVarUInt(node.stats.data_size, in);
|
||||||
if (node.data_size != 0)
|
if (node.stats.data_size != 0)
|
||||||
{
|
{
|
||||||
node.data = std::unique_ptr<char[]>(new char[node.data_size]);
|
node.data = std::unique_ptr<char[]>(new char[node.stats.data_size]);
|
||||||
in.readStrict(node.data.get(), node.data_size);
|
in.readStrict(node.data.get(), node.stats.data_size);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (version >= SnapshotVersion::V1)
|
if (version >= SnapshotVersion::V1)
|
||||||
@ -141,19 +141,19 @@ namespace
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Deserialize stat
|
/// Deserialize stat
|
||||||
readBinary(node.czxid, in);
|
readBinary(node.stats.czxid, in);
|
||||||
readBinary(node.mzxid, in);
|
readBinary(node.stats.mzxid, in);
|
||||||
int64_t ctime;
|
int64_t ctime;
|
||||||
readBinary(ctime, in);
|
readBinary(ctime, in);
|
||||||
node.setCtime(ctime);
|
node.stats.setCtime(ctime);
|
||||||
readBinary(node.mtime, in);
|
readBinary(node.stats.mtime, in);
|
||||||
readBinary(node.version, in);
|
readBinary(node.stats.version, in);
|
||||||
readBinary(node.cversion, in);
|
readBinary(node.stats.cversion, in);
|
||||||
readBinary(node.aversion, in);
|
readBinary(node.stats.aversion, in);
|
||||||
int64_t ephemeral_owner = 0;
|
int64_t ephemeral_owner = 0;
|
||||||
readBinary(ephemeral_owner, in);
|
readBinary(ephemeral_owner, in);
|
||||||
if (ephemeral_owner != 0)
|
if (ephemeral_owner != 0)
|
||||||
node.setEphemeralOwner(ephemeral_owner);
|
node.stats.setEphemeralOwner(ephemeral_owner);
|
||||||
|
|
||||||
if (version < SnapshotVersion::V6)
|
if (version < SnapshotVersion::V6)
|
||||||
{
|
{
|
||||||
@ -163,14 +163,14 @@ namespace
|
|||||||
int32_t num_children = 0;
|
int32_t num_children = 0;
|
||||||
readBinary(num_children, in);
|
readBinary(num_children, in);
|
||||||
if (ephemeral_owner == 0)
|
if (ephemeral_owner == 0)
|
||||||
node.setNumChildren(num_children);
|
node.stats.setNumChildren(num_children);
|
||||||
|
|
||||||
readBinary(node.pzxid, in);
|
readBinary(node.stats.pzxid, in);
|
||||||
|
|
||||||
int32_t seq_num = 0;
|
int32_t seq_num = 0;
|
||||||
readBinary(seq_num, in);
|
readBinary(seq_num, in);
|
||||||
if (ephemeral_owner == 0)
|
if (ephemeral_owner == 0)
|
||||||
node.setSeqNum(seq_num);
|
node.stats.setSeqNum(seq_num);
|
||||||
|
|
||||||
if (version >= SnapshotVersion::V4 && version <= SnapshotVersion::V5)
|
if (version >= SnapshotVersion::V4 && version <= SnapshotVersion::V5)
|
||||||
{
|
{
|
||||||
@ -256,7 +256,7 @@ void KeeperStorageSnapshot<Storage>::serialize(const KeeperStorageSnapshot<Stora
|
|||||||
/// Benign race condition possible while taking snapshot: NuRaft decide to create snapshot at some log id
|
/// Benign race condition possible while taking snapshot: NuRaft decide to create snapshot at some log id
|
||||||
/// and only after some time we lock storage and enable snapshot mode. So snapshot_container_size can be
|
/// and only after some time we lock storage and enable snapshot mode. So snapshot_container_size can be
|
||||||
/// slightly bigger than required.
|
/// slightly bigger than required.
|
||||||
if (node.mzxid > snapshot.zxid)
|
if (node.stats.mzxid > snapshot.zxid)
|
||||||
break;
|
break;
|
||||||
writeBinary(path, out);
|
writeBinary(path, out);
|
||||||
writeNode(node, snapshot.version, out);
|
writeNode(node, snapshot.version, out);
|
||||||
@ -306,7 +306,7 @@ void KeeperStorageSnapshot<Storage>::serialize(const KeeperStorageSnapshot<Stora
|
|||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<Storage> & deserialization_result, ReadBuffer & in, KeeperContextPtr keeper_context)
|
void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<Storage> & deserialization_result, ReadBuffer & in, KeeperContextPtr keeper_context) TSA_NO_THREAD_SAFETY_ANALYSIS
|
||||||
{
|
{
|
||||||
uint8_t version;
|
uint8_t version;
|
||||||
readBinary(version, in);
|
readBinary(version, in);
|
||||||
@ -435,13 +435,13 @@ void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<S
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
auto ephemeral_owner = node.ephemeralOwner();
|
auto ephemeral_owner = node.stats.ephemeralOwner();
|
||||||
if constexpr (!use_rocksdb)
|
if constexpr (!use_rocksdb)
|
||||||
if (!node.isEphemeral() && node.numChildren() > 0)
|
if (!node.stats.isEphemeral() && node.stats.numChildren() > 0)
|
||||||
node.getChildren().reserve(node.numChildren());
|
node.getChildren().reserve(node.stats.numChildren());
|
||||||
|
|
||||||
if (ephemeral_owner != 0)
|
if (ephemeral_owner != 0)
|
||||||
storage.ephemerals[node.ephemeralOwner()].insert(std::string{path});
|
storage.committed_ephemerals[node.stats.ephemeralOwner()].insert(std::string{path});
|
||||||
|
|
||||||
if (recalculate_digest)
|
if (recalculate_digest)
|
||||||
storage.nodes_digest += node.getDigest(path);
|
storage.nodes_digest += node.getDigest(path);
|
||||||
@ -467,16 +467,25 @@ void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<S
|
|||||||
{
|
{
|
||||||
if (itr.key != "/")
|
if (itr.key != "/")
|
||||||
{
|
{
|
||||||
if (itr.value.numChildren() != static_cast<int32_t>(itr.value.getChildren().size()))
|
if (itr.value.stats.numChildren() != static_cast<int32_t>(itr.value.getChildren().size()))
|
||||||
{
|
{
|
||||||
#ifdef NDEBUG
|
#ifdef NDEBUG
|
||||||
/// TODO (alesapin) remove this, it should be always CORRUPTED_DATA.
|
/// TODO (alesapin) remove this, it should be always CORRUPTED_DATA.
|
||||||
LOG_ERROR(getLogger("KeeperSnapshotManager"), "Children counter in stat.numChildren {}"
|
LOG_ERROR(
|
||||||
" is different from actual children size {} for node {}", itr.value.numChildren(), itr.value.getChildren().size(), itr.key);
|
getLogger("KeeperSnapshotManager"),
|
||||||
#else
|
"Children counter in stat.numChildren {}"
|
||||||
throw Exception(ErrorCodes::LOGICAL_ERROR, "Children counter in stat.numChildren {}"
|
|
||||||
" is different from actual children size {} for node {}",
|
" is different from actual children size {} for node {}",
|
||||||
itr.value.numChildren(), itr.value.getChildren().size(), itr.key);
|
itr.value.stats.numChildren(),
|
||||||
|
itr.value.getChildren().size(),
|
||||||
|
itr.key);
|
||||||
|
#else
|
||||||
|
throw Exception(
|
||||||
|
ErrorCodes::LOGICAL_ERROR,
|
||||||
|
"Children counter in stat.numChildren {}"
|
||||||
|
" is different from actual children size {} for node {}",
|
||||||
|
itr.value.stats.numChildren(),
|
||||||
|
itr.value.getChildren().size(),
|
||||||
|
itr.key);
|
||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -511,7 +520,7 @@ void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<S
|
|||||||
session_auth_counter++;
|
session_auth_counter++;
|
||||||
}
|
}
|
||||||
if (!ids.empty())
|
if (!ids.empty())
|
||||||
storage.session_and_auth[active_session_id] = ids;
|
storage.committed_session_and_auth[active_session_id] = ids;
|
||||||
}
|
}
|
||||||
current_session_size++;
|
current_session_size++;
|
||||||
}
|
}
|
||||||
@ -527,6 +536,8 @@ void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<S
|
|||||||
buffer->pos(0);
|
buffer->pos(0);
|
||||||
deserialization_result.cluster_config = ClusterConfig::deserialize(*buffer);
|
deserialization_result.cluster_config = ClusterConfig::deserialize(*buffer);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
storage.updateStats();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
@ -544,7 +555,7 @@ KeeperStorageSnapshot<Storage>::KeeperStorageSnapshot(Storage * storage_, uint64
|
|||||||
begin = storage->getSnapshotIteratorBegin();
|
begin = storage->getSnapshotIteratorBegin();
|
||||||
session_and_timeout = storage->getActiveSessions();
|
session_and_timeout = storage->getActiveSessions();
|
||||||
acl_map = storage->acl_map.getMapping();
|
acl_map = storage->acl_map.getMapping();
|
||||||
session_and_auth = storage->session_and_auth;
|
session_and_auth = storage->committed_session_and_auth;
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
@ -563,7 +574,7 @@ KeeperStorageSnapshot<Storage>::KeeperStorageSnapshot(
|
|||||||
begin = storage->getSnapshotIteratorBegin();
|
begin = storage->getSnapshotIteratorBegin();
|
||||||
session_and_timeout = storage->getActiveSessions();
|
session_and_timeout = storage->getActiveSessions();
|
||||||
acl_map = storage->acl_map.getMapping();
|
acl_map = storage->acl_map.getMapping();
|
||||||
session_and_auth = storage->session_and_auth;
|
session_and_auth = storage->committed_session_and_auth;
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
|
@ -36,6 +36,11 @@ namespace ProfileEvents
|
|||||||
extern const Event KeeperStorageLockWaitMicroseconds;
|
extern const Event KeeperStorageLockWaitMicroseconds;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
namespace CurrentMetrics
|
||||||
|
{
|
||||||
|
extern const Metric KeeperAliveConnections;
|
||||||
|
}
|
||||||
|
|
||||||
namespace DB
|
namespace DB
|
||||||
{
|
{
|
||||||
|
|
||||||
@ -56,6 +61,7 @@ IKeeperStateMachine::IKeeperStateMachine(
|
|||||||
, snapshots_queue(snapshots_queue_)
|
, snapshots_queue(snapshots_queue_)
|
||||||
, min_request_size_to_cache(keeper_context_->getCoordinationSettings()->min_request_size_for_cache)
|
, min_request_size_to_cache(keeper_context_->getCoordinationSettings()->min_request_size_for_cache)
|
||||||
, log(getLogger("KeeperStateMachine"))
|
, log(getLogger("KeeperStateMachine"))
|
||||||
|
, read_pool(CurrentMetrics::KeeperAliveConnections, CurrentMetrics::KeeperAliveConnections, CurrentMetrics::KeeperAliveConnections, 100, 10000, 10000)
|
||||||
, superdigest(superdigest_)
|
, superdigest(superdigest_)
|
||||||
, keeper_context(keeper_context_)
|
, keeper_context(keeper_context_)
|
||||||
, snapshot_manager_s3(snapshot_manager_s3_)
|
, snapshot_manager_s3(snapshot_manager_s3_)
|
||||||
@ -175,18 +181,20 @@ void assertDigest(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct TSA_SCOPED_LOCKABLE LockGuardWithStats final
|
template <bool shared = false>
|
||||||
|
struct LockGuardWithStats final
|
||||||
{
|
{
|
||||||
std::unique_lock<std::mutex> lock;
|
using LockType = std::conditional_t<shared, std::shared_lock<SharedMutex>, std::unique_lock<SharedMutex>>;
|
||||||
explicit LockGuardWithStats(std::mutex & mutex) TSA_ACQUIRE(mutex)
|
LockType lock;
|
||||||
|
explicit LockGuardWithStats(SharedMutex & mutex)
|
||||||
{
|
{
|
||||||
Stopwatch watch;
|
Stopwatch watch;
|
||||||
std::unique_lock l(mutex);
|
LockType l(mutex);
|
||||||
ProfileEvents::increment(ProfileEvents::KeeperStorageLockWaitMicroseconds, watch.elapsedMicroseconds());
|
ProfileEvents::increment(ProfileEvents::KeeperStorageLockWaitMicroseconds, watch.elapsedMicroseconds());
|
||||||
lock = std::move(l);
|
lock = std::move(l);
|
||||||
}
|
}
|
||||||
|
|
||||||
~LockGuardWithStats() TSA_RELEASE() = default;
|
~LockGuardWithStats() = default;
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
@ -312,13 +320,12 @@ bool KeeperStateMachine<Storage>::preprocess(const KeeperStorageBase::RequestFor
|
|||||||
if (op_num == Coordination::OpNum::SessionID || op_num == Coordination::OpNum::Reconfig)
|
if (op_num == Coordination::OpNum::SessionID || op_num == Coordination::OpNum::Reconfig)
|
||||||
return true;
|
return true;
|
||||||
|
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
|
||||||
|
|
||||||
if (storage->isFinalized())
|
if (storage->isFinalized())
|
||||||
return false;
|
return false;
|
||||||
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
|
LockGuardWithStats<true> lock(storage_mutex);
|
||||||
storage->preprocessRequest(
|
storage->preprocessRequest(
|
||||||
request_for_session.request,
|
request_for_session.request,
|
||||||
request_for_session.session_id,
|
request_for_session.session_id,
|
||||||
@ -335,7 +342,12 @@ bool KeeperStateMachine<Storage>::preprocess(const KeeperStorageBase::RequestFor
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (keeper_context->digestEnabled() && request_for_session.digest)
|
if (keeper_context->digestEnabled() && request_for_session.digest)
|
||||||
assertDigest(*request_for_session.digest, storage->getNodesDigest(false), *request_for_session.request, request_for_session.log_idx, false);
|
assertDigest(
|
||||||
|
*request_for_session.digest,
|
||||||
|
storage->getNodesDigest(false, /*lock_transaction_mutex=*/true),
|
||||||
|
*request_for_session.request,
|
||||||
|
request_for_session.log_idx,
|
||||||
|
false);
|
||||||
|
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
@ -343,7 +355,7 @@ bool KeeperStateMachine<Storage>::preprocess(const KeeperStorageBase::RequestFor
|
|||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void KeeperStateMachine<Storage>::reconfigure(const KeeperStorageBase::RequestForSession& request_for_session)
|
void KeeperStateMachine<Storage>::reconfigure(const KeeperStorageBase::RequestForSession& request_for_session)
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
KeeperStorageBase::ResponseForSession response = processReconfiguration(request_for_session);
|
KeeperStorageBase::ResponseForSession response = processReconfiguration(request_for_session);
|
||||||
if (!responses_queue.push(response))
|
if (!responses_queue.push(response))
|
||||||
{
|
{
|
||||||
@ -461,7 +473,7 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine<Storage>::commit(const uint64_t l
|
|||||||
response_for_session.response = response;
|
response_for_session.response = response;
|
||||||
response_for_session.request = request_for_session->request;
|
response_for_session.request = request_for_session->request;
|
||||||
|
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
session_id = storage->getSessionID(session_id_request.session_timeout_ms);
|
session_id = storage->getSessionID(session_id_request.session_timeout_ms);
|
||||||
LOG_DEBUG(log, "Session ID response {} with timeout {}", session_id, session_id_request.session_timeout_ms);
|
LOG_DEBUG(log, "Session ID response {} with timeout {}", session_id, session_id_request.session_timeout_ms);
|
||||||
response->session_id = session_id;
|
response->session_id = session_id;
|
||||||
@ -472,14 +484,15 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine<Storage>::commit(const uint64_t l
|
|||||||
if (op_num == Coordination::OpNum::Close)
|
if (op_num == Coordination::OpNum::Close)
|
||||||
|
|
||||||
{
|
{
|
||||||
std::lock_guard lock(request_cache_mutex);
|
std::lock_guard cache_lock(request_cache_mutex);
|
||||||
parsed_request_cache.erase(request_for_session->session_id);
|
parsed_request_cache.erase(request_for_session->session_id);
|
||||||
}
|
}
|
||||||
|
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
{
|
||||||
|
LockGuardWithStats<true> lock(storage_mutex);
|
||||||
|
std::lock_guard response_lock(process_and_responses_lock);
|
||||||
KeeperStorageBase::ResponsesForSessions responses_for_sessions
|
KeeperStorageBase::ResponsesForSessions responses_for_sessions
|
||||||
= storage->processRequest(request_for_session->request, request_for_session->session_id, request_for_session->zxid);
|
= storage->processRequest(request_for_session->request, request_for_session->session_id, request_for_session->zxid);
|
||||||
|
|
||||||
for (auto & response_for_session : responses_for_sessions)
|
for (auto & response_for_session : responses_for_sessions)
|
||||||
{
|
{
|
||||||
if (response_for_session.response->xid != Coordination::WATCH_XID)
|
if (response_for_session.response->xid != Coordination::WATCH_XID)
|
||||||
@ -487,9 +500,15 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine<Storage>::commit(const uint64_t l
|
|||||||
|
|
||||||
try_push(response_for_session);
|
try_push(response_for_session);
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (keeper_context->digestEnabled() && request_for_session->digest)
|
if (keeper_context->digestEnabled() && request_for_session->digest)
|
||||||
assertDigest(*request_for_session->digest, storage->getNodesDigest(true), *request_for_session->request, request_for_session->log_idx, true);
|
assertDigest(
|
||||||
|
*request_for_session->digest,
|
||||||
|
storage->getNodesDigest(true, /*lock_transaction_mutex=*/true),
|
||||||
|
*request_for_session->request,
|
||||||
|
request_for_session->log_idx,
|
||||||
|
true);
|
||||||
}
|
}
|
||||||
|
|
||||||
ProfileEvents::increment(ProfileEvents::KeeperCommits);
|
ProfileEvents::increment(ProfileEvents::KeeperCommits);
|
||||||
@ -534,8 +553,6 @@ bool KeeperStateMachine<Storage>::apply_snapshot(nuraft::snapshot & s)
|
|||||||
}
|
}
|
||||||
|
|
||||||
{ /// deserialize and apply snapshot to storage
|
{ /// deserialize and apply snapshot to storage
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
|
||||||
|
|
||||||
SnapshotDeserializationResult<Storage> snapshot_deserialization_result;
|
SnapshotDeserializationResult<Storage> snapshot_deserialization_result;
|
||||||
if (latest_snapshot_ptr)
|
if (latest_snapshot_ptr)
|
||||||
snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_ptr);
|
snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_ptr);
|
||||||
@ -543,6 +560,7 @@ bool KeeperStateMachine<Storage>::apply_snapshot(nuraft::snapshot & s)
|
|||||||
snapshot_deserialization_result
|
snapshot_deserialization_result
|
||||||
= snapshot_manager.deserializeSnapshotFromBuffer(snapshot_manager.deserializeSnapshotBufferFromDisk(s.get_last_log_idx()));
|
= snapshot_manager.deserializeSnapshotFromBuffer(snapshot_manager.deserializeSnapshotBufferFromDisk(s.get_last_log_idx()));
|
||||||
|
|
||||||
|
LockGuardWithStats storage_lock(storage_mutex);
|
||||||
/// maybe some logs were preprocessed with log idx larger than the snapshot idx
|
/// maybe some logs were preprocessed with log idx larger than the snapshot idx
|
||||||
/// we have to apply them to the new storage
|
/// we have to apply them to the new storage
|
||||||
storage->applyUncommittedState(*snapshot_deserialization_result.storage, snapshot_deserialization_result.snapshot_meta->get_last_log_idx());
|
storage->applyUncommittedState(*snapshot_deserialization_result.storage, snapshot_deserialization_result.snapshot_meta->get_last_log_idx());
|
||||||
@ -587,16 +605,7 @@ void KeeperStateMachine<Storage>::rollbackRequest(const KeeperStorageBase::Reque
|
|||||||
if (request_for_session.request->getOpNum() == Coordination::OpNum::SessionID)
|
if (request_for_session.request->getOpNum() == Coordination::OpNum::SessionID)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
storage->rollbackRequest(request_for_session.zxid, allow_missing);
|
|
||||||
}
|
|
||||||
|
|
||||||
template<typename Storage>
|
|
||||||
void KeeperStateMachine<Storage>::rollbackRequestNoLock(const KeeperStorageBase::RequestForSession & request_for_session, bool allow_missing)
|
|
||||||
{
|
|
||||||
if (request_for_session.request->getOpNum() == Coordination::OpNum::SessionID)
|
|
||||||
return;
|
|
||||||
|
|
||||||
storage->rollbackRequest(request_for_session.zxid, allow_missing);
|
storage->rollbackRequest(request_for_session.zxid, allow_missing);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -616,7 +625,7 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
|
|||||||
auto snapshot_meta_copy = nuraft::snapshot::deserialize(*snp_buf);
|
auto snapshot_meta_copy = nuraft::snapshot::deserialize(*snp_buf);
|
||||||
CreateSnapshotTask snapshot_task;
|
CreateSnapshotTask snapshot_task;
|
||||||
{ /// lock storage for a short period time to turn on "snapshot mode". After that we can read consistent storage state without locking.
|
{ /// lock storage for a short period time to turn on "snapshot mode". After that we can read consistent storage state without locking.
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
snapshot_task.snapshot = std::make_shared<KeeperStorageSnapshot<Storage>>(storage.get(), snapshot_meta_copy, getClusterConfig());
|
snapshot_task.snapshot = std::make_shared<KeeperStorageSnapshot<Storage>>(storage.get(), snapshot_meta_copy, getClusterConfig());
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -681,7 +690,7 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
|
|||||||
}
|
}
|
||||||
{
|
{
|
||||||
/// Destroy snapshot with lock
|
/// Destroy snapshot with lock
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
LOG_TRACE(log, "Clearing garbage after snapshot");
|
LOG_TRACE(log, "Clearing garbage after snapshot");
|
||||||
/// Turn off "snapshot mode" and clear outdate part of storage state
|
/// Turn off "snapshot mode" and clear outdate part of storage state
|
||||||
storage->clearGarbageAfterSnapshot();
|
storage->clearGarbageAfterSnapshot();
|
||||||
@ -824,10 +833,10 @@ template<typename Storage>
|
|||||||
void KeeperStateMachine<Storage>::processReadRequest(const KeeperStorageBase::RequestForSession & request_for_session)
|
void KeeperStateMachine<Storage>::processReadRequest(const KeeperStorageBase::RequestForSession & request_for_session)
|
||||||
{
|
{
|
||||||
/// Pure local request, just process it with storage
|
/// Pure local request, just process it with storage
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats<true> storage_lock(storage_mutex);
|
||||||
|
std::lock_guard response_lock(process_and_responses_lock);
|
||||||
auto responses = storage->processRequest(
|
auto responses = storage->processRequest(
|
||||||
request_for_session.request, request_for_session.session_id, std::nullopt, true /*check_acl*/, true /*is_local*/);
|
request_for_session.request, request_for_session.session_id, std::nullopt, true /*check_acl*/, true /*is_local*/);
|
||||||
|
|
||||||
for (auto & response_for_session : responses)
|
for (auto & response_for_session : responses)
|
||||||
{
|
{
|
||||||
if (response_for_session.response->xid != Coordination::WATCH_XID)
|
if (response_for_session.response->xid != Coordination::WATCH_XID)
|
||||||
@ -840,112 +849,116 @@ void KeeperStateMachine<Storage>::processReadRequest(const KeeperStorageBase::Re
|
|||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void KeeperStateMachine<Storage>::shutdownStorage()
|
void KeeperStateMachine<Storage>::shutdownStorage()
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
storage->finalize();
|
storage->finalize();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
std::vector<int64_t> KeeperStateMachine<Storage>::getDeadSessions()
|
std::vector<int64_t> KeeperStateMachine<Storage>::getDeadSessions()
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getDeadSessions();
|
return storage->getDeadSessions();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
int64_t KeeperStateMachine<Storage>::getNextZxid() const
|
int64_t KeeperStateMachine<Storage>::getNextZxid() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
|
||||||
return storage->getNextZXID();
|
return storage->getNextZXID();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
KeeperStorageBase::Digest KeeperStateMachine<Storage>::getNodesDigest() const
|
KeeperStorageBase::Digest KeeperStateMachine<Storage>::getNodesDigest() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getNodesDigest(false);
|
return storage->getNodesDigest(false, /*lock_transaction_mutex=*/true);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getLastProcessedZxid() const
|
uint64_t KeeperStateMachine<Storage>::getLastProcessedZxid() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
|
||||||
return storage->getZXID();
|
return storage->getZXID();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
template<typename Storage>
|
||||||
|
const KeeperStorageBase::Stats & KeeperStateMachine<Storage>::getStorageStats() const TSA_NO_THREAD_SAFETY_ANALYSIS
|
||||||
|
{
|
||||||
|
return storage->getStorageStats();
|
||||||
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getNodesCount() const
|
uint64_t KeeperStateMachine<Storage>::getNodesCount() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getNodesCount();
|
return storage->getNodesCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getTotalWatchesCount() const
|
uint64_t KeeperStateMachine<Storage>::getTotalWatchesCount() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getTotalWatchesCount();
|
return storage->getTotalWatchesCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getWatchedPathsCount() const
|
uint64_t KeeperStateMachine<Storage>::getWatchedPathsCount() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getWatchedPathsCount();
|
return storage->getWatchedPathsCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getSessionsWithWatchesCount() const
|
uint64_t KeeperStateMachine<Storage>::getSessionsWithWatchesCount() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getSessionsWithWatchesCount();
|
return storage->getSessionsWithWatchesCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getTotalEphemeralNodesCount() const
|
uint64_t KeeperStateMachine<Storage>::getTotalEphemeralNodesCount() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getTotalEphemeralNodesCount();
|
return storage->getTotalEphemeralNodesCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getSessionWithEphemeralNodesCount() const
|
uint64_t KeeperStateMachine<Storage>::getSessionWithEphemeralNodesCount() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getSessionWithEphemeralNodesCount();
|
return storage->getSessionWithEphemeralNodesCount();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void KeeperStateMachine<Storage>::dumpWatches(WriteBufferFromOwnString & buf) const
|
void KeeperStateMachine<Storage>::dumpWatches(WriteBufferFromOwnString & buf) const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
storage->dumpWatches(buf);
|
storage->dumpWatches(buf);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void KeeperStateMachine<Storage>::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
|
void KeeperStateMachine<Storage>::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
storage->dumpWatchesByPath(buf);
|
storage->dumpWatchesByPath(buf);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void KeeperStateMachine<Storage>::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
|
void KeeperStateMachine<Storage>::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
storage->dumpSessionsAndEphemerals(buf);
|
storage->dumpSessionsAndEphemerals(buf);
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getApproximateDataSize() const
|
uint64_t KeeperStateMachine<Storage>::getApproximateDataSize() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getApproximateDataSize();
|
return storage->getApproximateDataSize();
|
||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
uint64_t KeeperStateMachine<Storage>::getKeyArenaSize() const
|
uint64_t KeeperStateMachine<Storage>::getKeyArenaSize() const
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
return storage->getArenaDataSize();
|
return storage->getArenaDataSize();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -988,7 +1001,7 @@ ClusterConfigPtr IKeeperStateMachine::getClusterConfig() const
|
|||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void KeeperStateMachine<Storage>::recalculateStorageStats()
|
void KeeperStateMachine<Storage>::recalculateStorageStats()
|
||||||
{
|
{
|
||||||
LockGuardWithStats lock(storage_and_responses_lock);
|
LockGuardWithStats lock(storage_mutex);
|
||||||
LOG_INFO(log, "Recalculating storage stats");
|
LOG_INFO(log, "Recalculating storage stats");
|
||||||
storage->recalculateStats();
|
storage->recalculateStats();
|
||||||
LOG_INFO(log, "Done recalculating storage stats");
|
LOG_INFO(log, "Done recalculating storage stats");
|
||||||
|
@ -85,6 +85,8 @@ public:
|
|||||||
/// Introspection functions for 4lw commands
|
/// Introspection functions for 4lw commands
|
||||||
virtual uint64_t getLastProcessedZxid() const = 0;
|
virtual uint64_t getLastProcessedZxid() const = 0;
|
||||||
|
|
||||||
|
virtual const KeeperStorageBase::Stats & getStorageStats() const = 0;
|
||||||
|
|
||||||
virtual uint64_t getNodesCount() const = 0;
|
virtual uint64_t getNodesCount() const = 0;
|
||||||
virtual uint64_t getTotalWatchesCount() const = 0;
|
virtual uint64_t getTotalWatchesCount() const = 0;
|
||||||
virtual uint64_t getWatchedPathsCount() const = 0;
|
virtual uint64_t getWatchedPathsCount() const = 0;
|
||||||
@ -124,12 +126,16 @@ protected:
|
|||||||
/// Mutex for snapshots
|
/// Mutex for snapshots
|
||||||
mutable std::mutex snapshots_lock;
|
mutable std::mutex snapshots_lock;
|
||||||
|
|
||||||
/// Lock for storage and responses_queue. It's important to process requests
|
/// Lock for the storage
|
||||||
|
/// Storage works in thread-safe way ONLY for preprocessing/processing
|
||||||
|
/// In any other case, unique storage lock needs to be taken
|
||||||
|
mutable SharedMutex storage_mutex;
|
||||||
|
/// Lock for processing and responses_queue. It's important to process requests
|
||||||
/// and push them to the responses queue while holding this lock. Otherwise
|
/// and push them to the responses queue while holding this lock. Otherwise
|
||||||
/// we can get strange cases when, for example client send read request with
|
/// we can get strange cases when, for example client send read request with
|
||||||
/// watch and after that receive watch response and only receive response
|
/// watch and after that receive watch response and only receive response
|
||||||
/// for request.
|
/// for request.
|
||||||
mutable std::mutex storage_and_responses_lock;
|
mutable std::mutex process_and_responses_lock;
|
||||||
|
|
||||||
std::unordered_map<int64_t, std::unordered_map<Coordination::XID, std::shared_ptr<KeeperStorageBase::RequestForSession>>> parsed_request_cache;
|
std::unordered_map<int64_t, std::unordered_map<Coordination::XID, std::shared_ptr<KeeperStorageBase::RequestForSession>>> parsed_request_cache;
|
||||||
uint64_t min_request_size_to_cache{0};
|
uint64_t min_request_size_to_cache{0};
|
||||||
@ -146,6 +152,7 @@ protected:
|
|||||||
mutable std::mutex cluster_config_lock;
|
mutable std::mutex cluster_config_lock;
|
||||||
ClusterConfigPtr cluster_config;
|
ClusterConfigPtr cluster_config;
|
||||||
|
|
||||||
|
ThreadPool read_pool;
|
||||||
/// Special part of ACL system -- superdigest specified in server config.
|
/// Special part of ACL system -- superdigest specified in server config.
|
||||||
const std::string superdigest;
|
const std::string superdigest;
|
||||||
|
|
||||||
@ -153,10 +160,8 @@ protected:
|
|||||||
|
|
||||||
KeeperSnapshotManagerS3 * snapshot_manager_s3;
|
KeeperSnapshotManagerS3 * snapshot_manager_s3;
|
||||||
|
|
||||||
virtual KeeperStorageBase::ResponseForSession processReconfiguration(
|
virtual KeeperStorageBase::ResponseForSession processReconfiguration(const KeeperStorageBase::RequestForSession & request_for_session)
|
||||||
const KeeperStorageBase::RequestForSession& request_for_session)
|
= 0;
|
||||||
TSA_REQUIRES(storage_and_responses_lock) = 0;
|
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/// ClickHouse Keeper state machine. Wrapper for KeeperStorage.
|
/// ClickHouse Keeper state machine. Wrapper for KeeperStorage.
|
||||||
@ -189,10 +194,6 @@ public:
|
|||||||
// (can happen in case of exception during preprocessing)
|
// (can happen in case of exception during preprocessing)
|
||||||
void rollbackRequest(const KeeperStorageBase::RequestForSession & request_for_session, bool allow_missing) override;
|
void rollbackRequest(const KeeperStorageBase::RequestForSession & request_for_session, bool allow_missing) override;
|
||||||
|
|
||||||
void rollbackRequestNoLock(
|
|
||||||
const KeeperStorageBase::RequestForSession & request_for_session,
|
|
||||||
bool allow_missing) TSA_NO_THREAD_SAFETY_ANALYSIS;
|
|
||||||
|
|
||||||
/// Apply preliminarily saved (save_logical_snp_obj) snapshot to our state.
|
/// Apply preliminarily saved (save_logical_snp_obj) snapshot to our state.
|
||||||
bool apply_snapshot(nuraft::snapshot & s) override;
|
bool apply_snapshot(nuraft::snapshot & s) override;
|
||||||
|
|
||||||
@ -205,7 +206,7 @@ public:
|
|||||||
// This should be used only for tests or keeper-data-dumper because it violates
|
// This should be used only for tests or keeper-data-dumper because it violates
|
||||||
// TSA -- we can't acquire the lock outside of this class or return a storage under lock
|
// TSA -- we can't acquire the lock outside of this class or return a storage under lock
|
||||||
// in a reasonable way.
|
// in a reasonable way.
|
||||||
Storage & getStorageUnsafe() TSA_NO_THREAD_SAFETY_ANALYSIS
|
Storage & getStorageUnsafe()
|
||||||
{
|
{
|
||||||
return *storage;
|
return *storage;
|
||||||
}
|
}
|
||||||
@ -224,6 +225,8 @@ public:
|
|||||||
/// Introspection functions for 4lw commands
|
/// Introspection functions for 4lw commands
|
||||||
uint64_t getLastProcessedZxid() const override;
|
uint64_t getLastProcessedZxid() const override;
|
||||||
|
|
||||||
|
const KeeperStorageBase::Stats & getStorageStats() const override;
|
||||||
|
|
||||||
uint64_t getNodesCount() const override;
|
uint64_t getNodesCount() const override;
|
||||||
uint64_t getTotalWatchesCount() const override;
|
uint64_t getTotalWatchesCount() const override;
|
||||||
uint64_t getWatchedPathsCount() const override;
|
uint64_t getWatchedPathsCount() const override;
|
||||||
@ -245,12 +248,12 @@ public:
|
|||||||
|
|
||||||
private:
|
private:
|
||||||
/// Main state machine logic
|
/// Main state machine logic
|
||||||
std::unique_ptr<Storage> storage; //TSA_PT_GUARDED_BY(storage_and_responses_lock);
|
std::unique_ptr<Storage> storage;
|
||||||
|
|
||||||
/// Save/Load and Serialize/Deserialize logic for snapshots.
|
/// Save/Load and Serialize/Deserialize logic for snapshots.
|
||||||
KeeperSnapshotManager<Storage> snapshot_manager;
|
KeeperSnapshotManager<Storage> snapshot_manager;
|
||||||
|
|
||||||
KeeperStorageBase::ResponseForSession processReconfiguration(const KeeperStorageBase::RequestForSession & request_for_session)
|
KeeperStorageBase::ResponseForSession processReconfiguration(const KeeperStorageBase::RequestForSession & request_for_session) override;
|
||||||
TSA_REQUIRES(storage_and_responses_lock) override;
|
|
||||||
};
|
};
|
||||||
|
|
||||||
}
|
}
|
||||||
|
File diff suppressed because it is too large
Load Diff
@ -1,10 +1,16 @@
|
|||||||
#pragma once
|
#pragma once
|
||||||
|
|
||||||
#include <unordered_map>
|
#include <unordered_map>
|
||||||
|
#include <unordered_set>
|
||||||
#include <vector>
|
#include <vector>
|
||||||
#include <Coordination/ACLMap.h>
|
#include <Coordination/ACLMap.h>
|
||||||
#include <Coordination/SessionExpiryQueue.h>
|
#include <Coordination/SessionExpiryQueue.h>
|
||||||
#include <Coordination/SnapshotableHashTable.h>
|
#include <Coordination/SnapshotableHashTable.h>
|
||||||
|
#include "Common/StringHashForHeterogeneousLookup.h"
|
||||||
|
#include <Common/SharedMutex.h>
|
||||||
|
#include <Common/Concepts.h>
|
||||||
|
|
||||||
|
#include <base/defines.h>
|
||||||
|
|
||||||
#include <absl/container/flat_hash_set.h>
|
#include <absl/container/flat_hash_set.h>
|
||||||
|
|
||||||
@ -23,14 +29,11 @@ using ResponseCallback = std::function<void(const Coordination::ZooKeeperRespons
|
|||||||
using ChildrenSet = absl::flat_hash_set<StringRef, StringRefHash>;
|
using ChildrenSet = absl::flat_hash_set<StringRef, StringRefHash>;
|
||||||
using SessionAndTimeout = std::unordered_map<int64_t, int64_t>;
|
using SessionAndTimeout = std::unordered_map<int64_t, int64_t>;
|
||||||
|
|
||||||
/// KeeperRocksNodeInfo is used in RocksDB keeper.
|
struct NodeStats
|
||||||
/// It is serialized directly as POD to RocksDB.
|
|
||||||
struct KeeperRocksNodeInfo
|
|
||||||
{
|
{
|
||||||
int64_t czxid{0};
|
int64_t czxid{0};
|
||||||
int64_t mzxid{0};
|
int64_t mzxid{0};
|
||||||
int64_t pzxid{0};
|
int64_t pzxid{0};
|
||||||
uint64_t acl_id = 0; /// 0 -- no ACL by default
|
|
||||||
|
|
||||||
int64_t mtime{0};
|
int64_t mtime{0};
|
||||||
|
|
||||||
@ -38,225 +41,9 @@ struct KeeperRocksNodeInfo
|
|||||||
int32_t cversion{0};
|
int32_t cversion{0};
|
||||||
int32_t aversion{0};
|
int32_t aversion{0};
|
||||||
|
|
||||||
int32_t seq_num = 0;
|
uint32_t data_size{0};
|
||||||
mutable UInt64 digest = 0; /// we cached digest for this node.
|
|
||||||
|
|
||||||
/// as ctime can't be negative because it stores the timestamp when the
|
|
||||||
/// node was created, we can use the MSB for a bool
|
|
||||||
struct
|
|
||||||
{
|
|
||||||
bool is_ephemeral : 1;
|
|
||||||
int64_t ctime : 63;
|
|
||||||
} is_ephemeral_and_ctime{false, 0};
|
|
||||||
|
|
||||||
/// ephemeral notes cannot have children so a node can set either
|
|
||||||
/// ephemeral_owner OR seq_num + num_children
|
|
||||||
union
|
|
||||||
{
|
|
||||||
int64_t ephemeral_owner;
|
|
||||||
struct
|
|
||||||
{
|
|
||||||
int32_t seq_num;
|
|
||||||
int32_t num_children;
|
|
||||||
} children_info;
|
|
||||||
} ephemeral_or_children_data{0};
|
|
||||||
|
|
||||||
bool isEphemeral() const
|
|
||||||
{
|
|
||||||
return is_ephemeral_and_ctime.is_ephemeral;
|
|
||||||
}
|
|
||||||
|
|
||||||
int64_t ephemeralOwner() const
|
|
||||||
{
|
|
||||||
if (isEphemeral())
|
|
||||||
return ephemeral_or_children_data.ephemeral_owner;
|
|
||||||
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
void setEphemeralOwner(int64_t ephemeral_owner)
|
|
||||||
{
|
|
||||||
is_ephemeral_and_ctime.is_ephemeral = ephemeral_owner != 0;
|
|
||||||
ephemeral_or_children_data.ephemeral_owner = ephemeral_owner;
|
|
||||||
}
|
|
||||||
|
|
||||||
int32_t numChildren() const
|
|
||||||
{
|
|
||||||
if (isEphemeral())
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
return ephemeral_or_children_data.children_info.num_children;
|
|
||||||
}
|
|
||||||
|
|
||||||
void setNumChildren(int32_t num_children)
|
|
||||||
{
|
|
||||||
ephemeral_or_children_data.children_info.num_children = num_children;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// dummy interface for test
|
|
||||||
void addChild(StringRef) {}
|
|
||||||
auto getChildren() const
|
|
||||||
{
|
|
||||||
return std::vector<int>(numChildren());
|
|
||||||
}
|
|
||||||
|
|
||||||
void increaseNumChildren()
|
|
||||||
{
|
|
||||||
chassert(!isEphemeral());
|
|
||||||
++ephemeral_or_children_data.children_info.num_children;
|
|
||||||
}
|
|
||||||
|
|
||||||
void decreaseNumChildren()
|
|
||||||
{
|
|
||||||
chassert(!isEphemeral());
|
|
||||||
--ephemeral_or_children_data.children_info.num_children;
|
|
||||||
}
|
|
||||||
|
|
||||||
int32_t seqNum() const
|
|
||||||
{
|
|
||||||
if (isEphemeral())
|
|
||||||
return 0;
|
|
||||||
|
|
||||||
return ephemeral_or_children_data.children_info.seq_num;
|
|
||||||
}
|
|
||||||
|
|
||||||
void setSeqNum(int32_t seq_num_)
|
|
||||||
{
|
|
||||||
ephemeral_or_children_data.children_info.seq_num = seq_num_;
|
|
||||||
}
|
|
||||||
|
|
||||||
void increaseSeqNum()
|
|
||||||
{
|
|
||||||
chassert(!isEphemeral());
|
|
||||||
++ephemeral_or_children_data.children_info.seq_num;
|
|
||||||
}
|
|
||||||
|
|
||||||
int64_t ctime() const
|
|
||||||
{
|
|
||||||
return is_ephemeral_and_ctime.ctime;
|
|
||||||
}
|
|
||||||
|
|
||||||
void setCtime(uint64_t ctime)
|
|
||||||
{
|
|
||||||
is_ephemeral_and_ctime.ctime = ctime;
|
|
||||||
}
|
|
||||||
|
|
||||||
void copyStats(const Coordination::Stat & stat);
|
void copyStats(const Coordination::Stat & stat);
|
||||||
};
|
|
||||||
|
|
||||||
/// KeeperRocksNode is the memory structure used by RocksDB
|
|
||||||
struct KeeperRocksNode : public KeeperRocksNodeInfo
|
|
||||||
{
|
|
||||||
#if USE_ROCKSDB
|
|
||||||
friend struct RocksDBContainer<KeeperRocksNode>;
|
|
||||||
#endif
|
|
||||||
using Meta = KeeperRocksNodeInfo;
|
|
||||||
|
|
||||||
uint64_t size_bytes = 0; // only for compatible, should be deprecated
|
|
||||||
|
|
||||||
uint64_t sizeInBytes() const { return data_size + sizeof(KeeperRocksNodeInfo); }
|
|
||||||
void setData(String new_data)
|
|
||||||
{
|
|
||||||
data_size = static_cast<uint32_t>(new_data.size());
|
|
||||||
if (data_size != 0)
|
|
||||||
{
|
|
||||||
data = std::unique_ptr<char[]>(new char[new_data.size()]);
|
|
||||||
memcpy(data.get(), new_data.data(), data_size);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void shallowCopy(const KeeperRocksNode & other)
|
|
||||||
{
|
|
||||||
czxid = other.czxid;
|
|
||||||
mzxid = other.mzxid;
|
|
||||||
pzxid = other.pzxid;
|
|
||||||
acl_id = other.acl_id; /// 0 -- no ACL by default
|
|
||||||
|
|
||||||
mtime = other.mtime;
|
|
||||||
|
|
||||||
is_ephemeral_and_ctime = other.is_ephemeral_and_ctime;
|
|
||||||
|
|
||||||
ephemeral_or_children_data = other.ephemeral_or_children_data;
|
|
||||||
|
|
||||||
data_size = other.data_size;
|
|
||||||
if (data_size != 0)
|
|
||||||
{
|
|
||||||
data = std::unique_ptr<char[]>(new char[data_size]);
|
|
||||||
memcpy(data.get(), other.data.get(), data_size);
|
|
||||||
}
|
|
||||||
|
|
||||||
version = other.version;
|
|
||||||
cversion = other.cversion;
|
|
||||||
aversion = other.aversion;
|
|
||||||
|
|
||||||
/// cached_digest = other.cached_digest;
|
|
||||||
}
|
|
||||||
void invalidateDigestCache() const;
|
|
||||||
UInt64 getDigest(std::string_view path) const;
|
|
||||||
String getEncodedString();
|
|
||||||
void decodeFromString(const String & buffer_str);
|
|
||||||
void recalculateSize() {}
|
|
||||||
std::string_view getData() const noexcept { return {data.get(), data_size}; }
|
|
||||||
|
|
||||||
void setResponseStat(Coordination::Stat & response_stat) const
|
|
||||||
{
|
|
||||||
response_stat.czxid = czxid;
|
|
||||||
response_stat.mzxid = mzxid;
|
|
||||||
response_stat.ctime = ctime();
|
|
||||||
response_stat.mtime = mtime;
|
|
||||||
response_stat.version = version;
|
|
||||||
response_stat.cversion = cversion;
|
|
||||||
response_stat.aversion = aversion;
|
|
||||||
response_stat.ephemeralOwner = ephemeralOwner();
|
|
||||||
response_stat.dataLength = static_cast<int32_t>(data_size);
|
|
||||||
response_stat.numChildren = numChildren();
|
|
||||||
response_stat.pzxid = pzxid;
|
|
||||||
}
|
|
||||||
|
|
||||||
void reset()
|
|
||||||
{
|
|
||||||
serialized = false;
|
|
||||||
}
|
|
||||||
bool empty() const
|
|
||||||
{
|
|
||||||
return data_size == 0 && mzxid == 0;
|
|
||||||
}
|
|
||||||
std::unique_ptr<char[]> data{nullptr};
|
|
||||||
uint32_t data_size{0};
|
|
||||||
private:
|
|
||||||
bool serialized = false;
|
|
||||||
};
|
|
||||||
|
|
||||||
/// KeeperMemNode should have as minimal size as possible to reduce memory footprint
|
|
||||||
/// of stored nodes
|
|
||||||
/// New fields should be added to the struct only if it's really necessary
|
|
||||||
struct KeeperMemNode
|
|
||||||
{
|
|
||||||
int64_t czxid{0};
|
|
||||||
int64_t mzxid{0};
|
|
||||||
int64_t pzxid{0};
|
|
||||||
uint64_t acl_id = 0; /// 0 -- no ACL by default
|
|
||||||
|
|
||||||
int64_t mtime{0};
|
|
||||||
|
|
||||||
std::unique_ptr<char[]> data{nullptr};
|
|
||||||
uint32_t data_size{0};
|
|
||||||
|
|
||||||
int32_t version{0};
|
|
||||||
int32_t cversion{0};
|
|
||||||
int32_t aversion{0};
|
|
||||||
|
|
||||||
mutable uint64_t cached_digest = 0;
|
|
||||||
|
|
||||||
KeeperMemNode() = default;
|
|
||||||
|
|
||||||
KeeperMemNode & operator=(const KeeperMemNode & other);
|
|
||||||
KeeperMemNode(const KeeperMemNode & other);
|
|
||||||
|
|
||||||
KeeperMemNode & operator=(KeeperMemNode && other) noexcept;
|
|
||||||
KeeperMemNode(KeeperMemNode && other) noexcept;
|
|
||||||
|
|
||||||
bool empty() const;
|
|
||||||
|
|
||||||
bool isEphemeral() const
|
bool isEphemeral() const
|
||||||
{
|
{
|
||||||
@ -287,6 +74,7 @@ struct KeeperMemNode
|
|||||||
|
|
||||||
void setNumChildren(int32_t num_children)
|
void setNumChildren(int32_t num_children)
|
||||||
{
|
{
|
||||||
|
is_ephemeral_and_ctime.is_ephemeral = false;
|
||||||
ephemeral_or_children_data.children_info.num_children = num_children;
|
ephemeral_or_children_data.children_info.num_children = num_children;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -331,34 +119,6 @@ struct KeeperMemNode
|
|||||||
is_ephemeral_and_ctime.ctime = ctime;
|
is_ephemeral_and_ctime.ctime = ctime;
|
||||||
}
|
}
|
||||||
|
|
||||||
void copyStats(const Coordination::Stat & stat);
|
|
||||||
|
|
||||||
void setResponseStat(Coordination::Stat & response_stat) const;
|
|
||||||
|
|
||||||
/// Object memory size
|
|
||||||
uint64_t sizeInBytes() const;
|
|
||||||
|
|
||||||
void setData(const String & new_data);
|
|
||||||
|
|
||||||
std::string_view getData() const noexcept { return {data.get(), data_size}; }
|
|
||||||
|
|
||||||
void addChild(StringRef child_path);
|
|
||||||
|
|
||||||
void removeChild(StringRef child_path);
|
|
||||||
|
|
||||||
const auto & getChildren() const noexcept { return children; }
|
|
||||||
auto & getChildren() { return children; }
|
|
||||||
|
|
||||||
// Invalidate the calculated digest so it's recalculated again on the next
|
|
||||||
// getDigest call
|
|
||||||
void invalidateDigestCache() const;
|
|
||||||
|
|
||||||
// get the calculated digest of the node
|
|
||||||
UInt64 getDigest(std::string_view path) const;
|
|
||||||
|
|
||||||
// copy only necessary information for preprocessing and digest calculation
|
|
||||||
// (e.g. we don't need to copy list of children)
|
|
||||||
void shallowCopy(const KeeperMemNode & other);
|
|
||||||
private:
|
private:
|
||||||
/// as ctime can't be negative because it stores the timestamp when the
|
/// as ctime can't be negative because it stores the timestamp when the
|
||||||
/// node was created, we can use the MSB for a bool
|
/// node was created, we can use the MSB for a bool
|
||||||
@ -379,7 +139,132 @@ private:
|
|||||||
int32_t num_children;
|
int32_t num_children;
|
||||||
} children_info;
|
} children_info;
|
||||||
} ephemeral_or_children_data{0};
|
} ephemeral_or_children_data{0};
|
||||||
|
};
|
||||||
|
|
||||||
|
/// KeeperRocksNodeInfo is used in RocksDB keeper.
|
||||||
|
/// It is serialized directly as POD to RocksDB.
|
||||||
|
struct KeeperRocksNodeInfo
|
||||||
|
{
|
||||||
|
NodeStats stats;
|
||||||
|
uint64_t acl_id = 0; /// 0 -- no ACL by default
|
||||||
|
|
||||||
|
/// dummy interface for test
|
||||||
|
void addChild(StringRef) {}
|
||||||
|
auto getChildren() const
|
||||||
|
{
|
||||||
|
return std::vector<int>(stats.numChildren());
|
||||||
|
}
|
||||||
|
|
||||||
|
void copyStats(const Coordination::Stat & stat);
|
||||||
|
};
|
||||||
|
|
||||||
|
/// KeeperRocksNode is the memory structure used by RocksDB
|
||||||
|
struct KeeperRocksNode : public KeeperRocksNodeInfo
|
||||||
|
{
|
||||||
|
#if USE_ROCKSDB
|
||||||
|
friend struct RocksDBContainer<KeeperRocksNode>;
|
||||||
|
#endif
|
||||||
|
using Meta = KeeperRocksNodeInfo;
|
||||||
|
|
||||||
|
uint64_t size_bytes = 0; // only for compatible, should be deprecated
|
||||||
|
|
||||||
|
uint64_t sizeInBytes() const { return stats.data_size + sizeof(KeeperRocksNodeInfo); }
|
||||||
|
|
||||||
|
void setData(String new_data)
|
||||||
|
{
|
||||||
|
stats.data_size = static_cast<uint32_t>(new_data.size());
|
||||||
|
if (stats.data_size != 0)
|
||||||
|
{
|
||||||
|
data = std::unique_ptr<char[]>(new char[new_data.size()]);
|
||||||
|
memcpy(data.get(), new_data.data(), stats.data_size);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void shallowCopy(const KeeperRocksNode & other)
|
||||||
|
{
|
||||||
|
stats = other.stats;
|
||||||
|
acl_id = other.acl_id;
|
||||||
|
if (stats.data_size != 0)
|
||||||
|
{
|
||||||
|
data = std::unique_ptr<char[]>(new char[stats.data_size]);
|
||||||
|
memcpy(data.get(), other.data.get(), stats.data_size);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// cached_digest = other.cached_digest;
|
||||||
|
}
|
||||||
|
void invalidateDigestCache() const;
|
||||||
|
UInt64 getDigest(std::string_view path) const;
|
||||||
|
String getEncodedString();
|
||||||
|
void decodeFromString(const String & buffer_str);
|
||||||
|
void recalculateSize() {}
|
||||||
|
std::string_view getData() const noexcept { return {data.get(), stats.data_size}; }
|
||||||
|
|
||||||
|
void setResponseStat(Coordination::Stat & response_stat) const;
|
||||||
|
|
||||||
|
void reset()
|
||||||
|
{
|
||||||
|
serialized = false;
|
||||||
|
}
|
||||||
|
bool empty() const
|
||||||
|
{
|
||||||
|
return stats.data_size == 0 && stats.mzxid == 0;
|
||||||
|
}
|
||||||
|
std::unique_ptr<char[]> data{nullptr};
|
||||||
|
mutable UInt64 cached_digest = 0; /// we cached digest for this node.
|
||||||
|
private:
|
||||||
|
bool serialized = false;
|
||||||
|
};
|
||||||
|
|
||||||
|
/// KeeperMemNode should have as minimal size as possible to reduce memory footprint
|
||||||
|
/// of stored nodes
|
||||||
|
/// New fields should be added to the struct only if it's really necessary
|
||||||
|
struct KeeperMemNode
|
||||||
|
{
|
||||||
|
NodeStats stats;
|
||||||
|
std::unique_ptr<char[]> data{nullptr};
|
||||||
|
mutable uint64_t cached_digest = 0;
|
||||||
|
|
||||||
|
uint64_t acl_id = 0; /// 0 -- no ACL by default
|
||||||
|
|
||||||
|
KeeperMemNode() = default;
|
||||||
|
|
||||||
|
KeeperMemNode & operator=(const KeeperMemNode & other);
|
||||||
|
KeeperMemNode(const KeeperMemNode & other);
|
||||||
|
|
||||||
|
KeeperMemNode & operator=(KeeperMemNode && other) noexcept;
|
||||||
|
KeeperMemNode(KeeperMemNode && other) noexcept;
|
||||||
|
|
||||||
|
bool empty() const;
|
||||||
|
|
||||||
|
void copyStats(const Coordination::Stat & stat);
|
||||||
|
|
||||||
|
void setResponseStat(Coordination::Stat & response_stat) const;
|
||||||
|
|
||||||
|
/// Object memory size
|
||||||
|
uint64_t sizeInBytes() const;
|
||||||
|
|
||||||
|
void setData(const String & new_data);
|
||||||
|
|
||||||
|
std::string_view getData() const noexcept { return {data.get(), stats.data_size}; }
|
||||||
|
|
||||||
|
void addChild(StringRef child_path);
|
||||||
|
|
||||||
|
void removeChild(StringRef child_path);
|
||||||
|
|
||||||
|
const auto & getChildren() const noexcept { return children; }
|
||||||
|
auto & getChildren() { return children; }
|
||||||
|
|
||||||
|
// Invalidate the calculated digest so it's recalculated again on the next
|
||||||
|
// getDigest call
|
||||||
|
void invalidateDigestCache() const;
|
||||||
|
|
||||||
|
// get the calculated digest of the node
|
||||||
|
UInt64 getDigest(std::string_view path) const;
|
||||||
|
|
||||||
|
// copy only necessary information for preprocessing and digest calculation
|
||||||
|
// (e.g. we don't need to copy list of children)
|
||||||
|
void shallowCopy(const KeeperMemNode & other);
|
||||||
|
private:
|
||||||
ChildrenSet children{};
|
ChildrenSet children{};
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -430,18 +315,187 @@ public:
|
|||||||
};
|
};
|
||||||
|
|
||||||
using Ephemerals = std::unordered_map<int64_t, std::unordered_set<std::string>>;
|
using Ephemerals = std::unordered_map<int64_t, std::unordered_set<std::string>>;
|
||||||
using SessionAndWatcher = std::unordered_map<int64_t, std::unordered_set<std::string>>;
|
struct WatchInfo
|
||||||
|
{
|
||||||
|
std::string_view path;
|
||||||
|
bool is_list_watch;
|
||||||
|
|
||||||
|
bool operator==(const WatchInfo &) const = default;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct WatchInfoHash
|
||||||
|
{
|
||||||
|
auto operator()(WatchInfo info) const
|
||||||
|
{
|
||||||
|
SipHash hash;
|
||||||
|
hash.update(info.path);
|
||||||
|
hash.update(info.is_list_watch);
|
||||||
|
return hash.get64();
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
using SessionAndWatcher = std::unordered_map<int64_t, std::unordered_set<WatchInfo, WatchInfoHash>>;
|
||||||
using SessionIDs = std::unordered_set<int64_t>;
|
using SessionIDs = std::unordered_set<int64_t>;
|
||||||
|
|
||||||
/// Just vector of SHA1 from user:password
|
/// Just vector of SHA1 from user:password
|
||||||
using AuthIDs = std::vector<AuthID>;
|
using AuthIDs = std::vector<AuthID>;
|
||||||
using SessionAndAuth = std::unordered_map<int64_t, AuthIDs>;
|
using SessionAndAuth = std::unordered_map<int64_t, AuthIDs>;
|
||||||
using Watches = std::unordered_map<String /* path, relative of root_path */, SessionIDs>;
|
using Watches = std::unordered_map<
|
||||||
|
String /* path, relative of root_path */,
|
||||||
|
SessionIDs,
|
||||||
|
StringHashForHeterogeneousLookup,
|
||||||
|
StringHashForHeterogeneousLookup::transparent_key_equal>;
|
||||||
|
|
||||||
|
// Applying ZooKeeper request to storage consists of two steps:
|
||||||
|
// - preprocessing which, instead of applying the changes directly to storage,
|
||||||
|
// generates deltas with those changes, denoted with the request ZXID
|
||||||
|
// - processing which applies deltas with the correct ZXID to the storage
|
||||||
|
//
|
||||||
|
// Delta objects allow us two things:
|
||||||
|
// - fetch the latest, uncommitted state of an object by getting the committed
|
||||||
|
// state of that same object from the storage and applying the deltas
|
||||||
|
// in the same order as they are defined
|
||||||
|
// - quickly commit the changes to the storage
|
||||||
|
struct CreateNodeDelta
|
||||||
|
{
|
||||||
|
Coordination::Stat stat;
|
||||||
|
Coordination::ACLs acls;
|
||||||
|
String data;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct RemoveNodeDelta
|
||||||
|
{
|
||||||
|
int32_t version{-1};
|
||||||
|
NodeStats stat;
|
||||||
|
Coordination::ACLs acls;
|
||||||
|
String data;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct UpdateNodeStatDelta
|
||||||
|
{
|
||||||
|
template <is_any_of<KeeperMemNode, KeeperRocksNode> Node>
|
||||||
|
explicit UpdateNodeStatDelta(const Node & node)
|
||||||
|
: old_stats(node.stats)
|
||||||
|
, new_stats(node.stats)
|
||||||
|
{}
|
||||||
|
|
||||||
|
NodeStats old_stats;
|
||||||
|
NodeStats new_stats;
|
||||||
|
int32_t version{-1};
|
||||||
|
};
|
||||||
|
|
||||||
|
struct UpdateNodeDataDelta
|
||||||
|
{
|
||||||
|
|
||||||
|
std::string old_data;
|
||||||
|
std::string new_data;
|
||||||
|
int32_t version{-1};
|
||||||
|
};
|
||||||
|
|
||||||
|
struct SetACLDelta
|
||||||
|
{
|
||||||
|
Coordination::ACLs old_acls;
|
||||||
|
Coordination::ACLs new_acls;
|
||||||
|
int32_t version{-1};
|
||||||
|
};
|
||||||
|
|
||||||
|
struct ErrorDelta
|
||||||
|
{
|
||||||
|
Coordination::Error error;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct FailedMultiDelta
|
||||||
|
{
|
||||||
|
std::vector<Coordination::Error> error_codes;
|
||||||
|
Coordination::Error global_error{Coordination::Error::ZOK};
|
||||||
|
};
|
||||||
|
|
||||||
|
// Denotes end of a subrequest in multi request
|
||||||
|
struct SubDeltaEnd
|
||||||
|
{
|
||||||
|
};
|
||||||
|
|
||||||
|
struct AddAuthDelta
|
||||||
|
{
|
||||||
|
int64_t session_id;
|
||||||
|
std::shared_ptr<AuthID> auth_id;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct CloseSessionDelta
|
||||||
|
{
|
||||||
|
int64_t session_id;
|
||||||
|
};
|
||||||
|
|
||||||
|
using Operation = std::variant<
|
||||||
|
CreateNodeDelta,
|
||||||
|
RemoveNodeDelta,
|
||||||
|
UpdateNodeStatDelta,
|
||||||
|
UpdateNodeDataDelta,
|
||||||
|
SetACLDelta,
|
||||||
|
AddAuthDelta,
|
||||||
|
ErrorDelta,
|
||||||
|
SubDeltaEnd,
|
||||||
|
FailedMultiDelta,
|
||||||
|
CloseSessionDelta>;
|
||||||
|
|
||||||
|
struct Delta
|
||||||
|
{
|
||||||
|
Delta(String path_, int64_t zxid_, Operation operation_) : path(std::move(path_)), zxid(zxid_), operation(std::move(operation_)) { }
|
||||||
|
|
||||||
|
Delta(int64_t zxid_, Coordination::Error error) : Delta("", zxid_, ErrorDelta{error}) { }
|
||||||
|
|
||||||
|
Delta(int64_t zxid_, Operation subdelta) : Delta("", zxid_, subdelta) { }
|
||||||
|
|
||||||
|
String path;
|
||||||
|
int64_t zxid;
|
||||||
|
Operation operation;
|
||||||
|
};
|
||||||
|
|
||||||
|
using DeltaIterator = std::list<KeeperStorageBase::Delta>::const_iterator;
|
||||||
|
struct DeltaRange
|
||||||
|
{
|
||||||
|
DeltaIterator begin_it;
|
||||||
|
DeltaIterator end_it;
|
||||||
|
|
||||||
|
auto begin() const
|
||||||
|
{
|
||||||
|
return begin_it;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto end() const
|
||||||
|
{
|
||||||
|
return end_it;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool empty() const
|
||||||
|
{
|
||||||
|
return begin_it == end_it;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto & front() const
|
||||||
|
{
|
||||||
|
return *begin_it;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
struct Stats
|
||||||
|
{
|
||||||
|
std::atomic<uint64_t> nodes_count = 0;
|
||||||
|
std::atomic<uint64_t> approximate_data_size = 0;
|
||||||
|
std::atomic<uint64_t> total_watches_count = 0;
|
||||||
|
std::atomic<uint64_t> watched_paths_count = 0;
|
||||||
|
std::atomic<uint64_t> sessions_with_watches_count = 0;
|
||||||
|
std::atomic<uint64_t> session_with_ephemeral_nodes_count = 0;
|
||||||
|
std::atomic<uint64_t> total_emphemeral_nodes_count = 0;
|
||||||
|
std::atomic<int64_t> last_zxid = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
Stats stats;
|
||||||
|
|
||||||
static bool checkDigest(const Digest & first, const Digest & second);
|
static bool checkDigest(const Digest & first, const Digest & second);
|
||||||
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
/// Keeper state machine almost equal to the ZooKeeper's state machine.
|
/// Keeper state machine almost equal to the ZooKeeper's state machine.
|
||||||
/// Implements all logic of operations, data changes, sessions allocation.
|
/// Implements all logic of operations, data changes, sessions allocation.
|
||||||
/// In-memory and not thread safe.
|
/// In-memory and not thread safe.
|
||||||
@ -472,159 +526,73 @@ public:
|
|||||||
|
|
||||||
int64_t session_id_counter{1};
|
int64_t session_id_counter{1};
|
||||||
|
|
||||||
SessionAndAuth session_and_auth;
|
mutable SharedMutex auth_mutex;
|
||||||
|
SessionAndAuth committed_session_and_auth;
|
||||||
|
|
||||||
|
mutable SharedMutex storage_mutex;
|
||||||
/// Main hashtable with nodes. Contain all information about data.
|
/// Main hashtable with nodes. Contain all information about data.
|
||||||
/// All other structures expect session_and_timeout can be restored from
|
/// All other structures expect session_and_timeout can be restored from
|
||||||
/// container.
|
/// container.
|
||||||
Container container;
|
Container container;
|
||||||
|
|
||||||
// Applying ZooKeeper request to storage consists of two steps:
|
|
||||||
// - preprocessing which, instead of applying the changes directly to storage,
|
|
||||||
// generates deltas with those changes, denoted with the request ZXID
|
|
||||||
// - processing which applies deltas with the correct ZXID to the storage
|
|
||||||
//
|
|
||||||
// Delta objects allow us two things:
|
|
||||||
// - fetch the latest, uncommitted state of an object by getting the committed
|
|
||||||
// state of that same object from the storage and applying the deltas
|
|
||||||
// in the same order as they are defined
|
|
||||||
// - quickly commit the changes to the storage
|
|
||||||
struct CreateNodeDelta
|
|
||||||
{
|
|
||||||
Coordination::Stat stat;
|
|
||||||
Coordination::ACLs acls;
|
|
||||||
String data;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct RemoveNodeDelta
|
|
||||||
{
|
|
||||||
int32_t version{-1};
|
|
||||||
int64_t ephemeral_owner{0};
|
|
||||||
};
|
|
||||||
|
|
||||||
struct UpdateNodeDelta
|
|
||||||
{
|
|
||||||
std::function<void(Node &)> update_fn;
|
|
||||||
int32_t version{-1};
|
|
||||||
};
|
|
||||||
|
|
||||||
struct SetACLDelta
|
|
||||||
{
|
|
||||||
Coordination::ACLs acls;
|
|
||||||
int32_t version{-1};
|
|
||||||
};
|
|
||||||
|
|
||||||
struct ErrorDelta
|
|
||||||
{
|
|
||||||
Coordination::Error error;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct FailedMultiDelta
|
|
||||||
{
|
|
||||||
std::vector<Coordination::Error> error_codes;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Denotes end of a subrequest in multi request
|
|
||||||
struct SubDeltaEnd
|
|
||||||
{
|
|
||||||
};
|
|
||||||
|
|
||||||
struct AddAuthDelta
|
|
||||||
{
|
|
||||||
int64_t session_id;
|
|
||||||
AuthID auth_id;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct CloseSessionDelta
|
|
||||||
{
|
|
||||||
int64_t session_id;
|
|
||||||
};
|
|
||||||
|
|
||||||
using Operation = std::
|
|
||||||
variant<CreateNodeDelta, RemoveNodeDelta, UpdateNodeDelta, SetACLDelta, AddAuthDelta, ErrorDelta, SubDeltaEnd, FailedMultiDelta, CloseSessionDelta>;
|
|
||||||
|
|
||||||
struct Delta
|
|
||||||
{
|
|
||||||
Delta(String path_, int64_t zxid_, Operation operation_) : path(std::move(path_)), zxid(zxid_), operation(std::move(operation_)) { }
|
|
||||||
|
|
||||||
Delta(int64_t zxid_, Coordination::Error error) : Delta("", zxid_, ErrorDelta{error}) { }
|
|
||||||
|
|
||||||
Delta(int64_t zxid_, Operation subdelta) : Delta("", zxid_, subdelta) { }
|
|
||||||
|
|
||||||
String path;
|
|
||||||
int64_t zxid;
|
|
||||||
Operation operation;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct UncommittedState
|
struct UncommittedState
|
||||||
{
|
{
|
||||||
explicit UncommittedState(KeeperStorage & storage_) : storage(storage_) { }
|
explicit UncommittedState(KeeperStorage & storage_) : storage(storage_) { }
|
||||||
|
|
||||||
void addDelta(Delta new_delta);
|
void addDeltas(std::list<Delta> new_deltas);
|
||||||
void addDeltas(std::vector<Delta> new_deltas);
|
void cleanup(int64_t commit_zxid);
|
||||||
void commit(int64_t commit_zxid);
|
|
||||||
void rollback(int64_t rollback_zxid);
|
void rollback(int64_t rollback_zxid);
|
||||||
|
void rollback(std::list<Delta> rollback_deltas);
|
||||||
|
|
||||||
std::shared_ptr<Node> getNode(StringRef path) const;
|
std::shared_ptr<Node> getNode(StringRef path, bool should_lock_storage = true) const;
|
||||||
const Node * getActualNodeView(StringRef path, const Node & storage_node) const;
|
const Node * getActualNodeView(StringRef path, const Node & storage_node) const;
|
||||||
|
|
||||||
Coordination::ACLs getACLs(StringRef path) const;
|
Coordination::ACLs getACLs(StringRef path) const;
|
||||||
|
|
||||||
|
void applyDeltas(const std::list<Delta> & new_deltas);
|
||||||
void applyDelta(const Delta & delta);
|
void applyDelta(const Delta & delta);
|
||||||
|
void rollbackDelta(const Delta & delta);
|
||||||
|
|
||||||
bool hasACL(int64_t session_id, bool is_local, std::function<bool(const AuthID &)> predicate) const;
|
bool hasACL(int64_t session_id, bool is_local, std::function<bool(const AuthID &)> predicate) const;
|
||||||
|
|
||||||
void forEachAuthInSession(int64_t session_id, std::function<void(const AuthID &)> func) const;
|
void forEachAuthInSession(int64_t session_id, std::function<void(const AuthID &)> func) const;
|
||||||
|
|
||||||
std::shared_ptr<Node> tryGetNodeFromStorage(StringRef path) const;
|
std::shared_ptr<Node> tryGetNodeFromStorage(StringRef path, bool should_lock_storage = true) const;
|
||||||
|
|
||||||
std::unordered_map<int64_t, std::list<const AuthID *>> session_and_auth;
|
|
||||||
std::unordered_set<int64_t> closed_sessions;
|
std::unordered_set<int64_t> closed_sessions;
|
||||||
|
|
||||||
|
using ZxidToNodes = std::map<int64_t, std::unordered_set<std::string_view>>;
|
||||||
struct UncommittedNode
|
struct UncommittedNode
|
||||||
{
|
{
|
||||||
std::shared_ptr<Node> node{nullptr};
|
std::shared_ptr<Node> node{nullptr};
|
||||||
Coordination::ACLs acls{};
|
std::optional<Coordination::ACLs> acls{};
|
||||||
int64_t zxid{0};
|
std::unordered_set<uint64_t> applied_zxids{};
|
||||||
};
|
|
||||||
|
|
||||||
struct Hash
|
void materializeACL(const ACLMap & current_acl_map);
|
||||||
{
|
|
||||||
auto operator()(const std::string_view view) const
|
|
||||||
{
|
|
||||||
SipHash hash;
|
|
||||||
hash.update(view);
|
|
||||||
return hash.get64();
|
|
||||||
}
|
|
||||||
|
|
||||||
using is_transparent = void; // required to make find() work with different type than key_type
|
|
||||||
};
|
|
||||||
|
|
||||||
struct Equal
|
|
||||||
{
|
|
||||||
auto operator()(const std::string_view a,
|
|
||||||
const std::string_view b) const
|
|
||||||
{
|
|
||||||
return a == b;
|
|
||||||
}
|
|
||||||
|
|
||||||
using is_transparent = void; // required to make find() work with different type than key_type
|
|
||||||
};
|
};
|
||||||
|
|
||||||
struct PathCmp
|
struct PathCmp
|
||||||
{
|
{
|
||||||
using is_transparent = std::true_type;
|
|
||||||
|
|
||||||
auto operator()(const std::string_view a,
|
auto operator()(const std::string_view a,
|
||||||
const std::string_view b) const
|
const std::string_view b) const
|
||||||
{
|
{
|
||||||
return a.size() < b.size() || (a.size() == b.size() && a < b);
|
size_t level_a = std::count(a.begin(), a.end(), '/');
|
||||||
|
size_t level_b = std::count(b.begin(), b.end(), '/');
|
||||||
|
return level_a < level_b || (level_a == level_b && a < b);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
using is_transparent = void; // required to make find() work with different type than key_type
|
||||||
};
|
};
|
||||||
|
|
||||||
mutable std::map<std::string, UncommittedNode, PathCmp> nodes;
|
Ephemerals ephemerals;
|
||||||
std::unordered_map<std::string, std::list<const Delta *>, Hash, Equal> deltas_for_path;
|
|
||||||
|
|
||||||
std::list<Delta> deltas;
|
std::unordered_map<int64_t, std::list<std::pair<int64_t, std::shared_ptr<AuthID>>>> session_and_auth;
|
||||||
|
|
||||||
|
mutable std::map<std::string, UncommittedNode, PathCmp> nodes;
|
||||||
|
mutable ZxidToNodes zxid_to_nodes;
|
||||||
|
|
||||||
|
mutable std::mutex deltas_mutex;
|
||||||
|
std::list<Delta> deltas TSA_GUARDED_BY(deltas_mutex);
|
||||||
KeeperStorage<Container> & storage;
|
KeeperStorage<Container> & storage;
|
||||||
};
|
};
|
||||||
|
|
||||||
@ -634,7 +602,7 @@ public:
|
|||||||
// with zxid > last_zxid
|
// with zxid > last_zxid
|
||||||
void applyUncommittedState(KeeperStorage & other, int64_t last_log_idx);
|
void applyUncommittedState(KeeperStorage & other, int64_t last_log_idx);
|
||||||
|
|
||||||
Coordination::Error commit(int64_t zxid);
|
Coordination::Error commit(DeltaRange deltas);
|
||||||
|
|
||||||
// Create node in the storage
|
// Create node in the storage
|
||||||
// Returns false if it failed to create the node, true otherwise
|
// Returns false if it failed to create the node, true otherwise
|
||||||
@ -652,12 +620,11 @@ public:
|
|||||||
|
|
||||||
bool checkACL(StringRef path, int32_t permissions, int64_t session_id, bool is_local);
|
bool checkACL(StringRef path, int32_t permissions, int64_t session_id, bool is_local);
|
||||||
|
|
||||||
void unregisterEphemeralPath(int64_t session_id, const std::string & path);
|
std::mutex ephemeral_mutex;
|
||||||
|
|
||||||
/// Mapping session_id -> set of ephemeral nodes paths
|
/// Mapping session_id -> set of ephemeral nodes paths
|
||||||
Ephemerals ephemerals;
|
Ephemerals committed_ephemerals;
|
||||||
/// Mapping session_id -> set of watched nodes paths
|
size_t committed_ephemeral_nodes{0};
|
||||||
SessionAndWatcher sessions_and_watchers;
|
|
||||||
/// Expiration queue for session, allows to get dead sessions at some point of time
|
/// Expiration queue for session, allows to get dead sessions at some point of time
|
||||||
SessionExpiryQueue session_expiry_queue;
|
SessionExpiryQueue session_expiry_queue;
|
||||||
/// All active sessions with timeout
|
/// All active sessions with timeout
|
||||||
@ -666,8 +633,10 @@ public:
|
|||||||
/// ACLMap for more compact ACLs storage inside nodes.
|
/// ACLMap for more compact ACLs storage inside nodes.
|
||||||
ACLMap acl_map;
|
ACLMap acl_map;
|
||||||
|
|
||||||
|
mutable std::mutex transaction_mutex;
|
||||||
|
|
||||||
/// Global id of all requests applied to storage
|
/// Global id of all requests applied to storage
|
||||||
int64_t zxid{0};
|
int64_t zxid TSA_GUARDED_BY(transaction_mutex) = 0;
|
||||||
|
|
||||||
// older Keeper node (pre V5 snapshots) can create snapshots and receive logs from newer Keeper nodes
|
// older Keeper node (pre V5 snapshots) can create snapshots and receive logs from newer Keeper nodes
|
||||||
// this can lead to some inconsistencies, e.g. from snapshot it will use log_idx as zxid
|
// this can lead to some inconsistencies, e.g. from snapshot it will use log_idx as zxid
|
||||||
@ -684,11 +653,16 @@ public:
|
|||||||
int64_t log_idx = 0;
|
int64_t log_idx = 0;
|
||||||
};
|
};
|
||||||
|
|
||||||
std::deque<TransactionInfo> uncommitted_transactions;
|
std::list<TransactionInfo> uncommitted_transactions TSA_GUARDED_BY(transaction_mutex);
|
||||||
|
|
||||||
uint64_t nodes_digest{0};
|
uint64_t nodes_digest = 0;
|
||||||
|
|
||||||
bool finalized{false};
|
std::atomic<bool> finalized{false};
|
||||||
|
|
||||||
|
|
||||||
|
/// Mapping session_id -> set of watched nodes paths
|
||||||
|
SessionAndWatcher sessions_and_watchers;
|
||||||
|
size_t total_watches_count = 0;
|
||||||
|
|
||||||
/// Currently active watches (node_path -> subscribed sessions)
|
/// Currently active watches (node_path -> subscribed sessions)
|
||||||
Watches watches;
|
Watches watches;
|
||||||
@ -697,45 +671,30 @@ public:
|
|||||||
void clearDeadWatches(int64_t session_id);
|
void clearDeadWatches(int64_t session_id);
|
||||||
|
|
||||||
/// Get current committed zxid
|
/// Get current committed zxid
|
||||||
int64_t getZXID() const { return zxid; }
|
int64_t getZXID() const;
|
||||||
|
|
||||||
int64_t getNextZXID() const
|
int64_t getNextZXID() const;
|
||||||
{
|
int64_t getNextZXIDLocked() const TSA_REQUIRES(transaction_mutex);
|
||||||
if (uncommitted_transactions.empty())
|
|
||||||
return zxid + 1;
|
|
||||||
|
|
||||||
return uncommitted_transactions.back().zxid + 1;
|
Digest getNodesDigest(bool committed, bool lock_transaction_mutex) const;
|
||||||
}
|
|
||||||
|
|
||||||
Digest getNodesDigest(bool committed) const;
|
|
||||||
|
|
||||||
KeeperContextPtr keeper_context;
|
KeeperContextPtr keeper_context;
|
||||||
|
|
||||||
const String superdigest;
|
const String superdigest;
|
||||||
|
|
||||||
bool initialized{false};
|
std::atomic<bool> initialized{false};
|
||||||
|
|
||||||
KeeperStorage(int64_t tick_time_ms, const String & superdigest_, const KeeperContextPtr & keeper_context_, bool initialize_system_nodes = true);
|
KeeperStorage(int64_t tick_time_ms, const String & superdigest_, const KeeperContextPtr & keeper_context_, bool initialize_system_nodes = true);
|
||||||
|
|
||||||
void initializeSystemNodes();
|
void initializeSystemNodes() TSA_NO_THREAD_SAFETY_ANALYSIS;
|
||||||
|
|
||||||
/// Allocate new session id with the specified timeouts
|
/// Allocate new session id with the specified timeouts
|
||||||
int64_t getSessionID(int64_t session_timeout_ms)
|
int64_t getSessionID(int64_t session_timeout_ms);
|
||||||
{
|
|
||||||
auto result = session_id_counter++;
|
|
||||||
session_and_timeout.emplace(result, session_timeout_ms);
|
|
||||||
session_expiry_queue.addNewSessionOrUpdate(result, session_timeout_ms);
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Add session id. Used when restoring KeeperStorage from snapshot.
|
/// Add session id. Used when restoring KeeperStorage from snapshot.
|
||||||
void addSessionID(int64_t session_id, int64_t session_timeout_ms)
|
void addSessionID(int64_t session_id, int64_t session_timeout_ms) TSA_NO_THREAD_SAFETY_ANALYSIS;
|
||||||
{
|
|
||||||
session_and_timeout.emplace(session_id, session_timeout_ms);
|
|
||||||
session_expiry_queue.addNewSessionOrUpdate(session_id, session_timeout_ms);
|
|
||||||
}
|
|
||||||
|
|
||||||
UInt64 calculateNodesDigest(UInt64 current_digest, const std::vector<Delta> & new_deltas) const;
|
UInt64 calculateNodesDigest(UInt64 current_digest, const std::list<Delta> & new_deltas) const;
|
||||||
|
|
||||||
/// Process user request and return response.
|
/// Process user request and return response.
|
||||||
/// check_acl = false only when converting data from ZooKeeper.
|
/// check_acl = false only when converting data from ZooKeeper.
|
||||||
@ -762,42 +721,39 @@ public:
|
|||||||
/// Set of methods for creating snapshots
|
/// Set of methods for creating snapshots
|
||||||
|
|
||||||
/// Turn on snapshot mode, so data inside Container is not deleted, but replaced with new version.
|
/// Turn on snapshot mode, so data inside Container is not deleted, but replaced with new version.
|
||||||
void enableSnapshotMode(size_t up_to_version)
|
void enableSnapshotMode(size_t up_to_version);
|
||||||
{
|
|
||||||
container.enableSnapshotMode(up_to_version);
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Turn off snapshot mode.
|
/// Turn off snapshot mode.
|
||||||
void disableSnapshotMode()
|
void disableSnapshotMode();
|
||||||
{
|
|
||||||
container.disableSnapshotMode();
|
|
||||||
}
|
|
||||||
|
|
||||||
Container::const_iterator getSnapshotIteratorBegin() const { return container.begin(); }
|
Container::const_iterator getSnapshotIteratorBegin() const;
|
||||||
|
|
||||||
/// Clear outdated data from internal container.
|
/// Clear outdated data from internal container.
|
||||||
void clearGarbageAfterSnapshot() { container.clearOutdatedNodes(); }
|
void clearGarbageAfterSnapshot();
|
||||||
|
|
||||||
/// Get all active sessions
|
/// Get all active sessions
|
||||||
const SessionAndTimeout & getActiveSessions() const { return session_and_timeout; }
|
SessionAndTimeout getActiveSessions() const;
|
||||||
|
|
||||||
/// Get all dead sessions
|
/// Get all dead sessions
|
||||||
std::vector<int64_t> getDeadSessions() const { return session_expiry_queue.getExpiredSessions(); }
|
std::vector<int64_t> getDeadSessions() const;
|
||||||
|
|
||||||
|
void updateStats();
|
||||||
|
const Stats & getStorageStats() const;
|
||||||
|
|
||||||
/// Introspection functions mostly used in 4-letter commands
|
/// Introspection functions mostly used in 4-letter commands
|
||||||
uint64_t getNodesCount() const { return container.size(); }
|
uint64_t getNodesCount() const;
|
||||||
|
|
||||||
uint64_t getApproximateDataSize() const { return container.getApproximateDataSize(); }
|
uint64_t getApproximateDataSize() const;
|
||||||
|
|
||||||
uint64_t getArenaDataSize() const { return container.keyArenaSize(); }
|
uint64_t getArenaDataSize() const;
|
||||||
|
|
||||||
uint64_t getTotalWatchesCount() const;
|
uint64_t getTotalWatchesCount() const;
|
||||||
|
|
||||||
uint64_t getWatchedPathsCount() const { return watches.size() + list_watches.size(); }
|
uint64_t getWatchedPathsCount() const;
|
||||||
|
|
||||||
uint64_t getSessionsWithWatchesCount() const;
|
uint64_t getSessionsWithWatchesCount() const;
|
||||||
|
|
||||||
uint64_t getSessionWithEphemeralNodesCount() const { return ephemerals.size(); }
|
uint64_t getSessionWithEphemeralNodesCount() const;
|
||||||
uint64_t getTotalEphemeralNodesCount() const;
|
uint64_t getTotalEphemeralNodesCount() const;
|
||||||
|
|
||||||
void dumpWatches(WriteBufferFromOwnString & buf) const;
|
void dumpWatches(WriteBufferFromOwnString & buf) const;
|
||||||
|
@ -155,11 +155,11 @@ public:
|
|||||||
ReadBufferFromOwnString buffer(iter->value().ToStringView());
|
ReadBufferFromOwnString buffer(iter->value().ToStringView());
|
||||||
typename Node::Meta & meta = new_pair->value;
|
typename Node::Meta & meta = new_pair->value;
|
||||||
readPODBinary(meta, buffer);
|
readPODBinary(meta, buffer);
|
||||||
readVarUInt(new_pair->value.data_size, buffer);
|
readVarUInt(new_pair->value.stats.data_size, buffer);
|
||||||
if (new_pair->value.data_size)
|
if (new_pair->value.stats.data_size)
|
||||||
{
|
{
|
||||||
new_pair->value.data = std::unique_ptr<char[]>(new char[new_pair->value.data_size]);
|
new_pair->value.data = std::unique_ptr<char[]>(new char[new_pair->value.stats.data_size]);
|
||||||
buffer.readStrict(new_pair->value.data.get(), new_pair->value.data_size);
|
buffer.readStrict(new_pair->value.data.get(), new_pair->value.stats.data_size);
|
||||||
}
|
}
|
||||||
pair = new_pair;
|
pair = new_pair;
|
||||||
}
|
}
|
||||||
@ -211,7 +211,7 @@ public:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
std::vector<std::pair<std::string, Node>> getChildren(const std::string & key_)
|
std::vector<std::pair<std::string, Node>> getChildren(const std::string & key_, bool read_data = false)
|
||||||
{
|
{
|
||||||
rocksdb::ReadOptions read_options;
|
rocksdb::ReadOptions read_options;
|
||||||
read_options.total_order_seek = true;
|
read_options.total_order_seek = true;
|
||||||
@ -232,6 +232,15 @@ public:
|
|||||||
typename Node::Meta & meta = node;
|
typename Node::Meta & meta = node;
|
||||||
/// We do not read data here
|
/// We do not read data here
|
||||||
readPODBinary(meta, buffer);
|
readPODBinary(meta, buffer);
|
||||||
|
if (read_data)
|
||||||
|
{
|
||||||
|
readVarUInt(meta.stats.data_size, buffer);
|
||||||
|
if (meta.stats.data_size)
|
||||||
|
{
|
||||||
|
node.data = std::unique_ptr<char[]>(new char[meta.stats.data_size]);
|
||||||
|
buffer.readStrict(node.data.get(), meta.stats.data_size);
|
||||||
|
}
|
||||||
|
}
|
||||||
std::string real_key(iter->key().data() + len, iter->key().size() - len);
|
std::string real_key(iter->key().data() + len, iter->key().size() - len);
|
||||||
// std::cout << "real key: " << real_key << std::endl;
|
// std::cout << "real key: " << real_key << std::endl;
|
||||||
result.emplace_back(std::move(real_key), std::move(node));
|
result.emplace_back(std::move(real_key), std::move(node));
|
||||||
@ -268,11 +277,11 @@ public:
|
|||||||
typename Node::Meta & meta = kv->value;
|
typename Node::Meta & meta = kv->value;
|
||||||
readPODBinary(meta, buffer);
|
readPODBinary(meta, buffer);
|
||||||
/// TODO: Sometimes we don't need to load data.
|
/// TODO: Sometimes we don't need to load data.
|
||||||
readVarUInt(kv->value.data_size, buffer);
|
readVarUInt(kv->value.stats.data_size, buffer);
|
||||||
if (kv->value.data_size)
|
if (kv->value.stats.data_size)
|
||||||
{
|
{
|
||||||
kv->value.data = std::unique_ptr<char[]>(new char[kv->value.data_size]);
|
kv->value.data = std::unique_ptr<char[]>(new char[kv->value.stats.data_size]);
|
||||||
buffer.readStrict(kv->value.data.get(), kv->value.data_size);
|
buffer.readStrict(kv->value.data.get(), kv->value.stats.data_size);
|
||||||
}
|
}
|
||||||
return const_iterator(kv);
|
return const_iterator(kv);
|
||||||
}
|
}
|
||||||
@ -281,7 +290,7 @@ public:
|
|||||||
{
|
{
|
||||||
auto it = find(key);
|
auto it = find(key);
|
||||||
chassert(it != end());
|
chassert(it != end());
|
||||||
return MockNode(it->value.numChildren(), it->value.getData());
|
return MockNode(it->value.stats.numChildren(), it->value.getData());
|
||||||
}
|
}
|
||||||
|
|
||||||
const_iterator updateValue(StringRef key_, ValueUpdater updater)
|
const_iterator updateValue(StringRef key_, ValueUpdater updater)
|
||||||
|
@ -93,7 +93,7 @@ void deserializeACLMap(Storage & storage, ReadBuffer & in)
|
|||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
int64_t deserializeStorageData(Storage & storage, ReadBuffer & in, LoggerPtr log)
|
int64_t deserializeStorageData(Storage & storage, ReadBuffer & in, LoggerPtr log) TSA_NO_THREAD_SAFETY_ANALYSIS
|
||||||
{
|
{
|
||||||
int64_t max_zxid = 0;
|
int64_t max_zxid = 0;
|
||||||
std::string path;
|
std::string path;
|
||||||
@ -108,33 +108,33 @@ int64_t deserializeStorageData(Storage & storage, ReadBuffer & in, LoggerPtr log
|
|||||||
Coordination::read(node.acl_id, in);
|
Coordination::read(node.acl_id, in);
|
||||||
|
|
||||||
/// Deserialize stat
|
/// Deserialize stat
|
||||||
Coordination::read(node.czxid, in);
|
Coordination::read(node.stats.czxid, in);
|
||||||
Coordination::read(node.mzxid, in);
|
Coordination::read(node.stats.mzxid, in);
|
||||||
/// For some reason ZXID specified in filename can be smaller
|
/// For some reason ZXID specified in filename can be smaller
|
||||||
/// then actual zxid from nodes. In this case we will use zxid from nodes.
|
/// then actual zxid from nodes. In this case we will use zxid from nodes.
|
||||||
max_zxid = std::max(max_zxid, node.mzxid);
|
max_zxid = std::max(max_zxid, node.stats.mzxid);
|
||||||
|
|
||||||
int64_t ctime;
|
int64_t ctime;
|
||||||
Coordination::read(ctime, in);
|
Coordination::read(ctime, in);
|
||||||
node.setCtime(ctime);
|
node.stats.setCtime(ctime);
|
||||||
Coordination::read(node.mtime, in);
|
Coordination::read(node.stats.mtime, in);
|
||||||
Coordination::read(node.version, in);
|
Coordination::read(node.stats.version, in);
|
||||||
Coordination::read(node.cversion, in);
|
Coordination::read(node.stats.cversion, in);
|
||||||
Coordination::read(node.aversion, in);
|
Coordination::read(node.stats.aversion, in);
|
||||||
int64_t ephemeral_owner;
|
int64_t ephemeral_owner;
|
||||||
Coordination::read(ephemeral_owner, in);
|
Coordination::read(ephemeral_owner, in);
|
||||||
if (ephemeral_owner != 0)
|
if (ephemeral_owner != 0)
|
||||||
node.setEphemeralOwner(ephemeral_owner);
|
node.stats.setEphemeralOwner(ephemeral_owner);
|
||||||
Coordination::read(node.pzxid, in);
|
Coordination::read(node.stats.pzxid, in);
|
||||||
if (!path.empty())
|
if (!path.empty())
|
||||||
{
|
{
|
||||||
if (ephemeral_owner == 0)
|
if (ephemeral_owner == 0)
|
||||||
node.setSeqNum(node.cversion);
|
node.stats.setSeqNum(node.stats.cversion);
|
||||||
|
|
||||||
storage.container.insertOrReplace(path, node);
|
storage.container.insertOrReplace(path, node);
|
||||||
|
|
||||||
if (ephemeral_owner != 0)
|
if (ephemeral_owner != 0)
|
||||||
storage.ephemerals[ephemeral_owner].insert(path);
|
storage.committed_ephemerals[ephemeral_owner].insert(path);
|
||||||
|
|
||||||
storage.acl_map.addUsage(node.acl_id);
|
storage.acl_map.addUsage(node.acl_id);
|
||||||
}
|
}
|
||||||
@ -149,7 +149,13 @@ int64_t deserializeStorageData(Storage & storage, ReadBuffer & in, LoggerPtr log
|
|||||||
if (itr.key != "/")
|
if (itr.key != "/")
|
||||||
{
|
{
|
||||||
auto parent_path = parentNodePath(itr.key);
|
auto parent_path = parentNodePath(itr.key);
|
||||||
storage.container.updateValue(parent_path, [my_path = itr.key] (typename Storage::Node & value) { value.addChild(getBaseNodeName(my_path)); value.increaseNumChildren(); });
|
storage.container.updateValue(
|
||||||
|
parent_path,
|
||||||
|
[my_path = itr.key](typename Storage::Node & value)
|
||||||
|
{
|
||||||
|
value.addChild(getBaseNodeName(my_path));
|
||||||
|
value.stats.increaseNumChildren();
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -157,7 +163,7 @@ int64_t deserializeStorageData(Storage & storage, ReadBuffer & in, LoggerPtr log
|
|||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void deserializeKeeperStorageFromSnapshot(Storage & storage, const std::string & snapshot_path, LoggerPtr log)
|
void deserializeKeeperStorageFromSnapshot(Storage & storage, const std::string & snapshot_path, LoggerPtr log) TSA_NO_THREAD_SAFETY_ANALYSIS
|
||||||
{
|
{
|
||||||
LOG_INFO(log, "Deserializing storage snapshot {}", snapshot_path);
|
LOG_INFO(log, "Deserializing storage snapshot {}", snapshot_path);
|
||||||
int64_t zxid = getZxidFromName(snapshot_path);
|
int64_t zxid = getZxidFromName(snapshot_path);
|
||||||
@ -487,7 +493,7 @@ bool hasErrorsInMultiRequest(Coordination::ZooKeeperRequestPtr request)
|
|||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
bool deserializeTxn(Storage & storage, ReadBuffer & in, LoggerPtr /*log*/)
|
bool deserializeTxn(Storage & storage, ReadBuffer & in, LoggerPtr /*log*/) TSA_NO_THREAD_SAFETY_ANALYSIS
|
||||||
{
|
{
|
||||||
int64_t checksum;
|
int64_t checksum;
|
||||||
Coordination::read(checksum, in);
|
Coordination::read(checksum, in);
|
||||||
@ -568,7 +574,7 @@ void deserializeLogAndApplyToStorage(Storage & storage, const std::string & log_
|
|||||||
}
|
}
|
||||||
|
|
||||||
template<typename Storage>
|
template<typename Storage>
|
||||||
void deserializeLogsAndApplyToStorage(Storage & storage, const std::string & path, LoggerPtr log)
|
void deserializeLogsAndApplyToStorage(Storage & storage, const std::string & path, LoggerPtr log) TSA_NO_THREAD_SAFETY_ANALYSIS
|
||||||
{
|
{
|
||||||
std::map<int64_t, std::string> existing_logs;
|
std::map<int64_t, std::string> existing_logs;
|
||||||
for (const auto & p : fs::directory_iterator(path))
|
for (const auto & p : fs::directory_iterator(path))
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
#include <chrono>
|
#include <chrono>
|
||||||
#include <gtest/gtest.h>
|
#include <gtest/gtest.h>
|
||||||
|
|
||||||
|
#include "base/defines.h"
|
||||||
#include "config.h"
|
#include "config.h"
|
||||||
|
|
||||||
#if USE_NURAFT
|
#if USE_NURAFT
|
||||||
@ -1540,7 +1541,7 @@ void addNode(Storage & storage, const std::string & path, const std::string & da
|
|||||||
using Node = typename Storage::Node;
|
using Node = typename Storage::Node;
|
||||||
Node node{};
|
Node node{};
|
||||||
node.setData(data);
|
node.setData(data);
|
||||||
node.setEphemeralOwner(ephemeral_owner);
|
node.stats.setEphemeralOwner(ephemeral_owner);
|
||||||
storage.container.insertOrReplace(path, node);
|
storage.container.insertOrReplace(path, node);
|
||||||
auto child_it = storage.container.find(path);
|
auto child_it = storage.container.find(path);
|
||||||
auto child_path = DB::getBaseNodeName(child_it->key);
|
auto child_path = DB::getBaseNodeName(child_it->key);
|
||||||
@ -1549,7 +1550,7 @@ void addNode(Storage & storage, const std::string & path, const std::string & da
|
|||||||
[&](auto & parent)
|
[&](auto & parent)
|
||||||
{
|
{
|
||||||
parent.addChild(child_path);
|
parent.addChild(child_path);
|
||||||
parent.increaseNumChildren();
|
parent.stats.increaseNumChildren();
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1570,9 +1571,9 @@ TYPED_TEST(CoordinationTest, TestStorageSnapshotSimple)
|
|||||||
addNode(storage, "/hello1", "world", 1);
|
addNode(storage, "/hello1", "world", 1);
|
||||||
addNode(storage, "/hello2", "somedata", 3);
|
addNode(storage, "/hello2", "somedata", 3);
|
||||||
storage.session_id_counter = 5;
|
storage.session_id_counter = 5;
|
||||||
storage.zxid = 2;
|
TSA_SUPPRESS_WARNING_FOR_WRITE(storage.zxid) = 2;
|
||||||
storage.ephemerals[3] = {"/hello2"};
|
storage.committed_ephemerals[3] = {"/hello2"};
|
||||||
storage.ephemerals[1] = {"/hello1"};
|
storage.committed_ephemerals[1] = {"/hello1"};
|
||||||
storage.getSessionID(130);
|
storage.getSessionID(130);
|
||||||
storage.getSessionID(130);
|
storage.getSessionID(130);
|
||||||
|
|
||||||
@ -1601,10 +1602,10 @@ TYPED_TEST(CoordinationTest, TestStorageSnapshotSimple)
|
|||||||
EXPECT_EQ(restored_storage->container.getValue("/hello1").getData(), "world");
|
EXPECT_EQ(restored_storage->container.getValue("/hello1").getData(), "world");
|
||||||
EXPECT_EQ(restored_storage->container.getValue("/hello2").getData(), "somedata");
|
EXPECT_EQ(restored_storage->container.getValue("/hello2").getData(), "somedata");
|
||||||
EXPECT_EQ(restored_storage->session_id_counter, 7);
|
EXPECT_EQ(restored_storage->session_id_counter, 7);
|
||||||
EXPECT_EQ(restored_storage->zxid, 2);
|
EXPECT_EQ(restored_storage->getZXID(), 2);
|
||||||
EXPECT_EQ(restored_storage->ephemerals.size(), 2);
|
EXPECT_EQ(restored_storage->committed_ephemerals.size(), 2);
|
||||||
EXPECT_EQ(restored_storage->ephemerals[3].size(), 1);
|
EXPECT_EQ(restored_storage->committed_ephemerals[3].size(), 1);
|
||||||
EXPECT_EQ(restored_storage->ephemerals[1].size(), 1);
|
EXPECT_EQ(restored_storage->committed_ephemerals[1].size(), 1);
|
||||||
EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
|
EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2027,7 +2028,7 @@ TYPED_TEST(CoordinationTest, TestEphemeralNodeRemove)
|
|||||||
state_machine->commit(1, entry_c->get_buf());
|
state_machine->commit(1, entry_c->get_buf());
|
||||||
const auto & storage = state_machine->getStorageUnsafe();
|
const auto & storage = state_machine->getStorageUnsafe();
|
||||||
|
|
||||||
EXPECT_EQ(storage.ephemerals.size(), 1);
|
EXPECT_EQ(storage.committed_ephemerals.size(), 1);
|
||||||
std::shared_ptr<ZooKeeperRemoveRequest> request_d = std::make_shared<ZooKeeperRemoveRequest>();
|
std::shared_ptr<ZooKeeperRemoveRequest> request_d = std::make_shared<ZooKeeperRemoveRequest>();
|
||||||
request_d->path = "/hello";
|
request_d->path = "/hello";
|
||||||
/// Delete from other session
|
/// Delete from other session
|
||||||
@ -2035,7 +2036,7 @@ TYPED_TEST(CoordinationTest, TestEphemeralNodeRemove)
|
|||||||
state_machine->pre_commit(2, entry_d->get_buf());
|
state_machine->pre_commit(2, entry_d->get_buf());
|
||||||
state_machine->commit(2, entry_d->get_buf());
|
state_machine->commit(2, entry_d->get_buf());
|
||||||
|
|
||||||
EXPECT_EQ(storage.ephemerals.size(), 0);
|
EXPECT_EQ(storage.committed_ephemerals.size(), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@ -2280,6 +2281,62 @@ TYPED_TEST(CoordinationTest, TestPreprocessWhenCloseSessionIsPrecommitted)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
TYPED_TEST(CoordinationTest, TestMultiRequestWithNoAuth)
|
||||||
|
{
|
||||||
|
using namespace Coordination;
|
||||||
|
using namespace DB;
|
||||||
|
|
||||||
|
ChangelogDirTest snapshots("./snapshots");
|
||||||
|
this->setSnapshotDirectory("./snapshots");
|
||||||
|
|
||||||
|
using Storage = typename TestFixture::Storage;
|
||||||
|
|
||||||
|
ChangelogDirTest rocks("./rocksdb");
|
||||||
|
this->setRocksDBDirectory("./rocksdb");
|
||||||
|
ResponsesQueue queue(std::numeric_limits<size_t>::max());
|
||||||
|
SnapshotsQueue snapshots_queue{1};
|
||||||
|
int64_t session_without_auth = 1;
|
||||||
|
int64_t session_with_auth = 2;
|
||||||
|
size_t term = 0;
|
||||||
|
|
||||||
|
auto state_machine = std::make_shared<KeeperStateMachine<Storage>>(queue, snapshots_queue, this->keeper_context, nullptr);
|
||||||
|
state_machine->init();
|
||||||
|
|
||||||
|
auto & storage = state_machine->getStorageUnsafe();
|
||||||
|
|
||||||
|
auto auth_req = std::make_shared<ZooKeeperAuthRequest>();
|
||||||
|
auth_req->scheme = "digest";
|
||||||
|
auth_req->data = "test_user:test_password";
|
||||||
|
|
||||||
|
// Add auth data to the session
|
||||||
|
auto auth_entry = getLogEntryFromZKRequest(term, session_with_auth, state_machine->getNextZxid(), auth_req);
|
||||||
|
state_machine->pre_commit(1, auth_entry->get_buf());
|
||||||
|
state_machine->commit(1, auth_entry->get_buf());
|
||||||
|
|
||||||
|
std::string node_with_acl = "/node_with_acl";
|
||||||
|
{
|
||||||
|
auto create_req = std::make_shared<ZooKeeperCreateRequest>();
|
||||||
|
create_req->path = node_with_acl;
|
||||||
|
create_req->data = "notmodified";
|
||||||
|
create_req->acls = {{.permissions = ACL::Read, .scheme = "auth", .id = ""}};
|
||||||
|
auto create_entry = getLogEntryFromZKRequest(term, session_with_auth, state_machine->getNextZxid(), create_req);
|
||||||
|
state_machine->pre_commit(3, create_entry->get_buf());
|
||||||
|
state_machine->commit(3, create_entry->get_buf());
|
||||||
|
ASSERT_TRUE(storage.container.contains(node_with_acl));
|
||||||
|
}
|
||||||
|
Requests ops;
|
||||||
|
ops.push_back(zkutil::makeSetRequest(node_with_acl, "modified", -1));
|
||||||
|
ops.push_back(zkutil::makeCheckRequest("/nonexistentnode", -1));
|
||||||
|
auto multi_req = std::make_shared<ZooKeeperMultiRequest>(ops, ACLs{});
|
||||||
|
auto multi_entry = getLogEntryFromZKRequest(term, session_without_auth, state_machine->getNextZxid(), multi_req);
|
||||||
|
state_machine->pre_commit(4, multi_entry->get_buf());
|
||||||
|
state_machine->commit(4, multi_entry->get_buf());
|
||||||
|
|
||||||
|
auto node_it = storage.container.find(node_with_acl);
|
||||||
|
ASSERT_FALSE(node_it == storage.container.end());
|
||||||
|
ASSERT_TRUE(node_it->value.getData() == "notmodified");
|
||||||
|
}
|
||||||
|
|
||||||
TYPED_TEST(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
|
TYPED_TEST(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
|
||||||
{
|
{
|
||||||
using namespace Coordination;
|
using namespace Coordination;
|
||||||
@ -2534,9 +2591,9 @@ TYPED_TEST(CoordinationTest, TestStorageSnapshotDifferentCompressions)
|
|||||||
addNode(storage, "/hello1", "world", 1);
|
addNode(storage, "/hello1", "world", 1);
|
||||||
addNode(storage, "/hello2", "somedata", 3);
|
addNode(storage, "/hello2", "somedata", 3);
|
||||||
storage.session_id_counter = 5;
|
storage.session_id_counter = 5;
|
||||||
storage.zxid = 2;
|
TSA_SUPPRESS_WARNING_FOR_WRITE(storage.zxid) = 2;
|
||||||
storage.ephemerals[3] = {"/hello2"};
|
storage.committed_ephemerals[3] = {"/hello2"};
|
||||||
storage.ephemerals[1] = {"/hello1"};
|
storage.committed_ephemerals[1] = {"/hello1"};
|
||||||
storage.getSessionID(130);
|
storage.getSessionID(130);
|
||||||
storage.getSessionID(130);
|
storage.getSessionID(130);
|
||||||
|
|
||||||
@ -2561,10 +2618,10 @@ TYPED_TEST(CoordinationTest, TestStorageSnapshotDifferentCompressions)
|
|||||||
EXPECT_EQ(restored_storage->container.getValue("/hello1").getData(), "world");
|
EXPECT_EQ(restored_storage->container.getValue("/hello1").getData(), "world");
|
||||||
EXPECT_EQ(restored_storage->container.getValue("/hello2").getData(), "somedata");
|
EXPECT_EQ(restored_storage->container.getValue("/hello2").getData(), "somedata");
|
||||||
EXPECT_EQ(restored_storage->session_id_counter, 7);
|
EXPECT_EQ(restored_storage->session_id_counter, 7);
|
||||||
EXPECT_EQ(restored_storage->zxid, 2);
|
EXPECT_EQ(restored_storage->getZXID(), 2);
|
||||||
EXPECT_EQ(restored_storage->ephemerals.size(), 2);
|
EXPECT_EQ(restored_storage->committed_ephemerals.size(), 2);
|
||||||
EXPECT_EQ(restored_storage->ephemerals[3].size(), 1);
|
EXPECT_EQ(restored_storage->committed_ephemerals[3].size(), 1);
|
||||||
EXPECT_EQ(restored_storage->ephemerals[1].size(), 1);
|
EXPECT_EQ(restored_storage->committed_ephemerals[1].size(), 1);
|
||||||
EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
|
EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -2749,13 +2806,13 @@ TYPED_TEST(CoordinationTest, TestStorageSnapshotEqual)
|
|||||||
|
|
||||||
storage.session_id_counter = 5;
|
storage.session_id_counter = 5;
|
||||||
|
|
||||||
storage.ephemerals[3] = {"/hello"};
|
storage.committed_ephemerals[3] = {"/hello"};
|
||||||
storage.ephemerals[1] = {"/hello/somepath"};
|
storage.committed_ephemerals[1] = {"/hello/somepath"};
|
||||||
|
|
||||||
for (size_t j = 0; j < 3333; ++j)
|
for (size_t j = 0; j < 3333; ++j)
|
||||||
storage.getSessionID(130 * j);
|
storage.getSessionID(130 * j);
|
||||||
|
|
||||||
DB::KeeperStorageSnapshot<Storage> snapshot(&storage, storage.zxid);
|
DB::KeeperStorageSnapshot<Storage> snapshot(&storage, storage.getZXID());
|
||||||
|
|
||||||
auto buf = manager.serializeSnapshotToBuffer(snapshot);
|
auto buf = manager.serializeSnapshotToBuffer(snapshot);
|
||||||
|
|
||||||
@ -3259,7 +3316,7 @@ TYPED_TEST(CoordinationTest, TestCheckNotExistsRequest)
|
|||||||
create_path("/test_node");
|
create_path("/test_node");
|
||||||
auto node_it = storage.container.find("/test_node");
|
auto node_it = storage.container.find("/test_node");
|
||||||
ASSERT_NE(node_it, storage.container.end());
|
ASSERT_NE(node_it, storage.container.end());
|
||||||
auto node_version = node_it->value.version;
|
auto node_version = node_it->value.stats.version;
|
||||||
|
|
||||||
{
|
{
|
||||||
SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS");
|
SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS");
|
||||||
@ -3510,12 +3567,12 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest)
|
|||||||
{
|
{
|
||||||
SCOPED_TRACE("Recursive Remove Ephemeral");
|
SCOPED_TRACE("Recursive Remove Ephemeral");
|
||||||
create("/T7", zkutil::CreateMode::Ephemeral);
|
create("/T7", zkutil::CreateMode::Ephemeral);
|
||||||
ASSERT_EQ(storage.ephemerals.size(), 1);
|
ASSERT_EQ(storage.committed_ephemerals.size(), 1);
|
||||||
|
|
||||||
auto responses = remove_recursive("/T7", 100);
|
auto responses = remove_recursive("/T7", 100);
|
||||||
ASSERT_EQ(responses.size(), 1);
|
ASSERT_EQ(responses.size(), 1);
|
||||||
ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK);
|
ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK);
|
||||||
ASSERT_EQ(storage.ephemerals.size(), 0);
|
ASSERT_EQ(storage.committed_ephemerals.size(), 0);
|
||||||
ASSERT_FALSE(exists("/T7"));
|
ASSERT_FALSE(exists("/T7"));
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -3525,12 +3582,12 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveRequest)
|
|||||||
create("/T8/A", zkutil::CreateMode::Persistent);
|
create("/T8/A", zkutil::CreateMode::Persistent);
|
||||||
create("/T8/B", zkutil::CreateMode::Ephemeral);
|
create("/T8/B", zkutil::CreateMode::Ephemeral);
|
||||||
create("/T8/A/C", zkutil::CreateMode::Ephemeral);
|
create("/T8/A/C", zkutil::CreateMode::Ephemeral);
|
||||||
ASSERT_EQ(storage.ephemerals.size(), 1);
|
ASSERT_EQ(storage.committed_ephemerals.size(), 1);
|
||||||
|
|
||||||
auto responses = remove_recursive("/T8", 4);
|
auto responses = remove_recursive("/T8", 4);
|
||||||
ASSERT_EQ(responses.size(), 1);
|
ASSERT_EQ(responses.size(), 1);
|
||||||
ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK);
|
ASSERT_EQ(responses[0].response->error, Coordination::Error::ZOK);
|
||||||
ASSERT_EQ(storage.ephemerals.size(), 0);
|
ASSERT_EQ(storage.committed_ephemerals.size(), 0);
|
||||||
ASSERT_FALSE(exists("/T8"));
|
ASSERT_FALSE(exists("/T8"));
|
||||||
ASSERT_FALSE(exists("/T8/A"));
|
ASSERT_FALSE(exists("/T8/A"));
|
||||||
ASSERT_FALSE(exists("/T8/B"));
|
ASSERT_FALSE(exists("/T8/B"));
|
||||||
@ -3682,6 +3739,72 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveInMultiRequest)
|
|||||||
ASSERT_FALSE(exists("/A/B"));
|
ASSERT_FALSE(exists("/A/B"));
|
||||||
ASSERT_FALSE(exists("/A/B/D"));
|
ASSERT_FALSE(exists("/A/B/D"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
SCOPED_TRACE("Recursive Remove For Subtree With Updated Node");
|
||||||
|
int create_zxid = ++zxid;
|
||||||
|
auto ops = prepare_create_tree();
|
||||||
|
|
||||||
|
/// First create nodes
|
||||||
|
const auto create_request = std::make_shared<ZooKeeperMultiRequest>(ops, ACLs{});
|
||||||
|
storage.preprocessRequest(create_request, 1, 0, create_zxid);
|
||||||
|
auto create_responses = storage.processRequest(create_request, 1, create_zxid);
|
||||||
|
ASSERT_EQ(create_responses.size(), 1);
|
||||||
|
ASSERT_TRUE(is_multi_ok(create_responses[0].response));
|
||||||
|
|
||||||
|
/// Small limit
|
||||||
|
int remove_zxid = ++zxid;
|
||||||
|
ops = {
|
||||||
|
zkutil::makeSetRequest("/A/B", "", -1),
|
||||||
|
zkutil::makeRemoveRecursiveRequest("/A", 3),
|
||||||
|
};
|
||||||
|
auto remove_request = std::make_shared<ZooKeeperMultiRequest>(ops, ACLs{});
|
||||||
|
storage.preprocessRequest(remove_request, 1, 0, remove_zxid);
|
||||||
|
auto remove_responses = storage.processRequest(remove_request, 1, remove_zxid);
|
||||||
|
|
||||||
|
ASSERT_EQ(remove_responses.size(), 1);
|
||||||
|
ASSERT_FALSE(is_multi_ok(remove_responses[0].response));
|
||||||
|
|
||||||
|
/// Big limit
|
||||||
|
remove_zxid = ++zxid;
|
||||||
|
ops[1] = zkutil::makeRemoveRecursiveRequest("/A", 4);
|
||||||
|
remove_request = std::make_shared<ZooKeeperMultiRequest>(ops, ACLs{});
|
||||||
|
storage.preprocessRequest(remove_request, 1, 0, remove_zxid);
|
||||||
|
remove_responses = storage.processRequest(remove_request, 1, remove_zxid);
|
||||||
|
|
||||||
|
ASSERT_EQ(remove_responses.size(), 1);
|
||||||
|
ASSERT_TRUE(is_multi_ok(remove_responses[0].response));
|
||||||
|
ASSERT_FALSE(exists("/A"));
|
||||||
|
ASSERT_FALSE(exists("/A/C"));
|
||||||
|
ASSERT_FALSE(exists("/A/B"));
|
||||||
|
ASSERT_FALSE(exists("/A/B/D"));
|
||||||
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
SCOPED_TRACE("[BUG] Recursive Remove Level Sorting");
|
||||||
|
int new_zxid = ++zxid;
|
||||||
|
|
||||||
|
Coordination::Requests ops = {
|
||||||
|
zkutil::makeCreateRequest("/a", "", zkutil::CreateMode::Persistent),
|
||||||
|
zkutil::makeCreateRequest("/a/bbbbbb", "", zkutil::CreateMode::Persistent),
|
||||||
|
zkutil::makeCreateRequest("/A", "", zkutil::CreateMode::Persistent),
|
||||||
|
zkutil::makeCreateRequest("/A/B", "", zkutil::CreateMode::Persistent),
|
||||||
|
zkutil::makeCreateRequest("/A/CCCCCCCCCCCC", "", zkutil::CreateMode::Persistent),
|
||||||
|
zkutil::makeRemoveRecursiveRequest("/A", 3),
|
||||||
|
};
|
||||||
|
auto remove_request = std::make_shared<ZooKeeperMultiRequest>(ops, ACLs{});
|
||||||
|
storage.preprocessRequest(remove_request, 1, 0, new_zxid);
|
||||||
|
auto remove_responses = storage.processRequest(remove_request, 1, new_zxid);
|
||||||
|
|
||||||
|
ASSERT_EQ(remove_responses.size(), 1);
|
||||||
|
ASSERT_TRUE(is_multi_ok(remove_responses[0].response));
|
||||||
|
ASSERT_TRUE(exists("/a"));
|
||||||
|
ASSERT_TRUE(exists("/a/bbbbbb"));
|
||||||
|
ASSERT_FALSE(exists("/A"));
|
||||||
|
ASSERT_FALSE(exists("/A/B"));
|
||||||
|
ASSERT_FALSE(exists("/A/CCCCCCCCCCCC"));
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
TYPED_TEST(CoordinationTest, TestRemoveRecursiveWatches)
|
TYPED_TEST(CoordinationTest, TestRemoveRecursiveWatches)
|
||||||
@ -3767,14 +3890,26 @@ TYPED_TEST(CoordinationTest, TestRemoveRecursiveWatches)
|
|||||||
auto responses = storage.processRequest(remove_request, 1, new_zxid);
|
auto responses = storage.processRequest(remove_request, 1, new_zxid);
|
||||||
|
|
||||||
ASSERT_EQ(responses.size(), 7);
|
ASSERT_EQ(responses.size(), 7);
|
||||||
|
/// request response is last
|
||||||
|
ASSERT_EQ(dynamic_cast<Coordination::ZooKeeperWatchResponse *>(responses.back().response.get()), nullptr);
|
||||||
|
|
||||||
for (size_t i = 0; i < 7; ++i)
|
std::unordered_map<std::string, std::vector<Coordination::Event>> expected_watch_responses
|
||||||
|
{
|
||||||
|
{"/A/B/D", {Coordination::Event::DELETED}},
|
||||||
|
{"/A/B", {Coordination::Event::CHILD, Coordination::Event::DELETED}},
|
||||||
|
{"/A/C", {Coordination::Event::DELETED}},
|
||||||
|
{"/A", {Coordination::Event::CHILD, Coordination::Event::DELETED}},
|
||||||
|
};
|
||||||
|
|
||||||
|
std::unordered_map<std::string, std::vector<Coordination::Event>> actual_watch_responses;
|
||||||
|
for (size_t i = 0; i < 6; ++i)
|
||||||
{
|
{
|
||||||
ASSERT_EQ(responses[i].response->error, Coordination::Error::ZOK);
|
ASSERT_EQ(responses[i].response->error, Coordination::Error::ZOK);
|
||||||
|
|
||||||
if (const auto * watch_response = dynamic_cast<Coordination::ZooKeeperWatchResponse *>(responses[i].response.get()))
|
const auto & watch_response = dynamic_cast<Coordination::ZooKeeperWatchResponse &>(*responses[i].response);
|
||||||
ASSERT_EQ(watch_response->type, Coordination::Event::DELETED);
|
actual_watch_responses[watch_response.path].push_back(static_cast<Coordination::Event>(watch_response.type));
|
||||||
}
|
}
|
||||||
|
ASSERT_EQ(expected_watch_responses, actual_watch_responses);
|
||||||
|
|
||||||
ASSERT_EQ(storage.watches.size(), 0);
|
ASSERT_EQ(storage.watches.size(), 0);
|
||||||
ASSERT_EQ(storage.list_watches.size(), 0);
|
ASSERT_EQ(storage.list_watches.size(), 0);
|
||||||
|
@ -151,6 +151,15 @@ Names NamesAndTypesList::getNames() const
|
|||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
NameSet NamesAndTypesList::getNameSet() const
|
||||||
|
{
|
||||||
|
NameSet res;
|
||||||
|
res.reserve(size());
|
||||||
|
for (const NameAndTypePair & column : *this)
|
||||||
|
res.insert(column.name);
|
||||||
|
return res;
|
||||||
|
}
|
||||||
|
|
||||||
DataTypes NamesAndTypesList::getTypes() const
|
DataTypes NamesAndTypesList::getTypes() const
|
||||||
{
|
{
|
||||||
DataTypes res;
|
DataTypes res;
|
||||||
|
@ -100,6 +100,7 @@ public:
|
|||||||
void getDifference(const NamesAndTypesList & rhs, NamesAndTypesList & deleted, NamesAndTypesList & added) const;
|
void getDifference(const NamesAndTypesList & rhs, NamesAndTypesList & deleted, NamesAndTypesList & added) const;
|
||||||
|
|
||||||
Names getNames() const;
|
Names getNames() const;
|
||||||
|
NameSet getNameSet() const;
|
||||||
DataTypes getTypes() const;
|
DataTypes getTypes() const;
|
||||||
|
|
||||||
/// Remove columns which names are not in the `names`.
|
/// Remove columns which names are not in the `names`.
|
||||||
|
@ -890,18 +890,21 @@ public:
|
|||||||
Messaging::MessageTransport & mt,
|
Messaging::MessageTransport & mt,
|
||||||
const Poco::Net::SocketAddress & address)
|
const Poco::Net::SocketAddress & address)
|
||||||
{
|
{
|
||||||
AuthenticationType user_auth_type;
|
|
||||||
try
|
try
|
||||||
{
|
{
|
||||||
user_auth_type = session.getAuthenticationTypeOrLogInFailure(user_name);
|
const auto user_authentication_types = session.getAuthenticationTypesOrLogInFailure(user_name);
|
||||||
if (type_to_method.find(user_auth_type) != type_to_method.end())
|
|
||||||
|
for (auto user_authentication_type : user_authentication_types)
|
||||||
{
|
{
|
||||||
type_to_method[user_auth_type]->authenticate(user_name, session, mt, address);
|
if (type_to_method.find(user_authentication_type) != type_to_method.end())
|
||||||
|
{
|
||||||
|
type_to_method[user_authentication_type]->authenticate(user_name, session, mt, address);
|
||||||
mt.send(Messaging::AuthenticationOk(), true);
|
mt.send(Messaging::AuthenticationOk(), true);
|
||||||
LOG_DEBUG(log, "Authentication for user {} was successful.", user_name);
|
LOG_DEBUG(log, "Authentication for user {} was successful.", user_name);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
catch (const Exception&)
|
catch (const Exception&)
|
||||||
{
|
{
|
||||||
mt.send(Messaging::ErrorOrNoticeResponse(Messaging::ErrorOrNoticeResponse::ERROR, "28P01", "Invalid user or password"),
|
mt.send(Messaging::ErrorOrNoticeResponse(Messaging::ErrorOrNoticeResponse::ERROR, "28P01", "Invalid user or password"),
|
||||||
@ -913,7 +916,7 @@ public:
|
|||||||
mt.send(Messaging::ErrorOrNoticeResponse(Messaging::ErrorOrNoticeResponse::ERROR, "0A000", "Authentication method is not supported"),
|
mt.send(Messaging::ErrorOrNoticeResponse(Messaging::ErrorOrNoticeResponse::ERROR, "0A000", "Authentication method is not supported"),
|
||||||
true);
|
true);
|
||||||
|
|
||||||
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Authentication method is not supported: {}", user_auth_type);
|
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "None of the authentication methods registered for the user are supported");
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
@ -33,7 +33,9 @@ static constexpr auto DBMS_MIN_REVISION_WITH_AGGREGATE_FUNCTIONS_VERSIONING = 54
|
|||||||
|
|
||||||
static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = 1;
|
static constexpr auto DBMS_CLUSTER_PROCESSING_PROTOCOL_VERSION = 1;
|
||||||
|
|
||||||
static constexpr auto DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3;
|
static constexpr auto DBMS_MIN_SUPPORTED_PARALLEL_REPLICAS_PROTOCOL_VERSION = 3;
|
||||||
|
static constexpr auto DBMS_PARALLEL_REPLICAS_MIN_VERSION_WITH_MARK_SEGMENT_SIZE_FIELD = 4;
|
||||||
|
static constexpr auto DBMS_PARALLEL_REPLICAS_PROTOCOL_VERSION = 4;
|
||||||
static constexpr auto DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453;
|
static constexpr auto DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS = 54453;
|
||||||
|
|
||||||
static constexpr auto DBMS_MERGE_TREE_PART_INFO_VERSION = 1;
|
static constexpr auto DBMS_MERGE_TREE_PART_INFO_VERSION = 1;
|
||||||
@ -86,6 +88,8 @@ static constexpr auto DBMS_MIN_REVISION_WITH_ROWS_BEFORE_AGGREGATION = 54469;
|
|||||||
/// Packets size header
|
/// Packets size header
|
||||||
static constexpr auto DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470;
|
static constexpr auto DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470;
|
||||||
|
|
||||||
|
static constexpr auto DBMS_MIN_REVISION_WITH_VERSIONED_PARALLEL_REPLICAS_PROTOCOL = 54471;
|
||||||
|
|
||||||
/// Version of ClickHouse TCP protocol.
|
/// Version of ClickHouse TCP protocol.
|
||||||
///
|
///
|
||||||
/// Should be incremented manually on protocol changes.
|
/// Should be incremented manually on protocol changes.
|
||||||
@ -93,6 +97,6 @@ static constexpr auto DBMS_MIN_PROTOCOL_VERSION_WITH_CHUNKED_PACKETS = 54470;
|
|||||||
/// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION,
|
/// NOTE: DBMS_TCP_PROTOCOL_VERSION has nothing common with VERSION_REVISION,
|
||||||
/// later is just a number for server version (one number instead of commit SHA)
|
/// later is just a number for server version (one number instead of commit SHA)
|
||||||
/// for simplicity (sometimes it may be more convenient in some use cases).
|
/// for simplicity (sometimes it may be more convenient in some use cases).
|
||||||
static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54470;
|
static constexpr auto DBMS_TCP_PROTOCOL_VERSION = 54471;
|
||||||
|
|
||||||
}
|
}
|
||||||
|
@ -119,6 +119,7 @@ namespace DB
|
|||||||
M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
M(UInt64, max_part_num_to_warn, 100000lu, "If the number of parts is greater than this value, the server will create a warning that will displayed to user.", 0) \
|
||||||
M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
|
M(UInt64, max_table_num_to_throw, 0lu, "If number of tables is greater than this value, server will throw an exception. 0 means no limitation. View, remote tables, dictionary, system tables are not counted. Only count table in Atomic/Ordinary/Replicated/Lazy database engine.", 0) \
|
||||||
M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \
|
M(UInt64, max_database_num_to_throw, 0lu, "If number of databases is greater than this value, server will throw an exception. 0 means no limitation.", 0) \
|
||||||
|
M(UInt64, max_authentication_methods_per_user, 100, "The maximum number of authentication methods a user can be created with or altered. Changing this setting does not affect existing users. Zero means unlimited", 0) \
|
||||||
M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \
|
M(UInt64, concurrent_threads_soft_limit_num, 0, "Sets how many concurrent thread can be allocated before applying CPU pressure. Zero means unlimited.", 0) \
|
||||||
M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \
|
M(UInt64, concurrent_threads_soft_limit_ratio_to_cores, 0, "Same as concurrent_threads_soft_limit_num, but with ratio to cores.", 0) \
|
||||||
\
|
\
|
||||||
|
@ -946,7 +946,7 @@ class IColumn;
|
|||||||
M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
|
M(Bool, parallel_replicas_for_non_replicated_merge_tree, false, "If true, ClickHouse will use parallel replicas algorithm also for non-replicated MergeTree tables", 0) \
|
||||||
M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \
|
M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \
|
||||||
M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \
|
M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \
|
||||||
M(UInt64, parallel_replicas_mark_segment_size, 128, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing", 0) \
|
M(UInt64, parallel_replicas_mark_segment_size, 0, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing. Value should be in range [128; 16384]", 0) \
|
||||||
M(Bool, allow_archive_path_syntax, true, "File/S3 engines/table function will parse paths with '::' as '<archive> :: <file>' if archive has correct extension", 0) \
|
M(Bool, allow_archive_path_syntax, true, "File/S3 engines/table function will parse paths with '::' as '<archive> :: <file>' if archive has correct extension", 0) \
|
||||||
M(Bool, parallel_replicas_local_plan, false, "Build local plan for local replica", 0) \
|
M(Bool, parallel_replicas_local_plan, false, "Build local plan for local replica", 0) \
|
||||||
\
|
\
|
||||||
@ -972,7 +972,6 @@ class IColumn;
|
|||||||
\
|
\
|
||||||
M(Bool, allow_experimental_database_materialized_mysql, false, "Allow to create database with Engine=MaterializedMySQL(...).", 0) \
|
M(Bool, allow_experimental_database_materialized_mysql, false, "Allow to create database with Engine=MaterializedMySQL(...).", 0) \
|
||||||
M(Bool, allow_experimental_database_materialized_postgresql, false, "Allow to create database with Engine=MaterializedPostgreSQL(...).", 0) \
|
M(Bool, allow_experimental_database_materialized_postgresql, false, "Allow to create database with Engine=MaterializedPostgreSQL(...).", 0) \
|
||||||
\
|
|
||||||
/** Experimental feature for moving data between shards. */ \
|
/** Experimental feature for moving data between shards. */ \
|
||||||
M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \
|
M(Bool, allow_experimental_query_deduplication, false, "Experimental data deduplication for SELECT queries based on part UUIDs", 0) \
|
||||||
|
|
||||||
@ -1272,6 +1271,7 @@ class IColumn;
|
|||||||
M(Bool, output_format_orc_string_as_string, true, "Use ORC String type instead of Binary for String columns", 0) \
|
M(Bool, output_format_orc_string_as_string, true, "Use ORC String type instead of Binary for String columns", 0) \
|
||||||
M(ORCCompression, output_format_orc_compression_method, "zstd", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \
|
M(ORCCompression, output_format_orc_compression_method, "zstd", "Compression method for ORC output format. Supported codecs: lz4, snappy, zlib, zstd, none (uncompressed)", 0) \
|
||||||
M(UInt64, output_format_orc_row_index_stride, 10'000, "Target row index stride in ORC output format", 0) \
|
M(UInt64, output_format_orc_row_index_stride, 10'000, "Target row index stride in ORC output format", 0) \
|
||||||
|
M(Double, output_format_orc_dictionary_key_size_threshold, 0.0, "For a string column in ORC output format, if the number of distinct values is greater than this fraction of the total number of non-null rows, turn off dictionary encoding. Otherwise dictionary encoding is enabled", 0) \
|
||||||
\
|
\
|
||||||
M(CapnProtoEnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::CapnProtoEnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \
|
M(CapnProtoEnumComparingMode, format_capn_proto_enum_comparising_mode, FormatSettings::CapnProtoEnumComparingMode::BY_VALUES, "How to map ClickHouse Enum and CapnProto Enum", 0) \
|
||||||
\
|
\
|
||||||
|
@ -71,6 +71,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
|
|||||||
},
|
},
|
||||||
{"24.9",
|
{"24.9",
|
||||||
{
|
{
|
||||||
|
{"output_format_orc_dictionary_key_size_threshold", 0.0, 0.0, "For a string column in ORC output format, if the number of distinct values is greater than this fraction of the total number of non-null rows, turn off dictionary encoding. Otherwise dictionary encoding is enabled"},
|
||||||
{"input_format_json_empty_as_default", false, false, "Added new setting to allow to treat empty fields in JSON input as default values."},
|
{"input_format_json_empty_as_default", false, false, "Added new setting to allow to treat empty fields in JSON input as default values."},
|
||||||
{"input_format_try_infer_variants", false, false, "Try to infer Variant type in text formats when there is more than one possible type for column/array elements"},
|
{"input_format_try_infer_variants", false, false, "Try to infer Variant type in text formats when there is more than one possible type for column/array elements"},
|
||||||
{"join_output_by_rowlist_perkey_rows_threshold", 0, 5, "The lower limit of per-key average rows in the right table to determine whether to output by row list in hash join."},
|
{"join_output_by_rowlist_perkey_rows_threshold", 0, 5, "The lower limit of per-key average rows in the right table to determine whether to output by row list in hash join."},
|
||||||
@ -78,6 +79,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
|
|||||||
{"allow_materialized_view_with_bad_select", true, true, "Support (but not enable yet) stricter validation in CREATE MATERIALIZED VIEW"},
|
{"allow_materialized_view_with_bad_select", true, true, "Support (but not enable yet) stricter validation in CREATE MATERIALIZED VIEW"},
|
||||||
{"output_format_always_quote_identifiers", false, false, "New setting."},
|
{"output_format_always_quote_identifiers", false, false, "New setting."},
|
||||||
{"output_format_identifier_quoting_style", "Backticks", "Backticks", "New setting."},
|
{"output_format_identifier_quoting_style", "Backticks", "Backticks", "New setting."},
|
||||||
|
{"parallel_replicas_mark_segment_size", 128, 0, "Value for this setting now determined automatically"},
|
||||||
{"database_replicated_allow_replicated_engine_arguments", 1, 0, "Don't allow explicit arguments by default"},
|
{"database_replicated_allow_replicated_engine_arguments", 1, 0, "Don't allow explicit arguments by default"},
|
||||||
{"database_replicated_allow_explicit_uuid", 0, 0, "Added a new setting to disallow explicitly specifying table UUID"},
|
{"database_replicated_allow_explicit_uuid", 0, 0, "Added a new setting to disallow explicitly specifying table UUID"},
|
||||||
{"parallel_replicas_local_plan", false, false, "Use local plan for local replica in a query with parallel replicas"},
|
{"parallel_replicas_local_plan", false, false, "Use local plan for local replica in a query with parallel replicas"},
|
||||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user