mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 23:21:59 +00:00
Compare commits
682 Commits
aa117869d6
...
7a74f0223d
Author | SHA1 | Date | |
---|---|---|---|
|
7a74f0223d | ||
|
44b4bd38b9 | ||
|
40c7d5fd1a | ||
|
4e56c026cd | ||
|
2e776256e8 | ||
|
2b3098c641 | ||
|
c6a10151d9 | ||
|
bccbb0a6b5 | ||
|
1aceb608f3 | ||
|
64ea850e5b | ||
|
f711005eaa | ||
|
999bff236c | ||
|
9120462ed4 | ||
|
3ae055c74e | ||
|
e11220beb3 | ||
|
f84083d174 | ||
|
59fe7e1951 | ||
|
4ccebd9a24 | ||
|
99177c0daf | ||
|
cf33249988 | ||
|
c1e556df84 | ||
|
546f225d10 | ||
|
e68011e8f5 | ||
|
a264de35d1 | ||
|
2d26a0cbce | ||
|
a3c969b44d | ||
|
ed4d955df6 | ||
|
0fd196771e | ||
|
17fdd2bd37 | ||
|
334b28f6db | ||
|
e83d531772 | ||
|
dae913c53b | ||
|
07be02d297 | ||
|
6446c11a7b | ||
|
1c414b9987 | ||
|
170da747d5 | ||
|
456a41ee42 | ||
|
dd90fbe13b | ||
|
5aeeec0f42 | ||
|
1c5d0c7f93 | ||
|
b77c9cdd7f | ||
|
4563e79688 | ||
|
9bf2c6f968 | ||
|
75f781c18e | ||
|
d35e230ef6 | ||
|
98f7681497 | ||
|
ecedbcc763 | ||
|
8551162dcb | ||
|
5bdb1dc8aa | ||
|
529721923f | ||
|
353ff951aa | ||
|
e0bf6ec7e4 | ||
|
bf180940cd | ||
|
a44f877343 | ||
|
67d81a377e | ||
|
adcee45ad2 | ||
|
bdb35d9b63 | ||
|
4db7468386 | ||
|
0ff7c280f9 | ||
|
b18fb82ed0 | ||
|
7eac602350 | ||
|
7f493c81d0 | ||
|
5286fa65c4 | ||
|
d1ed49d84b | ||
|
58dd039aac | ||
|
e6f4afe569 | ||
|
2146ab4e4e | ||
|
1243f2fb54 | ||
|
e3e4e45278 | ||
|
144fd4082b | ||
|
6865d1e383 | ||
|
c6f901adaa | ||
|
a1fb0ad706 | ||
|
0951991c1d | ||
|
19aec5e572 | ||
|
a367de9977 | ||
|
bd0634ce9d | ||
|
6894e280b2 | ||
|
567270afa8 | ||
|
317bdad1e9 | ||
|
40ad02f229 | ||
|
042f0acdb9 | ||
|
59f73a2053 | ||
|
e4f6553eab | ||
|
39ebe113d9 | ||
|
b4a6ee77ab | ||
|
05bd7f4b4c | ||
|
c9b843a6bb | ||
|
6e1de8b8a4 | ||
|
514c1f7215 | ||
|
9c4456df1a | ||
|
559d9ac517 | ||
|
e785bb908e | ||
|
6e58cfc5b8 | ||
|
c4fc7a1bac | ||
|
94e2a9cc43 | ||
|
9fadfb98b4 | ||
|
c5b5b5841b | ||
|
86eb3d6425 | ||
|
0f4990d2e7 | ||
|
7785a9b15e | ||
|
7bb68c0f8a | ||
|
e27fbc79d0 | ||
|
eb9bdd1fb0 | ||
|
dc2e91d0b5 | ||
|
54c69998c9 | ||
|
3386cbb009 | ||
|
47944a4d0e | ||
|
c1ea3d4213 | ||
|
36902c66a0 | ||
|
239bbaa133 | ||
|
07fac5808d | ||
|
cad22e7a2d | ||
|
ed95e0781f | ||
|
dc88b973e9 | ||
|
ed2af768a1 | ||
|
ff7e1333c2 | ||
|
b2fc69a6bb | ||
|
c917b9ed9a | ||
|
6aa2fe525e | ||
|
03c5e4b107 | ||
|
3259e6396f | ||
|
445a5e9c9e | ||
|
47bed13b42 | ||
|
196399bcad | ||
|
557b3e370d | ||
|
014608fb6b | ||
|
a29ded4941 | ||
|
d2efae7511 | ||
|
6b55754bc9 | ||
|
07807f1450 | ||
|
926e4a3c38 | ||
|
7564681621 | ||
|
e33f5bb4e9 | ||
|
a258b6d0f2 | ||
|
1c308f970b | ||
|
7b37bddcd7 | ||
|
ddc5c80601 | ||
|
ec776fe8db | ||
|
84828120b3 | ||
|
d8fb85117e | ||
|
9917dc66d4 | ||
|
85a5f91b29 | ||
|
fc688ccbec | ||
|
14248d322e | ||
|
9f59fb6a5d | ||
|
b25a01922c | ||
|
8bec9a1e34 | ||
|
d7f920d2cf | ||
|
fb4e1feb16 | ||
|
608aae85cf | ||
|
841337d6fb | ||
|
240c05b128 | ||
|
3a012e5a96 | ||
|
67d1eb84aa | ||
|
fb552dd2c0 | ||
|
e45dd36343 | ||
|
bdf6ab8d45 | ||
|
c998df6c8c | ||
|
05f638ea52 | ||
|
926f7ca7a2 | ||
|
e5b7ba7717 | ||
|
fe0d96f72a | ||
|
62000b22fc | ||
|
5a23d19c75 | ||
|
2d48406a82 | ||
|
5445f55e62 | ||
|
3e05ffe675 | ||
|
ba0f03fce9 | ||
|
9f42828800 | ||
|
b1e816f60c | ||
|
1e15312729 | ||
|
01c5762fc5 | ||
|
2d2d611bd3 | ||
|
aa23ab4b21 | ||
|
8e0dc599b8 | ||
|
b706458a23 | ||
|
f4e18669c4 | ||
|
d7cc5e892e | ||
|
57cefae6f2 | ||
|
c198e1cb1f | ||
|
e00128c717 | ||
|
a7831c991f | ||
|
742f1eb41e | ||
|
d0842ee937 | ||
|
6ba8afc443 | ||
|
3bdd4a5173 | ||
|
61b3231a72 | ||
|
0a9560adb5 | ||
|
fc77a01de8 | ||
|
082d639043 | ||
|
d61e2e1c28 | ||
|
f9431a3150 | ||
|
77d288a3e5 | ||
|
e9fff481fa | ||
|
63aacef8d5 | ||
|
ade2dadd01 | ||
|
7d5fc90b19 | ||
|
df0eec90e8 | ||
|
ab6e49b930 | ||
|
ada1af5718 | ||
|
7705f9b009 | ||
|
39ae3f4da5 | ||
|
208fd6efe1 | ||
|
3a2dca0193 | ||
|
1f7e2845be | ||
|
8bd95a40af | ||
|
59f2c44a52 | ||
|
183593a568 | ||
|
5ca9f2e570 | ||
|
ba757696eb | ||
|
c31f6968c0 | ||
|
9270deb83e | ||
|
571cf33136 | ||
|
936eb3a567 | ||
|
1aa3e47e89 | ||
|
f97a1150b9 | ||
|
a82ab36c08 | ||
|
864f35dd11 | ||
|
072e81e5e5 | ||
|
93403eb38e | ||
|
ec0bdcc1cb | ||
|
4831414476 | ||
|
91b62c56d2 | ||
|
9bd6f9c6b4 | ||
|
268b823010 | ||
|
f73d48c61b | ||
|
764dd82345 | ||
|
8d4cc787fe | ||
|
e58e6dd4bd | ||
|
bff84d44e8 | ||
|
ee7b53646f | ||
|
7cc4df487a | ||
|
e106ae1232 | ||
|
d5b3228b98 | ||
|
85a7a5fb2e | ||
|
66ff828fae | ||
|
49e05329c2 | ||
|
92754f80cc | ||
|
469733d327 | ||
|
358dd5da32 | ||
|
c81dccdcdf | ||
|
c2543d0316 | ||
|
252effe32c | ||
|
0b3945e48a | ||
|
0e24114b16 | ||
|
c5b12d3e16 | ||
|
bb4549eb08 | ||
|
e085f14aa6 | ||
|
c66f71c795 | ||
|
b3f4e3e5a5 | ||
|
373ce71157 | ||
|
0cae881171 | ||
|
540e9bc893 | ||
|
f29d7841fe | ||
|
da632e4d83 | ||
|
1fb1bfe6f9 | ||
|
ff609bc501 | ||
|
9c5017323d | ||
|
e70bfd8f3c | ||
|
8cfec23dbe | ||
|
3b0b42cbbb | ||
|
628d0d3fc9 | ||
|
96dc3cd55a | ||
|
4db481f7da | ||
|
f6366252f2 | ||
|
98cd45153d | ||
|
d55ebb19e0 | ||
|
3c0f299148 | ||
|
efabf19926 | ||
|
0bd6a1fb04 | ||
|
1c682f1316 | ||
|
d2c54bc140 | ||
|
26aaa87a91 | ||
|
e5f4ba8017 | ||
|
af8d7c0b60 | ||
|
1926f6ca4a | ||
|
5a04826fa5 | ||
|
7d3d940b08 | ||
|
6b1eae5899 | ||
|
8d99fb9e7c | ||
|
13abc806a2 | ||
|
f1b4e7ea59 | ||
|
8a79e1959f | ||
|
9cccea93d2 | ||
|
ad65cdab90 | ||
|
99e5e550da | ||
|
5ae6572cd6 | ||
|
21a39e6b50 | ||
|
140bd01427 | ||
|
941c0f0c3b | ||
|
6441c59607 | ||
|
cf90bfaf79 | ||
|
fde4eb43d5 | ||
|
005a3f7a36 | ||
|
43c2b69b6c | ||
|
5c8bf52954 | ||
|
5fafb44446 | ||
|
236b09989d | ||
|
b601541ea5 | ||
|
0305f04494 | ||
|
aeda0db12c | ||
|
3fb4bf41bc | ||
|
13d3f9171c | ||
|
1da07b6fe5 | ||
|
a9409299d1 | ||
|
61d0440f85 | ||
|
5b3b9fad2f | ||
|
8d306bfd75 | ||
|
57ddde47ea | ||
|
1ccd88e6df | ||
|
0f63a32d12 | ||
|
17327a242d | ||
|
d6b3f94a35 | ||
|
5a6beb7ad0 | ||
|
1dc969c560 | ||
|
0349eeed37 | ||
|
66a216f63c | ||
|
02de47c9ba | ||
|
1e1ea07268 | ||
|
932caea5f7 | ||
|
6879aa130a | ||
|
43f3c886a2 | ||
|
ac0f013fab | ||
|
94ce517dc1 | ||
|
8c3fe9e2c2 | ||
|
c10de3969c | ||
|
3145aeda84 | ||
|
33fb3b7099 | ||
|
b2dbbea179 | ||
|
436462ce98 | ||
|
05ed5f28a5 | ||
|
7d0a6bd1b8 | ||
|
c383a743f7 | ||
|
6bac632b39 | ||
|
b8731dc4f4 | ||
|
d34f11a833 | ||
|
6249861d16 | ||
|
e6d70cf0f2 | ||
|
9c22f3d89f | ||
|
a41cadb150 | ||
|
b7e7c48842 | ||
|
6f3e817be9 | ||
|
04e80e675a | ||
|
b702a0e2a6 | ||
|
3d086a637f | ||
|
dd246fae0e | ||
|
8b340c864a | ||
|
26767b7955 | ||
|
73fa0f93a5 | ||
|
eb384d489c | ||
|
41415197ab | ||
|
c29be90fd6 | ||
|
2aa5447f09 | ||
|
19bb9e0853 | ||
|
511045840b | ||
|
bf9ceed6ad | ||
|
32311c1db9 | ||
|
242c2d99ba | ||
|
51b7916baa | ||
|
bf188495bd | ||
|
5e4dd5d32a | ||
|
4ea106712f | ||
|
f7c46114f4 | ||
|
57ceeb4a63 | ||
|
864a8a63df | ||
|
cd97103526 | ||
|
cd269f82d3 | ||
|
f9a9407186 | ||
|
b758838c73 | ||
|
3bba64e975 | ||
|
c9cfc8afd6 | ||
|
6ad72f1b8f | ||
|
3dae5def48 | ||
|
7634e646e8 | ||
|
8a433a2222 | ||
|
4d8bb01a77 | ||
|
178bf89652 | ||
|
2507daa3b1 | ||
|
701fb76edb | ||
|
9fb1cf20c7 | ||
|
eea644dc45 | ||
|
286bcb030e | ||
|
373f00c127 | ||
|
a5fba9744e | ||
|
c5817d528c | ||
|
237a81ff8c | ||
|
06debdc479 | ||
|
aa51f4cc88 | ||
|
92c94861a5 | ||
|
c0e239a14a | ||
|
c7ad0b897a | ||
|
f60d35161f | ||
|
a5318f6084 | ||
|
a75e1bafff | ||
|
75cbf0ca9a | ||
|
b62aac446e | ||
|
5efbae615c | ||
|
1c69a275a6 | ||
|
b2933280b2 | ||
|
356393b488 | ||
|
69ac9751a0 | ||
|
9c4f89c136 | ||
|
2c59df765c | ||
|
9cb78214fa | ||
|
899702f056 | ||
|
39e01d47b1 | ||
|
b1d62e2d0c | ||
|
0f944781d5 | ||
|
4c4abd767c | ||
|
7bbdaac3e1 | ||
|
35a1a015ce | ||
|
0cd4ce01f4 | ||
|
29bade22ee | ||
|
d568e8107b | ||
|
59f1e3fa67 | ||
|
ce78ec602c | ||
|
475344231a | ||
|
86e3299135 | ||
|
167e8db5d1 | ||
|
6e55bb7618 | ||
|
5a62403513 | ||
|
762fe6f217 | ||
|
61f623e12c | ||
|
26ed6defc0 | ||
|
999aced7c9 | ||
|
e2ea19f37a | ||
|
16a93ac1ce | ||
|
2e4cb23039 | ||
|
e6b071f4ae | ||
|
e11890818a | ||
|
b562966ef1 | ||
|
22c97480c2 | ||
|
4b46065bcd | ||
|
a115e468bd | ||
|
fe2aa961b6 | ||
|
392676b89a | ||
|
6297856d16 | ||
|
bb0b573193 | ||
|
1e49313ed6 | ||
|
ee57c9f804 | ||
|
2280aa5e30 | ||
|
7dab7ae9a4 | ||
|
5812029467 | ||
|
4513bd017f | ||
|
1975c5267f | ||
|
5274d951f8 | ||
|
5303391514 | ||
|
68344e723a | ||
|
19b7b97418 | ||
|
cea797e254 | ||
|
e0f083ac0b | ||
|
8782cf6885 | ||
|
35bd341389 | ||
|
f1ab911910 | ||
|
0768e0b265 | ||
|
ab79efe40f | ||
|
9baa5911f9 | ||
|
d57a6b1e74 | ||
|
db30c33d04 | ||
|
7156499001 | ||
|
0a79fd0c40 | ||
|
64244250c2 | ||
|
56f6062bd8 | ||
|
01ca2b6947 | ||
|
5258bb6d01 | ||
|
19e6274a40 | ||
|
37c2483869 | ||
|
1561a0115f | ||
|
467c961461 | ||
|
a8d07555d4 | ||
|
a828e3e923 | ||
|
aaa46a95c2 | ||
|
3332bce1dc | ||
|
acafa37e2d | ||
|
3c37c1e6c4 | ||
|
b97d78e7f4 | ||
|
a6b08187b3 | ||
|
a3bfb57da1 | ||
|
c7e2ae208d | ||
|
27fc62ae6a | ||
|
b3d50d096f | ||
|
146f3b5bef | ||
|
9ac9dea447 | ||
|
25f73dfb55 | ||
|
1c74206bf2 | ||
|
174550e1bf | ||
|
70a01f45ad | ||
|
c1345d38c8 | ||
|
699b9d4026 | ||
|
9c1be76c14 | ||
|
88cf6efb43 | ||
|
a237cc82fb | ||
|
98ee089331 | ||
|
19fe60157f | ||
|
bd7df8292c | ||
|
8e1d85b0b8 | ||
|
d5b1b81197 | ||
|
5b5652a2a6 | ||
|
e198b20509 | ||
|
3d78c499a4 | ||
|
5fbced454d | ||
|
8c5ab63345 | ||
|
a6b55563c7 | ||
|
361031f9a3 | ||
|
935a29485c | ||
|
c6384291e1 | ||
|
2892aa11e5 | ||
|
755e02f905 | ||
|
8af8488e46 | ||
|
44130d6765 | ||
|
74fd7fa01d | ||
|
4e8a96e9c1 | ||
|
2530fd233f | ||
|
ed2da98474 | ||
|
1e3f08ab3e | ||
|
6d5c707d2c | ||
|
60fa4be570 | ||
|
38a3c67075 | ||
|
ac0902b088 | ||
|
31f7615088 | ||
|
77bd0b2903 | ||
|
752dfead2c | ||
|
fce3a0463b | ||
|
c34de3cda8 | ||
|
b04b332d87 | ||
|
e1331977b5 | ||
|
0808d7f0fb | ||
|
8245e3d7ef | ||
|
b724f2c331 | ||
|
6004cb8ff4 | ||
|
838b8cb7a9 | ||
|
6d47ef8f22 | ||
|
bebef8d0d9 | ||
|
52d9b205ea | ||
|
66f750ea6f | ||
|
0fda9bf238 | ||
|
05c9ba1215 | ||
|
c7fce84729 | ||
|
8aefca5f1c | ||
|
55ba1b96c2 | ||
|
a71481de8f | ||
|
8e9c8a6670 | ||
|
e2c2e67c7b | ||
|
fa78f2db20 | ||
|
9616167325 | ||
|
3607493881 | ||
|
dc976c48d2 | ||
|
b03a296542 | ||
|
aeffae571c | ||
|
c952d9d815 | ||
|
5f93afede5 | ||
|
292800eb0a | ||
|
d1cbc4d86b | ||
|
54b9395384 | ||
|
084f878fb1 | ||
|
349af95cd1 | ||
|
04099b1f9d | ||
|
2bbf1de494 | ||
|
c2abeca568 | ||
|
d848a3b2bb | ||
|
b04e7abd2e | ||
|
69e9dd9b3d | ||
|
d8f2540b3c | ||
|
eccf5dd15e | ||
|
d4e440adb8 | ||
|
dd42467250 | ||
|
88006f6e09 | ||
|
0e702fc56d | ||
|
881f1a94ae | ||
|
e8fdacdece | ||
|
01e6b6b300 | ||
|
a5b9083f2c | ||
|
017d9557c5 | ||
|
f238530cc5 | ||
|
b09d3c5479 | ||
|
db2aab199d | ||
|
d10b79020e | ||
|
889aeff133 | ||
|
9875a50754 | ||
|
68ade3c621 | ||
|
a957874f72 | ||
|
da156add60 | ||
|
5cf92fe964 | ||
|
db0f6fd07b | ||
|
e138917890 | ||
|
06a34b0eaa | ||
|
5f63725bd9 | ||
|
a9b19b5a6e | ||
|
dd51d8fe64 | ||
|
103e31b07c | ||
|
bffbe2f866 | ||
|
f599c06b55 | ||
|
6ce70a59ef | ||
|
ceeb522b37 | ||
|
16ddf20c73 | ||
|
1f7be09d70 | ||
|
e5229770ee | ||
|
de6f2b0f91 | ||
|
f41c60ddd7 | ||
|
9e446361bd | ||
|
61f2a2d78b | ||
|
636a5ccff5 | ||
|
639560bd2d | ||
|
a5d977bfc1 | ||
|
26cab169aa | ||
|
a7d693573b | ||
|
75dc880c2e | ||
|
c29008528f | ||
|
bae1aef42e | ||
|
394317b917 | ||
|
b7104ca301 | ||
|
47e0be3e7e | ||
|
b724f49deb | ||
|
9f637cb767 | ||
|
4cff924fbf | ||
|
303d16c759 | ||
|
de9fa4d27b | ||
|
11dda21a19 | ||
|
386f7045a6 | ||
|
b50b93ee86 | ||
|
eecc09394e | ||
|
cd44ff7b9e | ||
|
f62c7012d7 | ||
|
97c86b3237 | ||
|
26743d9d08 | ||
|
ae6ff4cb68 | ||
|
42f67904d6 | ||
|
9592847587 | ||
|
3f5f59242e | ||
|
6563f0da78 | ||
|
08bfabc754 | ||
|
4f3ff0ecbd | ||
|
9c55ad799b | ||
|
e3caa59f83 | ||
|
1c6eafbfcf | ||
|
532bda8334 | ||
|
101eeae888 | ||
|
7596d5a0ca | ||
|
abec55e1a8 | ||
|
309ee4351f | ||
|
662c67dcc0 | ||
|
eb8af558f8 | ||
|
5b187b62f0 | ||
|
e8585a3740 | ||
|
fec6d366e5 | ||
|
647e956c95 | ||
|
6b021c6896 | ||
|
7f69df63fd | ||
|
01179a1eb4 | ||
|
cc379ce941 | ||
|
3ca66293ca | ||
|
80633baab2 | ||
|
18622685fa | ||
|
f1dd46e940 | ||
|
59bd7447fc | ||
|
ffd1a6c3a5 | ||
|
33af77cda7 | ||
|
220d32039c | ||
|
52d40350f6 | ||
|
83b79fce83 | ||
|
2fb3ec7abe | ||
|
6a099996d2 | ||
|
a4ec9d074f | ||
|
2850f7aaa3 | ||
|
54dd6aa7ee | ||
|
874116a107 | ||
|
1d78d8b3ee | ||
|
b3483d78b6 | ||
|
83d998edf3 | ||
|
fc2f7001ca | ||
|
3cef295711 | ||
|
b1e7853e38 | ||
|
2e4d31270e | ||
|
89004bd04d | ||
|
edbfb2dccc | ||
|
eb208333e0 | ||
|
e498e766b1 | ||
|
ef1e64a530 | ||
|
2ffbb7cf23 | ||
|
9715292cf6 | ||
|
2febfb4173 |
7
.github/workflows/merge_queue.yml
vendored
7
.github/workflows/merge_queue.yml
vendored
@ -58,13 +58,8 @@ jobs:
|
||||
test_name: Style check
|
||||
runner_type: style-checker-aarch64
|
||||
run_command: |
|
||||
python3 style_check.py
|
||||
python3 style_check.py --no-push
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
secrets:
|
||||
secret_envs: |
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
|
||||
RCSK
|
||||
FastTest:
|
||||
needs: [RunConfig, BuildDockers]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
|
||||
|
34
.github/workflows/nightly.yml
vendored
34
.github/workflows/nightly.yml
vendored
@ -27,7 +27,7 @@ jobs:
|
||||
id: runconfig
|
||||
run: |
|
||||
echo "::group::configure CI run"
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow NightlyBuilds --outfile ${{ runner.temp }}/ci_run_data.json
|
||||
echo "::endgroup::"
|
||||
|
||||
echo "::group::CI run configure results"
|
||||
@ -44,9 +44,39 @@ jobs:
|
||||
with:
|
||||
data: "${{ needs.RunConfig.outputs.data }}"
|
||||
set_latest: true
|
||||
|
||||
Builds_1:
|
||||
needs: [RunConfig]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
|
||||
uses: ./.github/workflows/reusable_build_stage.yml
|
||||
with:
|
||||
stage: Builds_1
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Tests_1:
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
stage: Tests_1
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Builds_2:
|
||||
needs: [RunConfig, Builds_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
|
||||
uses: ./.github/workflows/reusable_build_stage.yml
|
||||
with:
|
||||
stage: Builds_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
Tests_2:
|
||||
needs: [RunConfig, Builds_1, Tests_1]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
|
||||
uses: ./.github/workflows/reusable_test_stage.yml
|
||||
with:
|
||||
stage: Tests_2
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
|
||||
CheckWorkflow:
|
||||
if: ${{ !cancelled() }}
|
||||
needs: [RunConfig, BuildDockers]
|
||||
needs: [RunConfig, BuildDockers, Tests_2]
|
||||
runs-on: [self-hosted, style-checker-aarch64]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
|
5
.github/workflows/pull_request.yml
vendored
5
.github/workflows/pull_request.yml
vendored
@ -79,10 +79,7 @@ jobs:
|
||||
python3 style_check.py
|
||||
data: ${{ needs.RunConfig.outputs.data }}
|
||||
secrets:
|
||||
secret_envs: |
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
|
||||
RCSK
|
||||
robot_git_token: ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
|
||||
FastTest:
|
||||
needs: [RunConfig, BuildDockers, StyleCheck]
|
||||
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}
|
||||
|
17
.github/workflows/reusable_build.yml
vendored
17
.github/workflows/reusable_build.yml
vendored
@ -34,8 +34,11 @@ name: Build ClickHouse
|
||||
description: additional ENV variables to setup the job
|
||||
type: string
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
@ -58,10 +61,18 @@ jobs:
|
||||
run: |
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
${{inputs.additional_envs}}
|
||||
${{secrets.secret_envs}}
|
||||
DOCKER_TAG<<DOCKER_JSON
|
||||
${{ toJson(fromJson(inputs.data).docker_data.images) }}
|
||||
DOCKER_JSON
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.robot_git_token}}
|
||||
RCSK
|
||||
CI_DB_URL<<CIDBU
|
||||
${{ secrets.ci_db_url }}
|
||||
CIDBU
|
||||
CI_DB_PASSWORD<<CIDBP
|
||||
${{ secrets.ci_db_password }}
|
||||
CIDBP
|
||||
EOF
|
||||
python3 "$GITHUB_WORKSPACE"/tests/ci/ci_config.py --build-name "${{inputs.build_name}}" >> "$GITHUB_ENV"
|
||||
- name: Apply sparse checkout for contrib # in order to check that it doesn't break build
|
||||
|
11
.github/workflows/reusable_build_stage.yml
vendored
11
.github/workflows/reusable_build_stage.yml
vendored
@ -18,8 +18,11 @@ name: BuildStageWF
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
@ -39,4 +42,6 @@ jobs:
|
||||
checkout_depth: 0
|
||||
data: ${{ inputs.data }}
|
||||
secrets:
|
||||
secret_envs: ${{ secrets.secret_envs }}
|
||||
robot_git_token: ${{ secrets.robot_git_token }}
|
||||
ci_db_url: ${{ secrets.ci_db_url }}
|
||||
ci_db_password: ${{ secrets.ci_db_password }}
|
||||
|
17
.github/workflows/reusable_simple_job.yml
vendored
17
.github/workflows/reusable_simple_job.yml
vendored
@ -45,8 +45,11 @@ name: Simple job
|
||||
type: boolean
|
||||
default: false
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
|
||||
@ -77,7 +80,15 @@ jobs:
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
CHECK_NAME=${{ inputs.test_name }}
|
||||
${{inputs.additional_envs}}
|
||||
${{secrets.secret_envs}}
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.robot_git_token}}
|
||||
RCSK
|
||||
CI_DB_URL<<CIDBU
|
||||
${{ secrets.ci_db_url }}
|
||||
CIDBU
|
||||
CI_DB_PASSWORD<<CIDBP
|
||||
${{ secrets.ci_db_password }}
|
||||
CIDBP
|
||||
EOF
|
||||
- name: Common setup
|
||||
uses: ./.github/actions/common_setup
|
||||
|
17
.github/workflows/reusable_test.yml
vendored
17
.github/workflows/reusable_test.yml
vendored
@ -40,8 +40,11 @@ name: Testing workflow
|
||||
type: string
|
||||
default: "$GITHUB_WORKSPACE/tests/ci"
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
|
||||
@ -75,10 +78,18 @@ jobs:
|
||||
cat >> "$GITHUB_ENV" << 'EOF'
|
||||
CHECK_NAME=${{ inputs.test_name }}
|
||||
${{inputs.additional_envs}}
|
||||
${{secrets.secret_envs}}
|
||||
DOCKER_TAG<<DOCKER_JSON
|
||||
${{ toJson(fromJson(inputs.data).docker_data.images) }}
|
||||
DOCKER_JSON
|
||||
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
|
||||
${{secrets.robot_git_token}}
|
||||
RCSK
|
||||
CI_DB_URL<<CIDBU
|
||||
${{ secrets.ci_db_url }}
|
||||
CIDBU
|
||||
CI_DB_PASSWORD<<CIDBP
|
||||
${{ secrets.ci_db_password }}
|
||||
CIDBP
|
||||
EOF
|
||||
- name: Common setup
|
||||
uses: ./.github/actions/common_setup
|
||||
|
11
.github/workflows/reusable_test_stage.yml
vendored
11
.github/workflows/reusable_test_stage.yml
vendored
@ -15,8 +15,11 @@ name: StageWF
|
||||
type: string
|
||||
required: true
|
||||
secrets:
|
||||
secret_envs:
|
||||
description: if given, it's passed to the environments
|
||||
robot_git_token:
|
||||
required: false
|
||||
ci_db_url:
|
||||
required: false
|
||||
ci_db_password:
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
@ -32,4 +35,6 @@ jobs:
|
||||
runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
|
||||
data: ${{ inputs.data }}
|
||||
secrets:
|
||||
secret_envs: ${{ secrets.secret_envs }}
|
||||
robot_git_token: ${{ secrets.robot_git_token }}
|
||||
ci_db_url: ${{ secrets.ci_db_url }}
|
||||
ci_db_password: ${{ secrets.ci_db_password }}
|
||||
|
6
.gitmodules
vendored
6
.gitmodules
vendored
@ -1,6 +1,9 @@
|
||||
# Please do not use 'branch = ...' tags with submodule entries. Such tags make updating submodules a
|
||||
# little bit more convenient but they do *not* specify the tracked submodule branch. Thus, they are
|
||||
# more confusing than useful.
|
||||
[submodule "contrib/jwt-cpp"]
|
||||
path = contrib/jwt-cpp
|
||||
url = https://github.com/Thalhammer/jwt-cpp
|
||||
[submodule "contrib/zstd"]
|
||||
path = contrib/zstd
|
||||
url = https://github.com/facebook/zstd
|
||||
@ -348,6 +351,9 @@
|
||||
[submodule "contrib/idna"]
|
||||
path = contrib/idna
|
||||
url = https://github.com/ada-url/idna.git
|
||||
[submodule "contrib/google-cloud-cpp"]
|
||||
path = contrib/google-cloud-cpp
|
||||
url = https://github.com/ClickHouse/google-cloud-cpp.git
|
||||
[submodule "contrib/rust_vendor"]
|
||||
path = contrib/rust_vendor
|
||||
url = https://github.com/ClickHouse/rust_vendor.git
|
||||
|
@ -27,6 +27,7 @@ curl https://clickhouse.com/ | sh
|
||||
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
|
||||
* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
|
||||
* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
|
||||
* [Bluesky](https://bsky.app/profile/clickhouse.com) and [X](https://x.com/ClickHouseDB) for short news.
|
||||
* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
|
||||
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
|
||||
|
||||
@ -43,12 +44,13 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else
|
||||
Upcoming meetups
|
||||
|
||||
* [Ghent Meetup](https://www.meetup.com/clickhouse-belgium-user-group/events/303049405/) - November 19
|
||||
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21
|
||||
* [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26
|
||||
* [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3
|
||||
* [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/304382411) - December 9
|
||||
* [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/304268174) - December 9
|
||||
* [Kuala Lampur Meetup](https://www.meetup.com/clickhouse-malaysia-meetup-group/events/304576472/) - December 11
|
||||
* [San Francisco Meetup](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/304286951/) - December 12
|
||||
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - Feb 3
|
||||
|
||||
Recently completed meetups
|
||||
|
||||
|
@ -43,7 +43,7 @@ namespace Net
|
||||
/// Sets the following default values:
|
||||
/// - timeout: 60 seconds
|
||||
/// - keepAlive: true
|
||||
/// - maxKeepAliveRequests: 0
|
||||
/// - maxKeepAliveRequests: 100
|
||||
/// - keepAliveTimeout: 15 seconds
|
||||
|
||||
void setServerName(const std::string & serverName);
|
||||
@ -87,12 +87,12 @@ namespace Net
|
||||
const Poco::Timespan & getKeepAliveTimeout() const;
|
||||
/// Returns the connection timeout for HTTP connections.
|
||||
|
||||
void setMaxKeepAliveRequests(int maxKeepAliveRequests);
|
||||
void setMaxKeepAliveRequests(size_t maxKeepAliveRequests);
|
||||
/// Specifies the maximum number of requests allowed
|
||||
/// during a persistent connection. 0 means unlimited
|
||||
/// connections.
|
||||
|
||||
int getMaxKeepAliveRequests() const;
|
||||
size_t getMaxKeepAliveRequests() const;
|
||||
/// Returns the maximum number of requests allowed
|
||||
/// during a persistent connection, or 0 if
|
||||
/// unlimited connections are allowed.
|
||||
@ -106,7 +106,7 @@ namespace Net
|
||||
std::string _softwareVersion;
|
||||
Poco::Timespan _timeout;
|
||||
bool _keepAlive;
|
||||
int _maxKeepAliveRequests;
|
||||
size_t _maxKeepAliveRequests;
|
||||
Poco::Timespan _keepAliveTimeout;
|
||||
};
|
||||
|
||||
@ -138,7 +138,7 @@ namespace Net
|
||||
}
|
||||
|
||||
|
||||
inline int HTTPServerParams::getMaxKeepAliveRequests() const
|
||||
inline size_t HTTPServerParams::getMaxKeepAliveRequests() const
|
||||
{
|
||||
return _maxKeepAliveRequests;
|
||||
}
|
||||
|
@ -65,7 +65,7 @@ namespace Net
|
||||
private:
|
||||
bool _firstRequest;
|
||||
Poco::Timespan _keepAliveTimeout;
|
||||
int _maxKeepAliveRequests;
|
||||
size_t _maxKeepAliveRequests;
|
||||
};
|
||||
|
||||
|
||||
@ -74,7 +74,7 @@ namespace Net
|
||||
//
|
||||
inline bool HTTPServerSession::canKeepAlive() const
|
||||
{
|
||||
return _maxKeepAliveRequests != 0;
|
||||
return getKeepAlive() && _maxKeepAliveRequests > 0;
|
||||
}
|
||||
|
||||
|
||||
|
@ -22,7 +22,7 @@ namespace Net {
|
||||
HTTPServerParams::HTTPServerParams():
|
||||
_timeout(60000000),
|
||||
_keepAlive(true),
|
||||
_maxKeepAliveRequests(0),
|
||||
_maxKeepAliveRequests(100),
|
||||
_keepAliveTimeout(15000000)
|
||||
{
|
||||
}
|
||||
@ -63,7 +63,7 @@ void HTTPServerParams::setKeepAliveTimeout(const Poco::Timespan& timeout)
|
||||
}
|
||||
|
||||
|
||||
void HTTPServerParams::setMaxKeepAliveRequests(int maxKeepAliveRequests)
|
||||
void HTTPServerParams::setMaxKeepAliveRequests(size_t maxKeepAliveRequests)
|
||||
{
|
||||
poco_assert (maxKeepAliveRequests >= 0);
|
||||
_maxKeepAliveRequests = maxKeepAliveRequests;
|
||||
|
@ -50,7 +50,7 @@ bool HTTPServerSession::hasMoreRequests()
|
||||
--_maxKeepAliveRequests;
|
||||
return socket().poll(getTimeout(), Socket::SELECT_READ);
|
||||
}
|
||||
else if (_maxKeepAliveRequests != 0 && getKeepAlive())
|
||||
else if (canKeepAlive())
|
||||
{
|
||||
if (_maxKeepAliveRequests > 0)
|
||||
--_maxKeepAliveRequests;
|
||||
|
@ -18,7 +18,6 @@
|
||||
|
||||
|
||||
using Poco::Exception;
|
||||
using Poco::ErrorHandler;
|
||||
|
||||
|
||||
namespace Poco {
|
||||
@ -31,9 +30,7 @@ TCPServerConnection::TCPServerConnection(const StreamSocket& socket):
|
||||
}
|
||||
|
||||
|
||||
TCPServerConnection::~TCPServerConnection()
|
||||
{
|
||||
}
|
||||
TCPServerConnection::~TCPServerConnection() = default;
|
||||
|
||||
|
||||
void TCPServerConnection::start()
|
||||
|
0
ci/__init__.py
Normal file
0
ci/__init__.py
Normal file
14
ci/docker/stateful-test/Dockerfile
Normal file
14
ci/docker/stateful-test/Dockerfile
Normal file
@ -0,0 +1,14 @@
|
||||
ARG FROM_TAG=latest
|
||||
FROM clickhouse/stateless-test:$FROM_TAG
|
||||
|
||||
USER root
|
||||
|
||||
RUN apt-get update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
nodejs \
|
||||
npm \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \
|
||||
|
||||
USER clickhouse
|
117
ci/docker/stateless-test/Dockerfile
Normal file
117
ci/docker/stateless-test/Dockerfile
Normal file
@ -0,0 +1,117 @@
|
||||
# docker build -t clickhouse/stateless-test .
|
||||
FROM ubuntu:22.04
|
||||
|
||||
# ARG for quick switch to a given ubuntu mirror
|
||||
ARG apt_archive="http://archive.ubuntu.com"
|
||||
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
|
||||
|
||||
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
|
||||
|
||||
|
||||
RUN mkdir /etc/clickhouse-server /etc/clickhouse-keeper /etc/clickhouse-client && chmod 777 /etc/clickhouse-* \
|
||||
&& mkdir -p /var/lib/clickhouse /var/log/clickhouse-server && chmod 777 /var/log/clickhouse-server /var/lib/clickhouse
|
||||
|
||||
RUN addgroup --gid 1001 clickhouse && adduser --uid 1001 --gid 1001 --disabled-password clickhouse
|
||||
|
||||
# moreutils - provides ts fo FT
|
||||
# expect, bzip2 - requried by FT
|
||||
# bsdmainutils - provides hexdump for FT
|
||||
|
||||
# golang version 1.13 on Ubuntu 20 is enough for tests
|
||||
RUN apt-get update -y \
|
||||
&& env DEBIAN_FRONTEND=noninteractive \
|
||||
apt-get install --yes --no-install-recommends \
|
||||
awscli \
|
||||
brotli \
|
||||
lz4 \
|
||||
expect \
|
||||
moreutils \
|
||||
bzip2 \
|
||||
bsdmainutils \
|
||||
golang \
|
||||
lsof \
|
||||
mysql-client=8.0* \
|
||||
ncdu \
|
||||
netcat-openbsd \
|
||||
nodejs \
|
||||
npm \
|
||||
odbcinst \
|
||||
openjdk-11-jre-headless \
|
||||
openssl \
|
||||
postgresql-client \
|
||||
python3 \
|
||||
python3-pip \
|
||||
qemu-user-static \
|
||||
sqlite3 \
|
||||
sudo \
|
||||
tree \
|
||||
unixodbc \
|
||||
rustc \
|
||||
cargo \
|
||||
zstd \
|
||||
file \
|
||||
jq \
|
||||
pv \
|
||||
zip \
|
||||
unzip \
|
||||
p7zip-full \
|
||||
curl \
|
||||
wget \
|
||||
xz-utils \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
|
||||
|
||||
ARG PROTOC_VERSION=25.1
|
||||
RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip \
|
||||
&& unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local \
|
||||
&& rm protoc-${PROTOC_VERSION}-linux-x86_64.zip
|
||||
|
||||
COPY requirements.txt /
|
||||
RUN pip3 install --no-cache-dir -r /requirements.txt
|
||||
|
||||
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
|
||||
&& cd /tmp/clickhouse-odbc-tmp \
|
||||
&& curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \
|
||||
&& mkdir /usr/local/lib64 -p \
|
||||
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \
|
||||
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
|
||||
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
|
||||
&& sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \
|
||||
&& rm -rf /tmp/clickhouse-odbc-tmp
|
||||
|
||||
ENV TZ=Europe/Amsterdam
|
||||
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
|
||||
|
||||
ENV NUM_TRIES=1
|
||||
|
||||
# Unrelated to vars in setup_minio.sh, but should be the same there
|
||||
# to have the same binaries for local running scenario
|
||||
ARG MINIO_SERVER_VERSION=2024-08-03T04-33-23Z
|
||||
ARG MINIO_CLIENT_VERSION=2024-07-31T15-58-33Z
|
||||
ARG TARGETARCH
|
||||
|
||||
# Download Minio-related binaries
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
&& curl -L "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -o /minio \
|
||||
&& curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o /mc \
|
||||
&& chmod +x /mc /minio
|
||||
|
||||
ENV MINIO_ROOT_USER="clickhouse"
|
||||
ENV MINIO_ROOT_PASSWORD="clickhouse"
|
||||
|
||||
# for minio to work without root
|
||||
RUN chmod 777 /home
|
||||
ENV HOME="/home"
|
||||
ENV TEMP_DIR="/tmp/praktika"
|
||||
ENV PATH="/wd/tests:/tmp/praktika/input:$PATH"
|
||||
|
||||
RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
|
||||
&& tar -xvf hadoop-3.3.1.tar.gz \
|
||||
&& rm -rf hadoop-3.3.1.tar.gz \
|
||||
&& chmod 777 /hadoop-3.3.1
|
||||
|
||||
|
||||
RUN npm install -g azurite@3.30.0 \
|
||||
&& npm install -g tslib && npm install -g node
|
||||
|
||||
USER clickhouse
|
6
ci/docker/stateless-test/requirements.txt
Normal file
6
ci/docker/stateless-test/requirements.txt
Normal file
@ -0,0 +1,6 @@
|
||||
Jinja2==3.1.3
|
||||
numpy==1.26.4
|
||||
requests==2.32.3
|
||||
pandas==1.5.3
|
||||
scipy==1.12.0
|
||||
pyarrow==18.0.0
|
0
ci/jobs/__init__.py
Normal file
0
ci/jobs/__init__.py
Normal file
@ -13,11 +13,30 @@ class JobStages(metaclass=MetaClasses.WithIter):
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="ClickHouse Build Job")
|
||||
parser.add_argument("BUILD_TYPE", help="Type: <amd|arm_debug|release_sanitizer>")
|
||||
parser.add_argument("--param", help="Optional custom job start stage", default=None)
|
||||
parser.add_argument(
|
||||
"--build-type",
|
||||
help="Type: <amd|arm>,<debug|release>,<asan|msan|..>",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--param",
|
||||
help="Optional user-defined job start stage (for local run)",
|
||||
default=None,
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
CMAKE_CMD = """cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA \
|
||||
-DCMAKE_BUILD_TYPE={BUILD_TYPE} \
|
||||
-DSANITIZE={SANITIZER} \
|
||||
-DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 \
|
||||
-DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr \
|
||||
-DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
|
||||
{AUX_DEFS} \
|
||||
-DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 \
|
||||
-DCOMPILER_CACHE={CACHE_TYPE} \
|
||||
-DENABLE_BUILD_PROFILING=1 {DIR}"""
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_args()
|
||||
@ -33,23 +52,41 @@ def main():
|
||||
stages.pop(0)
|
||||
stages.insert(0, stage)
|
||||
|
||||
cmake_build_type = "Release"
|
||||
sanitizer = ""
|
||||
build_type = args.build_type
|
||||
assert (
|
||||
build_type
|
||||
), "build_type must be provided either as input argument or as a parameter of parametrized job in CI"
|
||||
build_type = build_type.lower()
|
||||
|
||||
if "debug" in args.BUILD_TYPE.lower():
|
||||
CACHE_TYPE = "sccache"
|
||||
|
||||
BUILD_TYPE = "RelWithDebInfo"
|
||||
SANITIZER = ""
|
||||
AUX_DEFS = " -DENABLE_TESTS=0 "
|
||||
|
||||
if "debug" in build_type:
|
||||
print("Build type set: debug")
|
||||
cmake_build_type = "Debug"
|
||||
|
||||
if "asan" in args.BUILD_TYPE.lower():
|
||||
BUILD_TYPE = "Debug"
|
||||
AUX_DEFS = " -DENABLE_TESTS=1 "
|
||||
elif "release" in build_type:
|
||||
print("Build type set: release")
|
||||
AUX_DEFS = (
|
||||
" -DENABLE_TESTS=0 -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1 "
|
||||
)
|
||||
elif "asan" in build_type:
|
||||
print("Sanitizer set: address")
|
||||
sanitizer = "address"
|
||||
SANITIZER = "address"
|
||||
else:
|
||||
assert False
|
||||
|
||||
# if Environment.is_local_run():
|
||||
# build_cache_type = "disabled"
|
||||
# else:
|
||||
build_cache_type = "sccache"
|
||||
cmake_cmd = CMAKE_CMD.format(
|
||||
BUILD_TYPE=BUILD_TYPE,
|
||||
CACHE_TYPE=CACHE_TYPE,
|
||||
SANITIZER=SANITIZER,
|
||||
AUX_DEFS=AUX_DEFS,
|
||||
DIR=Utils.cwd(),
|
||||
)
|
||||
|
||||
current_directory = Utils.cwd()
|
||||
build_dir = f"{Settings.TEMP_DIR}/build"
|
||||
|
||||
res = True
|
||||
@ -69,12 +106,7 @@ def main():
|
||||
results.append(
|
||||
Result.create_from_command_execution(
|
||||
name="Cmake configuration",
|
||||
command=f"cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE={cmake_build_type} \
|
||||
-DSANITIZE={sanitizer} -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DENABLE_TESTS=0 \
|
||||
-DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr \
|
||||
-DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
|
||||
-DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCOMPILER_CACHE={build_cache_type} -DENABLE_TESTS=1 \
|
||||
-DENABLE_BUILD_PROFILING=1 {current_directory}",
|
||||
command=cmake_cmd,
|
||||
workdir=build_dir,
|
||||
with_log=True,
|
||||
)
|
||||
@ -95,7 +127,7 @@ def main():
|
||||
Shell.check(f"ls -l {build_dir}/programs/")
|
||||
res = results[-1].is_ok()
|
||||
|
||||
Result.create_from(results=results, stopwatch=stop_watch).finish_job_accordingly()
|
||||
Result.create_from(results=results, stopwatch=stop_watch).complete_job()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
@ -379,4 +379,4 @@ if __name__ == "__main__":
|
||||
)
|
||||
)
|
||||
|
||||
Result.create_from(results=results, stopwatch=stop_watch).finish_job_accordingly()
|
||||
Result.create_from(results=results, stopwatch=stop_watch).complete_job()
|
||||
|
@ -1,120 +1,13 @@
|
||||
import argparse
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from praktika.result import Result
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import MetaClasses, Shell, Utils
|
||||
|
||||
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
|
||||
from ci.jobs.scripts.functional_tests_results import FTResultsProcessor
|
||||
|
||||
|
||||
class ClickHouseProc:
|
||||
def __init__(self):
|
||||
self.ch_config_dir = f"{Settings.TEMP_DIR}/etc/clickhouse-server"
|
||||
self.pid_file = f"{self.ch_config_dir}/clickhouse-server.pid"
|
||||
self.config_file = f"{self.ch_config_dir}/config.xml"
|
||||
self.user_files_path = f"{self.ch_config_dir}/user_files"
|
||||
self.test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
|
||||
self.command = f"clickhouse-server --config-file {self.config_file} --pid-file {self.pid_file} -- --path {self.ch_config_dir} --user_files_path {self.user_files_path} --top_level_domains_path {self.ch_config_dir}/top_level_domains --keeper_server.storage_path {self.ch_config_dir}/coordination"
|
||||
self.proc = None
|
||||
self.pid = 0
|
||||
nproc = int(Utils.cpu_count() / 2)
|
||||
self.fast_test_command = f"clickhouse-test --hung-check --fast-tests-only --no-random-settings --no-random-merge-tree-settings --no-long --testname --shard --zookeeper --check-zookeeper-session --order random --print-time --report-logs-stats --jobs {nproc} -- '' | ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a \"{self.test_output_file}\""
|
||||
# TODO: store info in case of failure
|
||||
self.info = ""
|
||||
self.info_file = ""
|
||||
|
||||
Utils.set_env("CLICKHOUSE_CONFIG_DIR", self.ch_config_dir)
|
||||
Utils.set_env("CLICKHOUSE_CONFIG", self.config_file)
|
||||
Utils.set_env("CLICKHOUSE_USER_FILES", self.user_files_path)
|
||||
Utils.set_env("CLICKHOUSE_SCHEMA_FILES", f"{self.ch_config_dir}/format_schemas")
|
||||
|
||||
def start(self):
|
||||
print("Starting ClickHouse server")
|
||||
Shell.check(f"rm {self.pid_file}")
|
||||
|
||||
def run_clickhouse():
|
||||
self.proc = Shell.run_async(
|
||||
self.command, verbose=True, suppress_output=True
|
||||
)
|
||||
|
||||
thread = threading.Thread(target=run_clickhouse)
|
||||
thread.daemon = True # Allow program to exit even if thread is still running
|
||||
thread.start()
|
||||
|
||||
# self.proc = Shell.run_async(self.command, verbose=True)
|
||||
|
||||
started = False
|
||||
try:
|
||||
for _ in range(5):
|
||||
pid = Shell.get_output(f"cat {self.pid_file}").strip()
|
||||
if not pid:
|
||||
Utils.sleep(1)
|
||||
continue
|
||||
started = True
|
||||
print(f"Got pid from fs [{pid}]")
|
||||
_ = int(pid)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not started:
|
||||
stdout = self.proc.stdout.read().strip() if self.proc.stdout else ""
|
||||
stderr = self.proc.stderr.read().strip() if self.proc.stderr else ""
|
||||
Utils.print_formatted_error("Failed to start ClickHouse", stdout, stderr)
|
||||
return False
|
||||
|
||||
print(f"ClickHouse server started successfully, pid [{pid}]")
|
||||
return True
|
||||
|
||||
def wait_ready(self):
|
||||
res, out, err = 0, "", ""
|
||||
attempts = 30
|
||||
delay = 2
|
||||
for attempt in range(attempts):
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
'clickhouse-client --query "select 1"', verbose=True
|
||||
)
|
||||
if out.strip() == "1":
|
||||
print("Server ready")
|
||||
break
|
||||
else:
|
||||
print(f"Server not ready, wait")
|
||||
Utils.sleep(delay)
|
||||
else:
|
||||
Utils.print_formatted_error(
|
||||
f"Server not ready after [{attempts*delay}s]", out, err
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def run_fast_test(self):
|
||||
if Path(self.test_output_file).exists():
|
||||
Path(self.test_output_file).unlink()
|
||||
exit_code = Shell.run(self.fast_test_command)
|
||||
return exit_code == 0
|
||||
|
||||
def terminate(self):
|
||||
print("Terminate ClickHouse process")
|
||||
timeout = 10
|
||||
if self.proc:
|
||||
Utils.terminate_process_group(self.proc.pid)
|
||||
|
||||
self.proc.terminate()
|
||||
try:
|
||||
self.proc.wait(timeout=10)
|
||||
print(f"Process {self.proc.pid} terminated gracefully.")
|
||||
except Exception:
|
||||
print(
|
||||
f"Process {self.proc.pid} did not terminate in {timeout} seconds, killing it..."
|
||||
)
|
||||
Utils.terminate_process_group(self.proc.pid, force=True)
|
||||
self.proc.wait() # Wait for the process to be fully killed
|
||||
print(f"Process {self.proc} was killed.")
|
||||
|
||||
|
||||
def clone_submodules():
|
||||
submodules_to_update = [
|
||||
"contrib/sysroot",
|
||||
@ -240,7 +133,7 @@ def main():
|
||||
Shell.check(f"rm -rf {build_dir} && mkdir -p {build_dir}")
|
||||
results.append(
|
||||
Result.create_from_command_execution(
|
||||
name="Checkout Submodules for Minimal Build",
|
||||
name="Checkout Submodules",
|
||||
command=clone_submodules,
|
||||
)
|
||||
)
|
||||
@ -295,8 +188,8 @@ def main():
|
||||
if res and JobStages.CONFIG in stages:
|
||||
commands = [
|
||||
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
|
||||
f"cp {current_directory}/programs/server/config.xml {current_directory}/programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
|
||||
f"{current_directory}/tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client",
|
||||
f"cp ./programs/server/config.xml ./programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
|
||||
f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --fast-test",
|
||||
# f"cp -a {current_directory}/programs/server/config.d/log_to_console.xml {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/",
|
||||
f"rm -f {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/secure_ports.xml",
|
||||
update_path_ch_config,
|
||||
@ -310,7 +203,7 @@ def main():
|
||||
)
|
||||
res = results[-1].is_ok()
|
||||
|
||||
CH = ClickHouseProc()
|
||||
CH = ClickHouseProc(fast_test=True)
|
||||
if res and JobStages.TEST in stages:
|
||||
stop_watch_ = Utils.Stopwatch()
|
||||
step_name = "Start ClickHouse Server"
|
||||
@ -322,15 +215,17 @@ def main():
|
||||
)
|
||||
|
||||
if res and JobStages.TEST in stages:
|
||||
stop_watch_ = Utils.Stopwatch()
|
||||
step_name = "Tests"
|
||||
print(step_name)
|
||||
res = res and CH.run_fast_test()
|
||||
if res:
|
||||
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
|
||||
results[-1].set_timing(stopwatch=stop_watch_)
|
||||
|
||||
CH.terminate()
|
||||
|
||||
Result.create_from(results=results, stopwatch=stop_watch).finish_job_accordingly()
|
||||
Result.create_from(results=results, stopwatch=stop_watch).complete_job()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
171
ci/jobs/functional_stateful_tests.py
Normal file
171
ci/jobs/functional_stateful_tests.py
Normal file
@ -0,0 +1,171 @@
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from praktika.result import Result
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import MetaClasses, Shell, Utils
|
||||
|
||||
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
|
||||
from ci.jobs.scripts.functional_tests_results import FTResultsProcessor
|
||||
|
||||
|
||||
class JobStages(metaclass=MetaClasses.WithIter):
|
||||
INSTALL_CLICKHOUSE = "install"
|
||||
START = "start"
|
||||
TEST = "test"
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="ClickHouse Build Job")
|
||||
parser.add_argument(
|
||||
"--ch-path", help="Path to clickhouse binary", default=f"{Settings.INPUT_DIR}"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test-options",
|
||||
help="Comma separated option(s): parallel|non-parallel|BATCH_NUM/BTATCH_TOT|..",
|
||||
default="",
|
||||
)
|
||||
parser.add_argument("--param", help="Optional job start stage", default=None)
|
||||
parser.add_argument("--test", help="Optional test name pattern", default="")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def run_test(
|
||||
no_parallel: bool, no_sequiential: bool, batch_num: int, batch_total: int, test=""
|
||||
):
|
||||
test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
|
||||
|
||||
test_command = f"clickhouse-test --jobs 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless \
|
||||
--hung-check --print-time \
|
||||
--capture-client-stacktrace --queries ./tests/queries -- '{test}' \
|
||||
| ts '%Y-%m-%d %H:%M:%S' | tee -a \"{test_output_file}\""
|
||||
if Path(test_output_file).exists():
|
||||
Path(test_output_file).unlink()
|
||||
Shell.run(test_command, verbose=True)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_args()
|
||||
test_options = args.test_options.split(",")
|
||||
no_parallel = "non-parallel" in test_options
|
||||
no_sequential = "parallel" in test_options
|
||||
batch_num, total_batches = 0, 0
|
||||
for to in test_options:
|
||||
if "/" in to:
|
||||
batch_num, total_batches = map(int, to.split("/"))
|
||||
|
||||
# os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output(
|
||||
# f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
|
||||
# verbose=True,
|
||||
# strict=True
|
||||
# )
|
||||
|
||||
ch_path = args.ch_path
|
||||
assert Path(
|
||||
ch_path + "/clickhouse"
|
||||
).is_file(), f"clickhouse binary not found under [{ch_path}]"
|
||||
|
||||
stop_watch = Utils.Stopwatch()
|
||||
|
||||
stages = list(JobStages)
|
||||
|
||||
logs_to_attach = []
|
||||
|
||||
stage = args.param or JobStages.INSTALL_CLICKHOUSE
|
||||
if stage:
|
||||
assert stage in JobStages, f"--param must be one of [{list(JobStages)}]"
|
||||
print(f"Job will start from stage [{stage}]")
|
||||
while stage in stages:
|
||||
stages.pop(0)
|
||||
stages.insert(0, stage)
|
||||
|
||||
res = True
|
||||
results = []
|
||||
|
||||
Utils.add_to_PATH(f"{ch_path}:tests")
|
||||
|
||||
if res and JobStages.INSTALL_CLICKHOUSE in stages:
|
||||
commands = [
|
||||
f"rm -rf /tmp/praktika/var/log/clickhouse-server/clickhouse-server.*",
|
||||
f"chmod +x {ch_path}/clickhouse",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local",
|
||||
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
|
||||
f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
|
||||
# TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
|
||||
f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage --no-azure",
|
||||
# clickhouse benchmark segfaults with --config-path, so provide client config by its default location
|
||||
f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/",
|
||||
# update_path_ch_config,
|
||||
# f"sed -i 's|>/var/|>{Settings.TEMP_DIR}/var/|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.xml",
|
||||
# f"sed -i 's|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/ssl_certs.xml",
|
||||
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
|
||||
f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
|
||||
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done",
|
||||
f"clickhouse-server --version",
|
||||
]
|
||||
results.append(
|
||||
Result.create_from_command_execution(
|
||||
name="Install ClickHouse", command=commands, with_log=True
|
||||
)
|
||||
)
|
||||
res = results[-1].is_ok()
|
||||
|
||||
CH = ClickHouseProc()
|
||||
if res and JobStages.START in stages:
|
||||
stop_watch_ = Utils.Stopwatch()
|
||||
step_name = "Start ClickHouse Server"
|
||||
print(step_name)
|
||||
minio_log = "/tmp/praktika/output/minio.log"
|
||||
res = res and CH.start_minio(test_type="stateful", log_file_path=minio_log)
|
||||
logs_to_attach += [minio_log]
|
||||
time.sleep(10)
|
||||
Shell.check("ps -ef | grep minio", verbose=True)
|
||||
res = res and Shell.check(
|
||||
"aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
|
||||
)
|
||||
res = res and CH.start()
|
||||
res = res and CH.wait_ready()
|
||||
if res:
|
||||
print("ch started")
|
||||
logs_to_attach += [
|
||||
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.log",
|
||||
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.err.log",
|
||||
]
|
||||
results.append(
|
||||
Result.create_from(
|
||||
name=step_name,
|
||||
status=res,
|
||||
stopwatch=stop_watch_,
|
||||
)
|
||||
)
|
||||
res = results[-1].is_ok()
|
||||
|
||||
if res and JobStages.TEST in stages:
|
||||
stop_watch_ = Utils.Stopwatch()
|
||||
step_name = "Tests"
|
||||
print(step_name)
|
||||
# assert Shell.check("clickhouse-client -q \"insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')\"", verbose=True)
|
||||
run_test(
|
||||
no_parallel=no_parallel,
|
||||
no_sequiential=no_sequential,
|
||||
batch_num=batch_num,
|
||||
batch_total=total_batches,
|
||||
test=args.test,
|
||||
)
|
||||
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
|
||||
results[-1].set_timing(stopwatch=stop_watch_)
|
||||
res = results[-1].is_ok()
|
||||
|
||||
Result.create_from(
|
||||
results=results, stopwatch=stop_watch, files=logs_to_attach if not res else []
|
||||
).complete_job()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
183
ci/jobs/functional_stateless_tests.py
Normal file
183
ci/jobs/functional_stateless_tests.py
Normal file
@ -0,0 +1,183 @@
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from praktika.result import Result
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import MetaClasses, Shell, Utils
|
||||
|
||||
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
|
||||
from ci.jobs.scripts.functional_tests_results import FTResultsProcessor
|
||||
|
||||
|
||||
class JobStages(metaclass=MetaClasses.WithIter):
|
||||
INSTALL_CLICKHOUSE = "install"
|
||||
START = "start"
|
||||
TEST = "test"
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="ClickHouse Build Job")
|
||||
parser.add_argument(
|
||||
"--ch-path", help="Path to clickhouse binary", default=f"{Settings.INPUT_DIR}"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test-options",
|
||||
help="Comma separated option(s): parallel|non-parallel|BATCH_NUM/BTATCH_TOT|..",
|
||||
default="",
|
||||
)
|
||||
parser.add_argument("--param", help="Optional job start stage", default=None)
|
||||
parser.add_argument("--test", help="Optional test name pattern", default="")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def run_stateless_test(
|
||||
no_parallel: bool, no_sequiential: bool, batch_num: int, batch_total: int, test=""
|
||||
):
|
||||
assert not (no_parallel and no_sequiential)
|
||||
test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
|
||||
aux = ""
|
||||
nproc = int(Utils.cpu_count() / 2)
|
||||
if batch_num and batch_total:
|
||||
aux = f"--run-by-hash-total {batch_total} --run-by-hash-num {batch_num-1}"
|
||||
statless_test_command = f"clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
|
||||
--no-drop-if-fail --capture-client-stacktrace --queries /repo/tests/queries --test-runs 1 --hung-check \
|
||||
{'--no-parallel' if no_parallel else ''} {'--no-sequential' if no_sequiential else ''} \
|
||||
--print-time --jobs {nproc} --report-coverage --report-logs-stats {aux} \
|
||||
--queries ./tests/queries -- '{test}' | ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a \"{test_output_file}\""
|
||||
if Path(test_output_file).exists():
|
||||
Path(test_output_file).unlink()
|
||||
Shell.run(statless_test_command, verbose=True)
|
||||
|
||||
|
||||
def main():
|
||||
|
||||
args = parse_args()
|
||||
test_options = args.test_options.split(",")
|
||||
no_parallel = "non-parallel" in test_options
|
||||
no_sequential = "parallel" in test_options
|
||||
batch_num, total_batches = 0, 0
|
||||
for to in test_options:
|
||||
if "/" in to:
|
||||
batch_num, total_batches = map(int, to.split("/"))
|
||||
|
||||
# os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output(
|
||||
# f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
|
||||
# verbose=True,
|
||||
# strict=True
|
||||
# )
|
||||
|
||||
ch_path = args.ch_path
|
||||
assert Path(
|
||||
ch_path + "/clickhouse"
|
||||
).is_file(), f"clickhouse binary not found under [{ch_path}]"
|
||||
|
||||
stop_watch = Utils.Stopwatch()
|
||||
|
||||
stages = list(JobStages)
|
||||
|
||||
logs_to_attach = []
|
||||
|
||||
stage = args.param or JobStages.INSTALL_CLICKHOUSE
|
||||
if stage:
|
||||
assert stage in JobStages, f"--param must be one of [{list(JobStages)}]"
|
||||
print(f"Job will start from stage [{stage}]")
|
||||
while stage in stages:
|
||||
stages.pop(0)
|
||||
stages.insert(0, stage)
|
||||
|
||||
res = True
|
||||
results = []
|
||||
|
||||
Utils.add_to_PATH(f"{ch_path}:tests")
|
||||
|
||||
if res and JobStages.INSTALL_CLICKHOUSE in stages:
|
||||
commands = [
|
||||
f"rm -rf /tmp/praktika/var/log/clickhouse-server/clickhouse-server.*",
|
||||
f"chmod +x {ch_path}/clickhouse",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor",
|
||||
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local",
|
||||
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
|
||||
f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
|
||||
# TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
|
||||
f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage --no-azure",
|
||||
# clickhouse benchmark segfaults with --config-path, so provide client config by its default location
|
||||
f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/",
|
||||
# update_path_ch_config,
|
||||
# f"sed -i 's|>/var/|>{Settings.TEMP_DIR}/var/|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.xml",
|
||||
# f"sed -i 's|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/ssl_certs.xml",
|
||||
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
|
||||
f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
|
||||
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done",
|
||||
f"clickhouse-server --version",
|
||||
]
|
||||
results.append(
|
||||
Result.create_from_command_execution(
|
||||
name="Install ClickHouse", command=commands, with_log=True
|
||||
)
|
||||
)
|
||||
res = results[-1].is_ok()
|
||||
|
||||
CH = ClickHouseProc()
|
||||
if res and JobStages.START in stages:
|
||||
stop_watch_ = Utils.Stopwatch()
|
||||
step_name = "Start ClickHouse Server"
|
||||
print(step_name)
|
||||
hdfs_log = "/tmp/praktika/output/hdfs_mini.log"
|
||||
minio_log = "/tmp/praktika/output/minio.log"
|
||||
res = res and CH.start_hdfs(log_file_path=hdfs_log)
|
||||
res = res and CH.start_minio(test_type="stateful", log_file_path=minio_log)
|
||||
logs_to_attach += [minio_log, hdfs_log]
|
||||
time.sleep(10)
|
||||
Shell.check("ps -ef | grep minio", verbose=True)
|
||||
Shell.check("ps -ef | grep hdfs", verbose=True)
|
||||
res = res and Shell.check(
|
||||
"aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
|
||||
)
|
||||
res = res and CH.start()
|
||||
res = res and CH.wait_ready()
|
||||
if res:
|
||||
print("ch started")
|
||||
logs_to_attach += [
|
||||
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.log",
|
||||
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.err.log",
|
||||
]
|
||||
results.append(
|
||||
Result.create_from(
|
||||
name=step_name,
|
||||
status=res,
|
||||
stopwatch=stop_watch_,
|
||||
)
|
||||
)
|
||||
res = results[-1].is_ok()
|
||||
|
||||
if res and JobStages.TEST in stages:
|
||||
stop_watch_ = Utils.Stopwatch()
|
||||
step_name = "Tests"
|
||||
print(step_name)
|
||||
assert Shell.check(
|
||||
"clickhouse-client -q \"insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')\"",
|
||||
verbose=True,
|
||||
)
|
||||
run_stateless_test(
|
||||
no_parallel=no_parallel,
|
||||
no_sequiential=no_sequential,
|
||||
batch_num=batch_num,
|
||||
batch_total=total_batches,
|
||||
test=args.test,
|
||||
)
|
||||
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
|
||||
results[-1].set_timing(stopwatch=stop_watch_)
|
||||
res = results[-1].is_ok()
|
||||
|
||||
Result.create_from(
|
||||
results=results, stopwatch=stop_watch, files=logs_to_attach if not res else []
|
||||
).complete_job()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
0
ci/jobs/scripts/__init__.py
Normal file
0
ci/jobs/scripts/__init__.py
Normal file
142
ci/jobs/scripts/clickhouse_proc.py
Normal file
142
ci/jobs/scripts/clickhouse_proc.py
Normal file
@ -0,0 +1,142 @@
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import Shell, Utils
|
||||
|
||||
|
||||
class ClickHouseProc:
|
||||
BACKUPS_XML = """
|
||||
<clickhouse>
|
||||
<backups>
|
||||
<type>local</type>
|
||||
<path>{CH_RUNTIME_DIR}/var/lib/clickhouse/disks/backups/</path>
|
||||
</backups>
|
||||
</clickhouse>
|
||||
"""
|
||||
|
||||
def __init__(self, fast_test=False):
|
||||
self.ch_config_dir = f"{Settings.TEMP_DIR}/etc/clickhouse-server"
|
||||
self.pid_file = f"{self.ch_config_dir}/clickhouse-server.pid"
|
||||
self.config_file = f"{self.ch_config_dir}/config.xml"
|
||||
self.user_files_path = f"{self.ch_config_dir}/user_files"
|
||||
self.test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
|
||||
self.command = f"clickhouse-server --config-file {self.config_file} --pid-file {self.pid_file} -- --path {self.ch_config_dir} --user_files_path {self.user_files_path} --top_level_domains_path {self.ch_config_dir}/top_level_domains --keeper_server.storage_path {self.ch_config_dir}/coordination"
|
||||
self.proc = None
|
||||
self.pid = 0
|
||||
nproc = int(Utils.cpu_count() / 2)
|
||||
self.fast_test_command = f"clickhouse-test --hung-check --fast-tests-only --no-random-settings --no-random-merge-tree-settings --no-long --testname --shard --zookeeper --check-zookeeper-session --order random --print-time --report-logs-stats --jobs {nproc} -- '' | ts '%Y-%m-%d %H:%M:%S' \
|
||||
| tee -a \"{self.test_output_file}\""
|
||||
# TODO: store info in case of failure
|
||||
self.info = ""
|
||||
self.info_file = ""
|
||||
|
||||
Utils.set_env("CLICKHOUSE_CONFIG_DIR", self.ch_config_dir)
|
||||
Utils.set_env("CLICKHOUSE_CONFIG", self.config_file)
|
||||
Utils.set_env("CLICKHOUSE_USER_FILES", self.user_files_path)
|
||||
# Utils.set_env("CLICKHOUSE_SCHEMA_FILES", f"{self.ch_config_dir}/format_schemas")
|
||||
|
||||
# if not fast_test:
|
||||
# with open(f"{self.ch_config_dir}/config.d/backups.xml", "w") as file:
|
||||
# file.write(self.BACKUPS_XML)
|
||||
|
||||
self.minio_proc = None
|
||||
|
||||
def start_hdfs(self, log_file_path):
|
||||
command = ["./ci/jobs/scripts/functional_tests/setup_hdfs_minicluster.sh"]
|
||||
with open(log_file_path, "w") as log_file:
|
||||
process = subprocess.Popen(
|
||||
command, stdout=log_file, stderr=subprocess.STDOUT
|
||||
)
|
||||
print(
|
||||
f"Started setup_hdfs_minicluster.sh asynchronously with PID {process.pid}"
|
||||
)
|
||||
return True
|
||||
|
||||
def start_minio(self, test_type, log_file_path):
|
||||
command = [
|
||||
"./ci/jobs/scripts/functional_tests/setup_minio.sh",
|
||||
test_type,
|
||||
"./tests",
|
||||
]
|
||||
with open(log_file_path, "w") as log_file:
|
||||
process = subprocess.Popen(
|
||||
command, stdout=log_file, stderr=subprocess.STDOUT
|
||||
)
|
||||
print(f"Started setup_minio.sh asynchronously with PID {process.pid}")
|
||||
return True
|
||||
|
||||
def start(self):
|
||||
print("Starting ClickHouse server")
|
||||
Shell.check(f"rm {self.pid_file}")
|
||||
self.proc = subprocess.Popen(self.command, stderr=subprocess.STDOUT, shell=True)
|
||||
started = False
|
||||
try:
|
||||
for _ in range(5):
|
||||
pid = Shell.get_output(f"cat {self.pid_file}").strip()
|
||||
if not pid:
|
||||
Utils.sleep(1)
|
||||
continue
|
||||
started = True
|
||||
print(f"Got pid from fs [{pid}]")
|
||||
_ = int(pid)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if not started:
|
||||
stdout = self.proc.stdout.read().strip() if self.proc.stdout else ""
|
||||
stderr = self.proc.stderr.read().strip() if self.proc.stderr else ""
|
||||
Utils.print_formatted_error("Failed to start ClickHouse", stdout, stderr)
|
||||
return False
|
||||
|
||||
print(f"ClickHouse server started successfully, pid [{pid}]")
|
||||
return True
|
||||
|
||||
def wait_ready(self):
|
||||
res, out, err = 0, "", ""
|
||||
attempts = 30
|
||||
delay = 2
|
||||
for attempt in range(attempts):
|
||||
res, out, err = Shell.get_res_stdout_stderr(
|
||||
'clickhouse-client --query "select 1"', verbose=True
|
||||
)
|
||||
if out.strip() == "1":
|
||||
print("Server ready")
|
||||
break
|
||||
else:
|
||||
print(f"Server not ready, wait")
|
||||
Utils.sleep(delay)
|
||||
else:
|
||||
Utils.print_formatted_error(
|
||||
f"Server not ready after [{attempts*delay}s]", out, err
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
def run_fast_test(self):
|
||||
if Path(self.test_output_file).exists():
|
||||
Path(self.test_output_file).unlink()
|
||||
exit_code = Shell.run(self.fast_test_command)
|
||||
return exit_code == 0
|
||||
|
||||
def terminate(self):
|
||||
print("Terminate ClickHouse process")
|
||||
timeout = 10
|
||||
if self.proc:
|
||||
Utils.terminate_process_group(self.proc.pid)
|
||||
|
||||
self.proc.terminate()
|
||||
try:
|
||||
self.proc.wait(timeout=10)
|
||||
print(f"Process {self.proc.pid} terminated gracefully.")
|
||||
except Exception:
|
||||
print(
|
||||
f"Process {self.proc.pid} did not terminate in {timeout} seconds, killing it..."
|
||||
)
|
||||
Utils.terminate_process_group(self.proc.pid, force=True)
|
||||
self.proc.wait() # Wait for the process to be fully killed
|
||||
print(f"Process {self.proc} was killed.")
|
||||
|
||||
if self.minio_proc:
|
||||
Utils.terminate_process_group(self.minio_proc.pid)
|
19
ci/jobs/scripts/functional_tests/setup_hdfs_minicluster.sh
Executable file
19
ci/jobs/scripts/functional_tests/setup_hdfs_minicluster.sh
Executable file
@ -0,0 +1,19 @@
|
||||
#!/bin/bash
|
||||
# shellcheck disable=SC2024
|
||||
|
||||
set -e -x -a -u
|
||||
|
||||
ls -lha
|
||||
|
||||
cd /hadoop-3.3.1
|
||||
|
||||
export JAVA_HOME=/usr
|
||||
mkdir -p target/test/data
|
||||
|
||||
bin/mapred minicluster -format -nomr -nnport 12222 &
|
||||
|
||||
while ! nc -z localhost 12222; do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
lsof -i :12222
|
162
ci/jobs/scripts/functional_tests/setup_minio.sh
Executable file
162
ci/jobs/scripts/functional_tests/setup_minio.sh
Executable file
@ -0,0 +1,162 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euxf -o pipefail
|
||||
|
||||
export MINIO_ROOT_USER=${MINIO_ROOT_USER:-clickhouse}
|
||||
export MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-clickhouse}
|
||||
TEST_DIR=${2:-/repo/tests/}
|
||||
|
||||
if [ -d "$TEMP_DIR" ]; then
|
||||
TEST_DIR=$(readlink -f $TEST_DIR)
|
||||
cd "$TEMP_DIR"
|
||||
# add / for minio mc in docker
|
||||
PATH="/:.:$PATH"
|
||||
fi
|
||||
|
||||
usage() {
|
||||
echo $"Usage: $0 <stateful|stateless> <test_path> (default path: /usr/share/clickhouse-test)"
|
||||
exit 1
|
||||
}
|
||||
|
||||
check_arg() {
|
||||
local query_dir
|
||||
if [ ! $# -eq 1 ]; then
|
||||
if [ ! $# -eq 2 ]; then
|
||||
echo "ERROR: need either one or two arguments, <stateful|stateless> <test_path> (default path: /usr/share/clickhouse-test)"
|
||||
usage
|
||||
fi
|
||||
fi
|
||||
case "$1" in
|
||||
stateless)
|
||||
query_dir="0_stateless"
|
||||
;;
|
||||
stateful)
|
||||
query_dir="1_stateful"
|
||||
;;
|
||||
*)
|
||||
echo "unknown test type ${test_type}"
|
||||
usage
|
||||
;;
|
||||
esac
|
||||
echo ${query_dir}
|
||||
}
|
||||
|
||||
find_arch() {
|
||||
local arch
|
||||
case $(uname -m) in
|
||||
x86_64)
|
||||
arch="amd64"
|
||||
;;
|
||||
aarch64)
|
||||
arch="arm64"
|
||||
;;
|
||||
*)
|
||||
echo "unknown architecture $(uname -m)";
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
echo ${arch}
|
||||
}
|
||||
|
||||
find_os() {
|
||||
local os
|
||||
os=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
echo "${os}"
|
||||
}
|
||||
|
||||
download_minio() {
|
||||
local os
|
||||
local arch
|
||||
local minio_server_version=${MINIO_SERVER_VERSION:-2024-08-03T04-33-23Z}
|
||||
local minio_client_version=${MINIO_CLIENT_VERSION:-2024-07-31T15-58-33Z}
|
||||
|
||||
os=$(find_os)
|
||||
arch=$(find_arch)
|
||||
wget "https://dl.min.io/server/minio/release/${os}-${arch}/archive/minio.RELEASE.${minio_server_version}" -O ./minio
|
||||
wget "https://dl.min.io/client/mc/release/${os}-${arch}/archive/mc.RELEASE.${minio_client_version}" -O ./mc
|
||||
chmod +x ./mc ./minio
|
||||
}
|
||||
|
||||
start_minio() {
|
||||
pwd
|
||||
mkdir -p ./minio_data
|
||||
minio --version
|
||||
nohup minio server --address ":11111" ./minio_data &
|
||||
wait_for_it
|
||||
lsof -i :11111
|
||||
sleep 5
|
||||
}
|
||||
|
||||
setup_minio() {
|
||||
local test_type=$1
|
||||
echo "setup_minio(), test_type=$test_type"
|
||||
mc alias set clickminio http://localhost:11111 clickhouse clickhouse
|
||||
mc admin user add clickminio test testtest
|
||||
mc admin policy attach clickminio readwrite --user=test ||:
|
||||
mc mb --ignore-existing clickminio/test
|
||||
if [ "$test_type" = "stateless" ]; then
|
||||
echo "Create @test bucket in minio"
|
||||
mc anonymous set public clickminio/test
|
||||
fi
|
||||
}
|
||||
|
||||
# uploads data to minio, by default after unpacking all tests
|
||||
# will be in /usr/share/clickhouse-test/queries
|
||||
upload_data() {
|
||||
local query_dir=$1
|
||||
local test_path=$2
|
||||
local data_path=${test_path}/queries/${query_dir}/data_minio
|
||||
echo "upload_data() data_path=$data_path"
|
||||
|
||||
# iterating over globs will cause redundant file variable to be
|
||||
# a path to a file, not a filename
|
||||
# shellcheck disable=SC2045
|
||||
if [ -d "${data_path}" ]; then
|
||||
mc cp --recursive "${data_path}"/ clickminio/test/
|
||||
fi
|
||||
}
|
||||
|
||||
setup_aws_credentials() {
|
||||
local minio_root_user=${MINIO_ROOT_USER:-clickhouse}
|
||||
local minio_root_password=${MINIO_ROOT_PASSWORD:-clickhouse}
|
||||
mkdir -p ~/.aws
|
||||
cat <<EOT >> ~/.aws/credentials
|
||||
[default]
|
||||
aws_access_key_id=${minio_root_user}
|
||||
aws_secret_access_key=${minio_root_password}
|
||||
EOT
|
||||
}
|
||||
|
||||
wait_for_it() {
|
||||
local counter=0
|
||||
local max_counter=60
|
||||
local url="http://localhost:11111"
|
||||
local params=(
|
||||
--silent
|
||||
--verbose
|
||||
)
|
||||
while ! curl "${params[@]}" "${url}" 2>&1 | grep AccessDenied
|
||||
do
|
||||
if [[ ${counter} == "${max_counter}" ]]; then
|
||||
echo "failed to setup minio"
|
||||
exit 0
|
||||
fi
|
||||
echo "trying to connect to minio"
|
||||
sleep 1
|
||||
counter=$((counter + 1))
|
||||
done
|
||||
}
|
||||
|
||||
main() {
|
||||
local query_dir
|
||||
query_dir=$(check_arg "$@")
|
||||
if ! (minio --version && mc --version); then
|
||||
download_minio
|
||||
fi
|
||||
start_minio
|
||||
setup_minio "$1"
|
||||
upload_data "${query_dir}" "$TEST_DIR"
|
||||
setup_aws_credentials
|
||||
}
|
||||
|
||||
main "$@"
|
@ -1,7 +1,6 @@
|
||||
import dataclasses
|
||||
from typing import List
|
||||
|
||||
from praktika.environment import Environment
|
||||
from praktika.result import Result
|
||||
|
||||
OK_SIGN = "[ OK "
|
||||
@ -233,6 +232,8 @@ class FTResultsProcessor:
|
||||
else:
|
||||
pass
|
||||
|
||||
info = f"Total: {s.total - s.skipped}, Failed: {s.failed}"
|
||||
|
||||
# TODO: !!!
|
||||
# def test_result_comparator(item):
|
||||
# # sort by status then by check name
|
||||
@ -250,10 +251,11 @@ class FTResultsProcessor:
|
||||
# test_results.sort(key=test_result_comparator)
|
||||
|
||||
return Result.create_from(
|
||||
name=Environment.JOB_NAME,
|
||||
name="Tests",
|
||||
results=test_results,
|
||||
status=state,
|
||||
files=[self.tests_output_file],
|
||||
info=info,
|
||||
with_info_from_results=False,
|
||||
)
|
||||
|
||||
|
@ -37,6 +37,30 @@ def create_parser():
|
||||
type=str,
|
||||
default=None,
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--test",
|
||||
help="Custom parameter to pass into a job script, it's up to job script how to use it, for local test",
|
||||
type=str,
|
||||
default="",
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--pr",
|
||||
help="PR number. Optional parameter for local run. Set if you want an required artifact to be uploaded from CI run in that PR",
|
||||
type=int,
|
||||
default=None,
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--sha",
|
||||
help="Commit sha. Optional parameter for local run. Set if you want an required artifact to be uploaded from CI run on that sha, head sha will be used if not set",
|
||||
type=str,
|
||||
default=None,
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--branch",
|
||||
help="Commit sha. Optional parameter for local run. Set if you want an required artifact to be uploaded from CI run on that branch, main branch name will be used if not set",
|
||||
type=str,
|
||||
default=None,
|
||||
)
|
||||
run_parser.add_argument(
|
||||
"--ci",
|
||||
help="When not set - dummy env will be generated, for local test",
|
||||
@ -85,9 +109,13 @@ if __name__ == "__main__":
|
||||
workflow=workflow,
|
||||
job=job,
|
||||
docker=args.docker,
|
||||
dummy_env=not args.ci,
|
||||
local_run=not args.ci,
|
||||
no_docker=args.no_docker,
|
||||
param=args.param,
|
||||
test=args.test,
|
||||
pr=args.pr,
|
||||
branch=args.branch,
|
||||
sha=args.sha,
|
||||
)
|
||||
else:
|
||||
parser.print_help()
|
||||
|
@ -6,7 +6,7 @@ from types import SimpleNamespace
|
||||
from typing import Any, Dict, List, Type
|
||||
|
||||
from praktika import Workflow
|
||||
from praktika._settings import _Settings
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import MetaClasses, T
|
||||
|
||||
|
||||
@ -30,13 +30,12 @@ class _Environment(MetaClasses.Serializable):
|
||||
INSTANCE_ID: str
|
||||
INSTANCE_LIFE_CYCLE: str
|
||||
LOCAL_RUN: bool = False
|
||||
PARAMETER: Any = None
|
||||
REPORT_INFO: List[str] = dataclasses.field(default_factory=list)
|
||||
name = "environment"
|
||||
|
||||
@classmethod
|
||||
def file_name_static(cls, _name=""):
|
||||
return f"{_Settings.TEMP_DIR}/{cls.name}.json"
|
||||
return f"{Settings.TEMP_DIR}/{cls.name}.json"
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: Type[T], obj: Dict[str, Any]) -> T:
|
||||
@ -67,12 +66,12 @@ class _Environment(MetaClasses.Serializable):
|
||||
|
||||
@staticmethod
|
||||
def get_needs_statuses():
|
||||
if Path(_Settings.WORKFLOW_STATUS_FILE).is_file():
|
||||
with open(_Settings.WORKFLOW_STATUS_FILE, "r", encoding="utf8") as f:
|
||||
if Path(Settings.WORKFLOW_STATUS_FILE).is_file():
|
||||
with open(Settings.WORKFLOW_STATUS_FILE, "r", encoding="utf8") as f:
|
||||
return json.load(f)
|
||||
else:
|
||||
print(
|
||||
f"ERROR: Status file [{_Settings.WORKFLOW_STATUS_FILE}] does not exist"
|
||||
f"ERROR: Status file [{Settings.WORKFLOW_STATUS_FILE}] does not exist"
|
||||
)
|
||||
raise RuntimeError()
|
||||
|
||||
@ -159,7 +158,8 @@ class _Environment(MetaClasses.Serializable):
|
||||
@classmethod
|
||||
def get_s3_prefix_static(cls, pr_number, branch, sha, latest=False):
|
||||
prefix = ""
|
||||
if pr_number > 0:
|
||||
assert sha or latest
|
||||
if pr_number and pr_number > 0:
|
||||
prefix += f"{pr_number}"
|
||||
else:
|
||||
prefix += f"{branch}"
|
||||
@ -171,18 +171,15 @@ class _Environment(MetaClasses.Serializable):
|
||||
|
||||
# TODO: find a better place for the function. This file should not import praktika.settings
|
||||
# as it's requires reading users config, that's why imports nested inside the function
|
||||
def get_report_url(self):
|
||||
def get_report_url(self, settings, latest=False):
|
||||
import urllib
|
||||
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import Utils
|
||||
|
||||
path = Settings.HTML_S3_PATH
|
||||
for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items():
|
||||
path = settings.HTML_S3_PATH
|
||||
for bucket, endpoint in settings.S3_BUCKET_TO_HTTP_ENDPOINT.items():
|
||||
if bucket in path:
|
||||
path = path.replace(bucket, endpoint)
|
||||
break
|
||||
REPORT_URL = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}&name_1={urllib.parse.quote(self.JOB_NAME, safe='')}"
|
||||
REPORT_URL = f"https://{path}/{Path(settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={'latest' if latest else self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}&name_1={urllib.parse.quote(self.JOB_NAME, safe='')}"
|
||||
return REPORT_URL
|
||||
|
||||
def is_local_run(self):
|
||||
|
@ -1,124 +0,0 @@
|
||||
import dataclasses
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class _Settings:
|
||||
######################################
|
||||
# Pipeline generation settings #
|
||||
######################################
|
||||
CI_PATH = "./ci"
|
||||
WORKFLOW_PATH_PREFIX: str = "./.github/workflows"
|
||||
WORKFLOWS_DIRECTORY: str = f"{CI_PATH}/workflows"
|
||||
SETTINGS_DIRECTORY: str = f"{CI_PATH}/settings"
|
||||
CI_CONFIG_JOB_NAME = "Config Workflow"
|
||||
DOCKER_BUILD_JOB_NAME = "Docker Builds"
|
||||
FINISH_WORKFLOW_JOB_NAME = "Finish Workflow"
|
||||
READY_FOR_MERGE_STATUS_NAME = "Ready for Merge"
|
||||
CI_CONFIG_RUNS_ON: Optional[List[str]] = None
|
||||
DOCKER_BUILD_RUNS_ON: Optional[List[str]] = None
|
||||
VALIDATE_FILE_PATHS: bool = True
|
||||
|
||||
######################################
|
||||
# Runtime Settings #
|
||||
######################################
|
||||
MAX_RETRIES_S3 = 3
|
||||
MAX_RETRIES_GH = 3
|
||||
|
||||
######################################
|
||||
# S3 (artifact storage) settings #
|
||||
######################################
|
||||
S3_ARTIFACT_PATH: str = ""
|
||||
|
||||
######################################
|
||||
# CI workspace settings #
|
||||
######################################
|
||||
TEMP_DIR: str = "/tmp/praktika"
|
||||
OUTPUT_DIR: str = f"{TEMP_DIR}/output"
|
||||
INPUT_DIR: str = f"{TEMP_DIR}/input"
|
||||
PYTHON_INTERPRETER: str = "python3"
|
||||
PYTHON_PACKET_MANAGER: str = "pip3"
|
||||
PYTHON_VERSION: str = "3.9"
|
||||
INSTALL_PYTHON_FOR_NATIVE_JOBS: bool = False
|
||||
INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS: str = "./ci/requirements.txt"
|
||||
ENVIRONMENT_VAR_FILE: str = f"{TEMP_DIR}/environment.json"
|
||||
RUN_LOG: str = f"{TEMP_DIR}/praktika_run.log"
|
||||
|
||||
SECRET_GH_APP_ID: str = "GH_APP_ID"
|
||||
SECRET_GH_APP_PEM_KEY: str = "GH_APP_PEM_KEY"
|
||||
|
||||
ENV_SETUP_SCRIPT: str = "/tmp/praktika_setup_env.sh"
|
||||
WORKFLOW_STATUS_FILE: str = f"{TEMP_DIR}/workflow_status.json"
|
||||
|
||||
######################################
|
||||
# CI Cache settings #
|
||||
######################################
|
||||
CACHE_VERSION: int = 1
|
||||
CACHE_DIGEST_LEN: int = 20
|
||||
CACHE_S3_PATH: str = ""
|
||||
CACHE_LOCAL_PATH: str = f"{TEMP_DIR}/ci_cache"
|
||||
|
||||
######################################
|
||||
# Report settings #
|
||||
######################################
|
||||
HTML_S3_PATH: str = ""
|
||||
HTML_PAGE_FILE: str = "./praktika/json.html"
|
||||
TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"])
|
||||
S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None
|
||||
|
||||
DOCKERHUB_USERNAME: str = ""
|
||||
DOCKERHUB_SECRET: str = ""
|
||||
DOCKER_WD: str = "/wd"
|
||||
|
||||
######################################
|
||||
# CI DB Settings #
|
||||
######################################
|
||||
SECRET_CI_DB_URL: str = "CI_DB_URL"
|
||||
SECRET_CI_DB_PASSWORD: str = "CI_DB_PASSWORD"
|
||||
CI_DB_DB_NAME = ""
|
||||
CI_DB_TABLE_NAME = ""
|
||||
CI_DB_INSERT_TIMEOUT_SEC = 5
|
||||
|
||||
|
||||
_USER_DEFINED_SETTINGS = [
|
||||
"S3_ARTIFACT_PATH",
|
||||
"CACHE_S3_PATH",
|
||||
"HTML_S3_PATH",
|
||||
"S3_BUCKET_TO_HTTP_ENDPOINT",
|
||||
"TEXT_CONTENT_EXTENSIONS",
|
||||
"TEMP_DIR",
|
||||
"OUTPUT_DIR",
|
||||
"INPUT_DIR",
|
||||
"CI_CONFIG_RUNS_ON",
|
||||
"DOCKER_BUILD_RUNS_ON",
|
||||
"CI_CONFIG_JOB_NAME",
|
||||
"PYTHON_INTERPRETER",
|
||||
"PYTHON_VERSION",
|
||||
"PYTHON_PACKET_MANAGER",
|
||||
"INSTALL_PYTHON_FOR_NATIVE_JOBS",
|
||||
"INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS",
|
||||
"MAX_RETRIES_S3",
|
||||
"MAX_RETRIES_GH",
|
||||
"VALIDATE_FILE_PATHS",
|
||||
"DOCKERHUB_USERNAME",
|
||||
"DOCKERHUB_SECRET",
|
||||
"READY_FOR_MERGE_STATUS_NAME",
|
||||
"SECRET_CI_DB_URL",
|
||||
"SECRET_CI_DB_PASSWORD",
|
||||
"CI_DB_DB_NAME",
|
||||
"CI_DB_TABLE_NAME",
|
||||
"CI_DB_INSERT_TIMEOUT_SEC",
|
||||
"SECRET_GH_APP_PEM_KEY",
|
||||
"SECRET_GH_APP_ID",
|
||||
]
|
||||
|
||||
|
||||
class GHRunners:
|
||||
ubuntu = "ubuntu-latest"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for setting in _USER_DEFINED_SETTINGS:
|
||||
print(_Settings().__getattribute__(setting))
|
||||
# print(dataclasses.asdict(_Settings()))
|
@ -52,7 +52,7 @@ class CIDB:
|
||||
check_status=result.status,
|
||||
check_duration_ms=int(result.duration * 1000),
|
||||
check_start_time=Utils.timestamp_to_str(result.start_time),
|
||||
report_url=env.get_report_url(),
|
||||
report_url=env.get_report_url(settings=Settings),
|
||||
pull_request_url=env.CHANGE_URL,
|
||||
base_ref=env.BASE_BRANCH,
|
||||
base_repo=env.REPOSITORY,
|
||||
|
@ -23,7 +23,7 @@ class Digest:
|
||||
hash_string = hash_obj.hexdigest()
|
||||
return hash_string
|
||||
|
||||
def calc_job_digest(self, job_config: Job.Config):
|
||||
def calc_job_digest(self, job_config: Job.Config, docker_digests):
|
||||
config = job_config.digest_config
|
||||
if not config:
|
||||
return "f" * Settings.CACHE_DIGEST_LEN
|
||||
@ -31,32 +31,32 @@ class Digest:
|
||||
cache_key = self._hash_digest_config(config)
|
||||
|
||||
if cache_key in self.digest_cache:
|
||||
return self.digest_cache[cache_key]
|
||||
|
||||
print(
|
||||
f"calc digest for job [{job_config.name}]: hash_key [{cache_key}] - from cache"
|
||||
)
|
||||
digest = self.digest_cache[cache_key]
|
||||
else:
|
||||
included_files = Utils.traverse_paths(
|
||||
job_config.digest_config.include_paths,
|
||||
job_config.digest_config.exclude_paths,
|
||||
sorted=True,
|
||||
)
|
||||
|
||||
print(
|
||||
f"calc digest for job [{job_config.name}]: hash_key [{cache_key}], include [{len(included_files)}] files"
|
||||
)
|
||||
# Sort files to ensure consistent hash calculation
|
||||
included_files.sort()
|
||||
|
||||
# Calculate MD5 hash
|
||||
res = ""
|
||||
if not included_files:
|
||||
res = "f" * Settings.CACHE_DIGEST_LEN
|
||||
print(f"NOTE: empty digest config [{config}] - return dummy digest")
|
||||
else:
|
||||
hash_md5 = hashlib.md5()
|
||||
for file_path in included_files:
|
||||
res = self._calc_file_digest(file_path, hash_md5)
|
||||
assert res
|
||||
self.digest_cache[cache_key] = res
|
||||
return res
|
||||
for i, file_path in enumerate(included_files):
|
||||
hash_md5 = self._calc_file_digest(file_path, hash_md5)
|
||||
digest = hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
|
||||
self.digest_cache[cache_key] = digest
|
||||
|
||||
if job_config.run_in_docker:
|
||||
# respect docker digest in the job digest
|
||||
docker_digest = docker_digests[job_config.run_in_docker.split("+")[0]]
|
||||
digest = "-".join([docker_digest, digest])
|
||||
|
||||
return digest
|
||||
|
||||
def calc_docker_digest(
|
||||
self,
|
||||
@ -103,10 +103,10 @@ class Digest:
|
||||
print(
|
||||
f"WARNING: No valid file resolved by link {file_path} -> {resolved_path} - skipping digest calculation"
|
||||
)
|
||||
return hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
|
||||
return hash_md5
|
||||
|
||||
with open(resolved_path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hash_md5.update(chunk)
|
||||
|
||||
return hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
|
||||
return hash_md5
|
||||
|
@ -1,3 +0,0 @@
|
||||
from praktika._environment import _Environment
|
||||
|
||||
Environment = _Environment.get()
|
@ -18,7 +18,7 @@ class GH:
|
||||
ret_code, out, err = Shell.get_res_stdout_stderr(command, verbose=True)
|
||||
res = ret_code == 0
|
||||
if not res and "Validation Failed" in err:
|
||||
print("ERROR: GH command validation error")
|
||||
print(f"ERROR: GH command validation error.")
|
||||
break
|
||||
if not res and "Bad credentials" in err:
|
||||
print("ERROR: GH credentials/auth failure")
|
||||
|
@ -1,6 +1,5 @@
|
||||
from praktika._environment import _Environment
|
||||
from praktika.cache import Cache
|
||||
from praktika.mangle import _get_workflows
|
||||
from praktika.runtime import RunConfig
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import Utils
|
||||
@ -8,11 +7,10 @@ from praktika.utils import Utils
|
||||
|
||||
class CacheRunnerHooks:
|
||||
@classmethod
|
||||
def configure(cls, _workflow):
|
||||
workflow_config = RunConfig.from_fs(_workflow.name)
|
||||
def configure(cls, workflow):
|
||||
workflow_config = RunConfig.from_fs(workflow.name)
|
||||
docker_digests = workflow_config.digest_dockers
|
||||
cache = Cache()
|
||||
assert _Environment.get().WORKFLOW_NAME
|
||||
workflow = _get_workflows(name=_Environment.get().WORKFLOW_NAME)[0]
|
||||
print(f"Workflow Configure, workflow [{workflow.name}]")
|
||||
assert (
|
||||
workflow.enable_cache
|
||||
@ -20,11 +18,13 @@ class CacheRunnerHooks:
|
||||
artifact_digest_map = {}
|
||||
job_digest_map = {}
|
||||
for job in workflow.jobs:
|
||||
digest = cache.digest.calc_job_digest(
|
||||
job_config=job, docker_digests=docker_digests
|
||||
)
|
||||
if not job.digest_config:
|
||||
print(
|
||||
f"NOTE: job [{job.name}] has no Config.digest_config - skip cache check, always run"
|
||||
)
|
||||
digest = cache.digest.calc_job_digest(job_config=job)
|
||||
job_digest_map[job.name] = digest
|
||||
if job.provides:
|
||||
# assign the job digest also to the artifacts it provides
|
||||
@ -50,7 +50,6 @@ class CacheRunnerHooks:
|
||||
), f"BUG, Workflow with enabled cache must have job digests after configuration, wf [{workflow.name}]"
|
||||
|
||||
print("Check remote cache")
|
||||
job_to_cache_record = {}
|
||||
for job_name, job_digest in workflow_config.digest_jobs.items():
|
||||
record = cache.fetch_success(job_name=job_name, job_digest=job_digest)
|
||||
if record:
|
||||
@ -60,7 +59,7 @@ class CacheRunnerHooks:
|
||||
)
|
||||
workflow_config.cache_success.append(job_name)
|
||||
workflow_config.cache_success_base64.append(Utils.to_base64(job_name))
|
||||
job_to_cache_record[job_name] = record
|
||||
workflow_config.cache_jobs[job_name] = record
|
||||
|
||||
print("Check artifacts to reuse")
|
||||
for job in workflow.jobs:
|
||||
@ -68,7 +67,7 @@ class CacheRunnerHooks:
|
||||
if job.provides:
|
||||
for artifact_name in job.provides:
|
||||
workflow_config.cache_artifacts[artifact_name] = (
|
||||
job_to_cache_record[job.name]
|
||||
workflow_config.cache_jobs[job.name]
|
||||
)
|
||||
|
||||
print(f"Write config to GH's job output")
|
||||
|
@ -1,63 +1,125 @@
|
||||
import dataclasses
|
||||
import json
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from praktika._environment import _Environment
|
||||
from praktika.gh import GH
|
||||
from praktika.parser import WorkflowConfigParser
|
||||
from praktika.result import Result, ResultInfo
|
||||
from praktika.result import Result, ResultInfo, _ResultS3
|
||||
from praktika.runtime import RunConfig
|
||||
from praktika.s3 import S3
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import Shell, Utils
|
||||
from praktika.utils import Utils
|
||||
|
||||
|
||||
@dataclasses.dataclass
|
||||
class GitCommit:
|
||||
date: str
|
||||
message: str
|
||||
# date: str
|
||||
# message: str
|
||||
sha: str
|
||||
|
||||
@staticmethod
|
||||
def from_json(json_data: str) -> List["GitCommit"]:
|
||||
def from_json(file) -> List["GitCommit"]:
|
||||
commits = []
|
||||
json_data = None
|
||||
try:
|
||||
data = json.loads(json_data)
|
||||
|
||||
with open(file, "r", encoding="utf-8") as f:
|
||||
json_data = json.load(f)
|
||||
commits = [
|
||||
GitCommit(
|
||||
message=commit["messageHeadline"],
|
||||
sha=commit["oid"],
|
||||
date=commit["committedDate"],
|
||||
# message=commit["messageHeadline"],
|
||||
sha=commit["sha"],
|
||||
# date=commit["committedDate"],
|
||||
)
|
||||
for commit in data.get("commits", [])
|
||||
for commit in json_data
|
||||
]
|
||||
except Exception as e:
|
||||
print(
|
||||
f"ERROR: Failed to deserialize commit's data: [{json_data}], ex: [{e}]"
|
||||
f"ERROR: Failed to deserialize commit's data [{json_data}], ex: [{e}]"
|
||||
)
|
||||
|
||||
return commits
|
||||
|
||||
@classmethod
|
||||
def update_s3_data(cls):
|
||||
env = _Environment.get()
|
||||
sha = env.SHA
|
||||
if not sha:
|
||||
print("WARNING: Failed to retrieve commit sha")
|
||||
return
|
||||
commits = cls.pull_from_s3()
|
||||
for commit in commits:
|
||||
if sha == commit.sha:
|
||||
print(
|
||||
f"INFO: Sha already present in commits data [{sha}] - skip data update"
|
||||
)
|
||||
return
|
||||
commits.append(GitCommit(sha=sha))
|
||||
cls.push_to_s3(commits)
|
||||
return
|
||||
|
||||
@classmethod
|
||||
def dump(cls, commits):
|
||||
commits_ = []
|
||||
for commit in commits:
|
||||
commits_.append(dataclasses.asdict(commit))
|
||||
with open(cls.file_name(), "w", encoding="utf8") as f:
|
||||
json.dump(commits_, f)
|
||||
|
||||
@classmethod
|
||||
def pull_from_s3(cls):
|
||||
local_path = Path(cls.file_name())
|
||||
file_name = local_path.name
|
||||
env = _Environment.get()
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{cls.get_s3_prefix(pr_number=env.PR_NUMBER, branch=env.BRANCH)}/{file_name}"
|
||||
if not S3.copy_file_from_s3(s3_path=s3_path, local_path=local_path):
|
||||
print(f"WARNING: failed to cp file [{s3_path}] from s3")
|
||||
return []
|
||||
return cls.from_json(local_path)
|
||||
|
||||
@classmethod
|
||||
def push_to_s3(cls, commits):
|
||||
print(f"INFO: push commits data to s3, commits num [{len(commits)}]")
|
||||
cls.dump(commits)
|
||||
local_path = Path(cls.file_name())
|
||||
file_name = local_path.name
|
||||
env = _Environment.get()
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{cls.get_s3_prefix(pr_number=env.PR_NUMBER, branch=env.BRANCH)}/{file_name}"
|
||||
if not S3.copy_file_to_s3(s3_path=s3_path, local_path=local_path, text=True):
|
||||
print(f"WARNING: failed to cp file [{local_path}] to s3")
|
||||
|
||||
@classmethod
|
||||
def get_s3_prefix(cls, pr_number, branch):
|
||||
prefix = ""
|
||||
assert pr_number or branch
|
||||
if pr_number and pr_number > 0:
|
||||
prefix += f"{pr_number}"
|
||||
else:
|
||||
prefix += f"{branch}"
|
||||
return prefix
|
||||
|
||||
@classmethod
|
||||
def file_name(cls):
|
||||
return f"{Settings.TEMP_DIR}/commits.json"
|
||||
|
||||
# def _get_pr_commits(pr_number):
|
||||
# res = []
|
||||
# if not pr_number:
|
||||
# return res
|
||||
# output = Shell.get_output(f"gh pr view {pr_number} --json commits")
|
||||
# if output:
|
||||
# res = GitCommit.from_json(output)
|
||||
# return res
|
||||
|
||||
|
||||
class HtmlRunnerHooks:
|
||||
@classmethod
|
||||
def configure(cls, _workflow):
|
||||
|
||||
def _get_pr_commits(pr_number):
|
||||
res = []
|
||||
if not pr_number:
|
||||
return res
|
||||
output = Shell.get_output(f"gh pr view {pr_number} --json commits")
|
||||
if output:
|
||||
res = GitCommit.from_json(output)
|
||||
return res
|
||||
|
||||
# generate pending Results for all jobs in the workflow
|
||||
if _workflow.enable_cache:
|
||||
skip_jobs = RunConfig.from_fs(_workflow.name).cache_success
|
||||
job_cache_records = RunConfig.from_fs(_workflow.name).cache_jobs
|
||||
else:
|
||||
skip_jobs = []
|
||||
|
||||
@ -67,36 +129,22 @@ class HtmlRunnerHooks:
|
||||
if job.name not in skip_jobs:
|
||||
result = Result.generate_pending(job.name)
|
||||
else:
|
||||
result = Result.generate_skipped(job.name)
|
||||
result = Result.generate_skipped(job.name, job_cache_records[job.name])
|
||||
results.append(result)
|
||||
summary_result = Result.generate_pending(_workflow.name, results=results)
|
||||
summary_result.aux_links.append(env.CHANGE_URL)
|
||||
summary_result.aux_links.append(env.RUN_URL)
|
||||
summary_result.links.append(env.CHANGE_URL)
|
||||
summary_result.links.append(env.RUN_URL)
|
||||
summary_result.start_time = Utils.timestamp()
|
||||
page_url = "/".join(
|
||||
["https:/", Settings.HTML_S3_PATH, str(Path(Settings.HTML_PAGE_FILE).name)]
|
||||
)
|
||||
for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items():
|
||||
page_url = page_url.replace(bucket, endpoint)
|
||||
# TODO: add support for non-PRs (use branch?)
|
||||
page_url += f"?PR={env.PR_NUMBER}&sha=latest&name_0={urllib.parse.quote(env.WORKFLOW_NAME, safe='')}"
|
||||
summary_result.html_link = page_url
|
||||
|
||||
# clean the previous latest results in PR if any
|
||||
if env.PR_NUMBER:
|
||||
S3.clean_latest_result()
|
||||
S3.copy_result_to_s3(
|
||||
summary_result,
|
||||
unlock=False,
|
||||
)
|
||||
|
||||
assert _ResultS3.copy_result_to_s3_with_version(summary_result, version=0)
|
||||
page_url = env.get_report_url(settings=Settings)
|
||||
print(f"CI Status page url [{page_url}]")
|
||||
|
||||
res1 = GH.post_commit_status(
|
||||
name=_workflow.name,
|
||||
status=Result.Status.PENDING,
|
||||
description="",
|
||||
url=page_url,
|
||||
url=env.get_report_url(settings=Settings, latest=True),
|
||||
)
|
||||
res2 = GH.post_pr_comment(
|
||||
comment_body=f"Workflow [[{_workflow.name}]({page_url})], commit [{_Environment.get().SHA[:8]}]",
|
||||
@ -106,23 +154,15 @@ class HtmlRunnerHooks:
|
||||
Utils.raise_with_error(
|
||||
"Failed to set both GH commit status and PR comment with Workflow Status, cannot proceed"
|
||||
)
|
||||
|
||||
if env.PR_NUMBER:
|
||||
commits = _get_pr_commits(env.PR_NUMBER)
|
||||
# TODO: upload commits data to s3 to visualise it on a report page
|
||||
print(commits)
|
||||
# TODO: enable for branch, add commit number limiting
|
||||
GitCommit.update_s3_data()
|
||||
|
||||
@classmethod
|
||||
def pre_run(cls, _workflow, _job):
|
||||
result = Result.from_fs(_job.name)
|
||||
S3.copy_result_from_s3(
|
||||
Result.file_name_static(_workflow.name),
|
||||
)
|
||||
workflow_result = Result.from_fs(_workflow.name)
|
||||
workflow_result.update_sub_result(result)
|
||||
S3.copy_result_to_s3(
|
||||
workflow_result,
|
||||
unlock=True,
|
||||
_ResultS3.update_workflow_results(
|
||||
workflow_name=_workflow.name, new_sub_results=result
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@ -132,14 +172,13 @@ class HtmlRunnerHooks:
|
||||
@classmethod
|
||||
def post_run(cls, _workflow, _job, info_errors):
|
||||
result = Result.from_fs(_job.name)
|
||||
env = _Environment.get()
|
||||
S3.copy_result_from_s3(
|
||||
Result.file_name_static(_workflow.name),
|
||||
lock=True,
|
||||
)
|
||||
workflow_result = Result.from_fs(_workflow.name)
|
||||
print(f"Workflow info [{workflow_result.info}], info_errors [{info_errors}]")
|
||||
_ResultS3.upload_result_files_to_s3(result)
|
||||
_ResultS3.copy_result_to_s3(result)
|
||||
|
||||
env = _Environment.get()
|
||||
|
||||
new_sub_results = [result]
|
||||
new_result_info = ""
|
||||
env_info = env.REPORT_INFO
|
||||
if env_info:
|
||||
print(
|
||||
@ -151,14 +190,8 @@ class HtmlRunnerHooks:
|
||||
info_str = f"{_job.name}:\n"
|
||||
info_str += "\n".join(info_errors)
|
||||
print("Update workflow results with new info")
|
||||
workflow_result.set_info(info_str)
|
||||
new_result_info = info_str
|
||||
|
||||
old_status = workflow_result.status
|
||||
|
||||
S3.upload_result_files_to_s3(result)
|
||||
workflow_result.update_sub_result(result)
|
||||
|
||||
skipped_job_results = []
|
||||
if not result.is_ok():
|
||||
print(
|
||||
"Current job failed - find dependee jobs in the workflow and set their statuses to skipped"
|
||||
@ -171,7 +204,7 @@ class HtmlRunnerHooks:
|
||||
print(
|
||||
f"NOTE: Set job [{dependee_job.name}] status to [{Result.Status.SKIPPED}] due to current failure"
|
||||
)
|
||||
skipped_job_results.append(
|
||||
new_sub_results.append(
|
||||
Result(
|
||||
name=dependee_job.name,
|
||||
status=Result.Status.SKIPPED,
|
||||
@ -179,20 +212,18 @@ class HtmlRunnerHooks:
|
||||
+ f" [{_job.name}]",
|
||||
)
|
||||
)
|
||||
for skipped_job_result in skipped_job_results:
|
||||
workflow_result.update_sub_result(skipped_job_result)
|
||||
|
||||
S3.copy_result_to_s3(
|
||||
workflow_result,
|
||||
unlock=True,
|
||||
)
|
||||
if workflow_result.status != old_status:
|
||||
print(
|
||||
f"Update GH commit status [{result.name}]: [{old_status} -> {workflow_result.status}], link [{workflow_result.html_link}]"
|
||||
updated_status = _ResultS3.update_workflow_results(
|
||||
new_info=new_result_info,
|
||||
new_sub_results=new_sub_results,
|
||||
workflow_name=_workflow.name,
|
||||
)
|
||||
|
||||
if updated_status:
|
||||
print(f"Update GH commit status [{result.name}]: [{updated_status}]")
|
||||
GH.post_commit_status(
|
||||
name=workflow_result.name,
|
||||
status=GH.convert_to_gh_status(workflow_result.status),
|
||||
name=_workflow.name,
|
||||
status=GH.convert_to_gh_status(updated_status),
|
||||
description="",
|
||||
url=workflow_result.html_link,
|
||||
url=env.get_report_url(settings=Settings, latest=True),
|
||||
)
|
||||
|
@ -52,30 +52,58 @@ class Job:
|
||||
self,
|
||||
parameter: Optional[List[Any]] = None,
|
||||
runs_on: Optional[List[List[str]]] = None,
|
||||
provides: Optional[List[List[str]]] = None,
|
||||
requires: Optional[List[List[str]]] = None,
|
||||
timeout: Optional[List[int]] = None,
|
||||
):
|
||||
assert (
|
||||
parameter or runs_on
|
||||
), "Either :parameter or :runs_on must be non empty list for parametrisation"
|
||||
if runs_on:
|
||||
assert isinstance(runs_on, list) and isinstance(runs_on[0], list)
|
||||
if not parameter:
|
||||
parameter = [None] * len(runs_on)
|
||||
if not runs_on:
|
||||
runs_on = [None] * len(parameter)
|
||||
if not timeout:
|
||||
timeout = [None] * len(parameter)
|
||||
if not provides:
|
||||
provides = [None] * len(parameter)
|
||||
if not requires:
|
||||
requires = [None] * len(parameter)
|
||||
assert (
|
||||
len(parameter) == len(runs_on) == len(timeout)
|
||||
), "Parametrization lists must be of the same size"
|
||||
len(parameter)
|
||||
== len(runs_on)
|
||||
== len(timeout)
|
||||
== len(provides)
|
||||
== len(requires)
|
||||
), f"Parametrization lists must be of the same size [{len(parameter)}, {len(runs_on)}, {len(timeout)}, {len(provides)}, {len(requires)}]"
|
||||
|
||||
res = []
|
||||
for parameter_, runs_on_, timeout_ in zip(parameter, runs_on, timeout):
|
||||
for parameter_, runs_on_, timeout_, provides_, requires_ in zip(
|
||||
parameter, runs_on, timeout, provides, requires
|
||||
):
|
||||
obj = copy.deepcopy(self)
|
||||
assert (
|
||||
not obj.provides
|
||||
), "Job.Config.provides must be empty for parametrized jobs"
|
||||
if parameter_:
|
||||
obj.parameter = parameter_
|
||||
obj.command = obj.command.format(PARAMETER=parameter_)
|
||||
if runs_on_:
|
||||
obj.runs_on = runs_on_
|
||||
if timeout_:
|
||||
obj.timeout = timeout_
|
||||
if provides_:
|
||||
assert (
|
||||
not obj.provides
|
||||
), "Job.Config.provides must be empty for parametrized jobs"
|
||||
obj.provides = provides_
|
||||
if requires_:
|
||||
assert (
|
||||
not obj.requires
|
||||
), "Job.Config.requires and parametrize(requires=...) are both set"
|
||||
obj.requires = requires_
|
||||
obj.name = obj.get_job_name_with_parameter()
|
||||
res.append(obj)
|
||||
return res
|
||||
@ -84,13 +112,16 @@ class Job:
|
||||
name, parameter, runs_on = self.name, self.parameter, self.runs_on
|
||||
res = name
|
||||
name_params = []
|
||||
if parameter:
|
||||
if isinstance(parameter, list) or isinstance(parameter, dict):
|
||||
name_params.append(json.dumps(parameter))
|
||||
elif parameter is not None:
|
||||
else:
|
||||
name_params.append(parameter)
|
||||
if runs_on:
|
||||
elif runs_on:
|
||||
assert isinstance(runs_on, list)
|
||||
name_params.append(json.dumps(runs_on))
|
||||
else:
|
||||
assert False
|
||||
if name_params:
|
||||
name_params = [str(param) for param in name_params]
|
||||
res += f" ({', '.join(name_params)})"
|
||||
|
@ -89,15 +89,27 @@
|
||||
letter-spacing: -0.5px;
|
||||
}
|
||||
|
||||
.dropdown-value {
|
||||
width: 100px;
|
||||
font-weight: normal;
|
||||
font-family: inherit;
|
||||
background-color: transparent;
|
||||
color: inherit;
|
||||
/*border: none;*/
|
||||
/*outline: none;*/
|
||||
/*cursor: pointer;*/
|
||||
}
|
||||
|
||||
#result-container {
|
||||
background-color: var(--tile-background);
|
||||
margin-left: calc(var(--status-width) + 20px);
|
||||
padding: 20px;
|
||||
padding: 0;
|
||||
box-sizing: border-box;
|
||||
text-align: center;
|
||||
font-size: 18px;
|
||||
font-weight: normal;
|
||||
flex-grow: 1;
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
|
||||
#footer {
|
||||
@ -189,10 +201,7 @@
|
||||
}
|
||||
|
||||
th.name-column, td.name-column {
|
||||
max-width: 400px; /* Set the maximum width for the column */
|
||||
white-space: nowrap; /* Prevent text from wrapping */
|
||||
overflow: hidden; /* Hide the overflowed text */
|
||||
text-overflow: ellipsis; /* Show ellipsis (...) for overflowed text */
|
||||
min-width: 350px;
|
||||
}
|
||||
|
||||
th.status-column, td.status-column {
|
||||
@ -282,6 +291,12 @@
|
||||
}
|
||||
}
|
||||
|
||||
function updateUrlParameter(paramName, paramValue) {
|
||||
const url = new URL(window.location.href);
|
||||
url.searchParams.set(paramName, paramValue);
|
||||
window.location.href = url.toString();
|
||||
}
|
||||
|
||||
// Attach the toggle function to the click event of the icon
|
||||
document.getElementById('theme-toggle').addEventListener('click', toggleTheme);
|
||||
|
||||
@ -291,14 +306,14 @@
|
||||
const monthNames = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
|
||||
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
const month = monthNames[date.getMonth()];
|
||||
const year = date.getFullYear();
|
||||
//const year = date.getFullYear();
|
||||
const hours = String(date.getHours()).padStart(2, '0');
|
||||
const minutes = String(date.getMinutes()).padStart(2, '0');
|
||||
const seconds = String(date.getSeconds()).padStart(2, '0');
|
||||
//const milliseconds = String(date.getMilliseconds()).padStart(2, '0');
|
||||
|
||||
return showDate
|
||||
? `${day}-${month}-${year} ${hours}:${minutes}:${seconds}`
|
||||
? `${day}'${month} ${hours}:${minutes}:${seconds}`
|
||||
: `${hours}:${minutes}:${seconds}`;
|
||||
}
|
||||
|
||||
@ -328,7 +343,7 @@
|
||||
const milliseconds = Math.floor((duration % 1) * 1000);
|
||||
|
||||
const formattedSeconds = String(seconds);
|
||||
const formattedMilliseconds = String(milliseconds).padStart(3, '0');
|
||||
const formattedMilliseconds = String(milliseconds).padStart(2, '0').slice(-2);
|
||||
|
||||
return `${formattedSeconds}.${formattedMilliseconds}`;
|
||||
}
|
||||
@ -346,8 +361,7 @@
|
||||
return 'status-other';
|
||||
}
|
||||
|
||||
function addKeyValueToStatus(key, value) {
|
||||
|
||||
function addKeyValueToStatus(key, value, options = null) {
|
||||
const statusContainer = document.getElementById('status-container');
|
||||
|
||||
let keyValuePair = document.createElement('div');
|
||||
@ -357,12 +371,40 @@
|
||||
keyElement.className = 'json-key';
|
||||
keyElement.textContent = key + ':';
|
||||
|
||||
const valueElement = document.createElement('div');
|
||||
valueElement.className = 'json-value';
|
||||
valueElement.textContent = value;
|
||||
let valueElement;
|
||||
|
||||
keyValuePair.appendChild(keyElement)
|
||||
keyValuePair.appendChild(valueElement)
|
||||
if (options) {
|
||||
// Create dropdown if options are provided
|
||||
valueElement = document.createElement('select');
|
||||
valueElement.className = 'dropdown-value';
|
||||
|
||||
options.forEach(optionValue => {
|
||||
const option = document.createElement('option');
|
||||
option.value = optionValue;
|
||||
option.textContent = optionValue.slice(0, 10);
|
||||
|
||||
// Set the initially selected option
|
||||
if (optionValue === value) {
|
||||
option.selected = true;
|
||||
}
|
||||
|
||||
valueElement.appendChild(option);
|
||||
});
|
||||
|
||||
// Update the URL parameter when the selected value changes
|
||||
valueElement.addEventListener('change', (event) => {
|
||||
const selectedValue = event.target.value;
|
||||
updateUrlParameter(key, selectedValue);
|
||||
});
|
||||
} else {
|
||||
// Create a simple text display if no options are provided
|
||||
valueElement = document.createElement('div');
|
||||
valueElement.className = 'json-value';
|
||||
valueElement.textContent = value || 'N/A'; // Display 'N/A' if value is null
|
||||
}
|
||||
|
||||
keyValuePair.appendChild(keyElement);
|
||||
keyValuePair.appendChild(valueElement);
|
||||
statusContainer.appendChild(keyValuePair);
|
||||
}
|
||||
|
||||
@ -486,12 +528,12 @@
|
||||
const columns = ['name', 'status', 'start_time', 'duration', 'info'];
|
||||
|
||||
const columnSymbols = {
|
||||
name: '📂',
|
||||
status: '✔️',
|
||||
name: '🗂️',
|
||||
status: '🧾',
|
||||
start_time: '🕒',
|
||||
duration: '⏳',
|
||||
info: 'ℹ️',
|
||||
files: '📄'
|
||||
info: '📝',
|
||||
files: '📎'
|
||||
};
|
||||
|
||||
function createResultsTable(results, nest_level) {
|
||||
@ -500,16 +542,14 @@
|
||||
const thead = document.createElement('thead');
|
||||
const tbody = document.createElement('tbody');
|
||||
|
||||
// Get the current URL parameters
|
||||
const currentUrl = new URL(window.location.href);
|
||||
|
||||
// Create table headers based on the fixed columns
|
||||
const headerRow = document.createElement('tr');
|
||||
columns.forEach(column => {
|
||||
const th = document.createElement('th');
|
||||
th.textContent = th.textContent = columnSymbols[column] || column;
|
||||
th.textContent = columnSymbols[column] || column;
|
||||
th.style.cursor = 'pointer'; // Make headers clickable
|
||||
th.addEventListener('click', () => sortTable(results, column, tbody, nest_level)); // Add click event to sort the table
|
||||
th.setAttribute('data-sort-direction', 'asc'); // Default sort direction
|
||||
th.addEventListener('click', () => sortTable(results, column, columnSymbols[column] || column, tbody, nest_level, columns)); // Add click event to sort the table
|
||||
headerRow.appendChild(th);
|
||||
});
|
||||
thead.appendChild(headerRow);
|
||||
@ -561,8 +601,7 @@
|
||||
td.classList.add('time-column');
|
||||
td.textContent = value ? formatDuration(value) : '';
|
||||
} else if (column === 'info') {
|
||||
// For info and other columns, just display the value
|
||||
td.textContent = value || '';
|
||||
td.textContent = value.includes('\n') ? '↵' : (value || '');
|
||||
td.classList.add('info-column');
|
||||
}
|
||||
|
||||
@ -573,39 +612,33 @@
|
||||
});
|
||||
}
|
||||
|
||||
function sortTable(results, key, tbody, nest_level) {
|
||||
function sortTable(results, column, key, tbody, nest_level, columns) {
|
||||
// Find the table header element for the given key
|
||||
let th = null;
|
||||
const tableHeaders = document.querySelectorAll('th'); // Select all table headers
|
||||
tableHeaders.forEach(header => {
|
||||
if (header.textContent.trim().toLowerCase() === key.toLowerCase()) {
|
||||
th = header;
|
||||
}
|
||||
});
|
||||
const tableHeaders = document.querySelectorAll('th');
|
||||
let th = Array.from(tableHeaders).find(header => header.textContent === key);
|
||||
|
||||
if (!th) {
|
||||
console.error(`No table header found for key: ${key}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Determine the current sort direction
|
||||
let ascending = th.getAttribute('data-sort-direction') === 'asc' ? false : true;
|
||||
const ascending = th.getAttribute('data-sort-direction') === 'asc';
|
||||
th.setAttribute('data-sort-direction', ascending ? 'desc' : 'asc');
|
||||
|
||||
// Toggle the sort direction for the next click
|
||||
th.setAttribute('data-sort-direction', ascending ? 'asc' : 'desc');
|
||||
|
||||
// Sort the results array by the given key
|
||||
results.sort((a, b) => {
|
||||
if (a[key] < b[key]) return ascending ? -1 : 1;
|
||||
if (a[key] > b[key]) return ascending ? 1 : -1;
|
||||
if (a[column] < b[column]) return ascending ? -1 : 1;
|
||||
if (a[column] > b[column]) return ascending ? 1 : -1;
|
||||
return 0;
|
||||
});
|
||||
|
||||
// Clear the existing rows in tbody
|
||||
tbody.innerHTML = '';
|
||||
|
||||
// Re-populate the table with sorted data
|
||||
populateTableRows(tbody, results, columns, nest_level);
|
||||
}
|
||||
|
||||
function loadJSON(PR, sha, nameParams) {
|
||||
function loadResultsJSON(PR, sha, nameParams) {
|
||||
const infoElement = document.getElementById('info-container');
|
||||
let lastModifiedTime = null;
|
||||
const task = nameParams[0].toLowerCase();
|
||||
@ -630,12 +663,9 @@
|
||||
let targetData = navigatePath(data, nameParams);
|
||||
let nest_level = nameParams.length;
|
||||
|
||||
if (targetData) {
|
||||
infoElement.style.display = 'none';
|
||||
|
||||
// Handle footer links if present
|
||||
if (Array.isArray(data.aux_links) && data.aux_links.length > 0) {
|
||||
data.aux_links.forEach(link => {
|
||||
// Add footer links from top-level Result
|
||||
if (Array.isArray(data.links) && data.links.length > 0) {
|
||||
data.links.forEach(link => {
|
||||
const a = document.createElement('a');
|
||||
a.href = link;
|
||||
a.textContent = link.split('/').pop();
|
||||
@ -644,6 +674,10 @@
|
||||
});
|
||||
}
|
||||
|
||||
if (targetData) {
|
||||
//infoElement.style.display = 'none';
|
||||
infoElement.innerHTML = (targetData.info || '').replace(/\n/g, '<br>');
|
||||
|
||||
addStatusToStatus(targetData.status, targetData.start_time, targetData.duration)
|
||||
|
||||
// Handle links
|
||||
@ -721,22 +755,62 @@
|
||||
}
|
||||
});
|
||||
|
||||
let path_commits_json = '';
|
||||
let commitsArray = [];
|
||||
|
||||
if (PR) {
|
||||
addKeyValueToStatus("PR", PR)
|
||||
addKeyValueToStatus("PR", PR);
|
||||
const baseUrl = window.location.origin + window.location.pathname.replace('/json.html', '');
|
||||
path_commits_json = `${baseUrl}/${encodeURIComponent(PR)}/commits.json`;
|
||||
} else {
|
||||
console.error("TODO")
|
||||
// Placeholder for a different path when PR is missing
|
||||
console.error("PR parameter is missing. Setting alternate commits path.");
|
||||
path_commits_json = '/path/to/alternative/commits.json';
|
||||
}
|
||||
addKeyValueToStatus("sha", sha);
|
||||
|
||||
function loadCommitsArray(path) {
|
||||
return fetch(path, { cache: "no-cache" })
|
||||
.then(response => {
|
||||
if (!response.ok) {
|
||||
console.error(`HTTP error! status: ${response.status}`)
|
||||
return [];
|
||||
}
|
||||
return response.json();
|
||||
})
|
||||
.then(data => {
|
||||
if (Array.isArray(data) && data.every(item => typeof item === 'object' && item.hasOwnProperty('sha'))) {
|
||||
return data.map(item => item.sha);
|
||||
} else {
|
||||
throw new Error('Invalid data format: expected array of objects with a "sha" key');
|
||||
}
|
||||
})
|
||||
.catch(error => {
|
||||
console.error('Error loading commits JSON:', error);
|
||||
return []; // Return an empty array if an error occurs
|
||||
});
|
||||
}
|
||||
|
||||
loadCommitsArray(path_commits_json)
|
||||
.then(data => {
|
||||
commitsArray = data;
|
||||
})
|
||||
.finally(() => {
|
||||
// Proceed with the rest of the initialization
|
||||
addKeyValueToStatus("sha", sha || "latest", commitsArray.concat(["latest"]));
|
||||
|
||||
if (nameParams[1]) {
|
||||
addKeyValueToStatus("job", nameParams[1]);
|
||||
}
|
||||
addKeyValueToStatus("workflow", nameParams[0]);
|
||||
|
||||
// Check if all required parameters are present to load JSON
|
||||
if (PR && sha && root_name) {
|
||||
loadJSON(PR, sha, nameParams);
|
||||
const shaToLoad = (sha === 'latest') ? commitsArray[commitsArray.length - 1] : sha;
|
||||
loadResultsJSON(PR, shaToLoad, nameParams);
|
||||
} else {
|
||||
document.getElementById('title').textContent = 'Error: Missing required URL parameters: PR, sha, or name_0';
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
window.onload = init;
|
||||
|
@ -1,11 +1,10 @@
|
||||
import copy
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict
|
||||
|
||||
from praktika import Job
|
||||
from praktika._settings import _USER_DEFINED_SETTINGS, _Settings
|
||||
from praktika.utils import ContextManager, Utils
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import Utils
|
||||
|
||||
|
||||
def _get_workflows(name=None, file=None):
|
||||
@ -14,14 +13,13 @@ def _get_workflows(name=None, file=None):
|
||||
"""
|
||||
res = []
|
||||
|
||||
with ContextManager.cd():
|
||||
directory = Path(_Settings.WORKFLOWS_DIRECTORY)
|
||||
directory = Path(Settings.WORKFLOWS_DIRECTORY)
|
||||
for py_file in directory.glob("*.py"):
|
||||
if file and file not in str(py_file):
|
||||
continue
|
||||
module_name = py_file.name.removeprefix(".py")
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name, f"{_Settings.WORKFLOWS_DIRECTORY}/{module_name}"
|
||||
module_name, f"{Settings.WORKFLOWS_DIRECTORY}/{module_name}"
|
||||
)
|
||||
assert spec
|
||||
foo = importlib.util.module_from_spec(spec)
|
||||
@ -58,7 +56,6 @@ def _update_workflow_artifacts(workflow):
|
||||
artifact_job = {}
|
||||
for job in workflow.jobs:
|
||||
for artifact_name in job.provides:
|
||||
assert artifact_name not in artifact_job
|
||||
artifact_job[artifact_name] = job.name
|
||||
for artifact in workflow.artifacts:
|
||||
artifact._provided_by = artifact_job[artifact.name]
|
||||
@ -108,30 +105,3 @@ def _update_workflow_with_native_jobs(workflow):
|
||||
for job in workflow.jobs:
|
||||
aux_job.requires.append(job.name)
|
||||
workflow.jobs.append(aux_job)
|
||||
|
||||
|
||||
def _get_user_settings() -> Dict[str, Any]:
|
||||
"""
|
||||
Gets user's settings
|
||||
"""
|
||||
res = {} # type: Dict[str, Any]
|
||||
|
||||
directory = Path(_Settings.SETTINGS_DIRECTORY)
|
||||
for py_file in directory.glob("*.py"):
|
||||
module_name = py_file.name.removeprefix(".py")
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name, f"{_Settings.SETTINGS_DIRECTORY}/{module_name}"
|
||||
)
|
||||
assert spec
|
||||
foo = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader
|
||||
spec.loader.exec_module(foo)
|
||||
for setting in _USER_DEFINED_SETTINGS:
|
||||
try:
|
||||
value = getattr(foo, setting)
|
||||
res[setting] = value
|
||||
print(f"Apply user defined setting [{setting} = {value}]")
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return res
|
||||
|
@ -10,9 +10,8 @@ from praktika.gh import GH
|
||||
from praktika.hook_cache import CacheRunnerHooks
|
||||
from praktika.hook_html import HtmlRunnerHooks
|
||||
from praktika.mangle import _get_workflows
|
||||
from praktika.result import Result, ResultInfo
|
||||
from praktika.result import Result, ResultInfo, _ResultS3
|
||||
from praktika.runtime import RunConfig
|
||||
from praktika.s3 import S3
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import Shell, Utils
|
||||
|
||||
@ -151,7 +150,7 @@ def _config_workflow(workflow: Workflow.Config, job_name):
|
||||
status = Result.Status.ERROR
|
||||
print("ERROR: ", info)
|
||||
else:
|
||||
Shell.check(f"{Settings.PYTHON_INTERPRETER} -m praktika --generate")
|
||||
assert Shell.check(f"{Settings.PYTHON_INTERPRETER} -m praktika yaml")
|
||||
exit_code, output, err = Shell.get_res_stdout_stderr(
|
||||
f"git diff-index HEAD -- {Settings.WORKFLOW_PATH_PREFIX}"
|
||||
)
|
||||
@ -225,6 +224,7 @@ def _config_workflow(workflow: Workflow.Config, job_name):
|
||||
cache_success=[],
|
||||
cache_success_base64=[],
|
||||
cache_artifacts={},
|
||||
cache_jobs={},
|
||||
).dump()
|
||||
|
||||
# checks:
|
||||
@ -250,6 +250,9 @@ def _config_workflow(workflow: Workflow.Config, job_name):
|
||||
info_lines.append(job_name + ": " + info)
|
||||
results.append(result_)
|
||||
|
||||
if workflow.enable_merge_commit:
|
||||
assert False, "NOT implemented"
|
||||
|
||||
# config:
|
||||
if workflow.dockers:
|
||||
print("Calculate docker's digests")
|
||||
@ -307,9 +310,8 @@ def _finish_workflow(workflow, job_name):
|
||||
print(env.get_needs_statuses())
|
||||
|
||||
print("Check Workflow results")
|
||||
S3.copy_result_from_s3(
|
||||
_ResultS3.copy_result_from_s3(
|
||||
Result.file_name_static(workflow.name),
|
||||
lock=False,
|
||||
)
|
||||
workflow_result = Result.from_fs(workflow.name)
|
||||
|
||||
@ -339,10 +341,12 @@ def _finish_workflow(workflow, job_name):
|
||||
f"NOTE: Result for [{result.name}] has not ok status [{result.status}]"
|
||||
)
|
||||
ready_for_merge_status = Result.Status.FAILED
|
||||
failed_results.append(result.name.split("(", maxsplit=1)[0]) # cut name
|
||||
failed_results.append(result.name)
|
||||
|
||||
if failed_results:
|
||||
ready_for_merge_description = f"failed: {', '.join(failed_results)}"
|
||||
ready_for_merge_description = (
|
||||
f'Failed {len(failed_results)} "Required for Merge" jobs'
|
||||
)
|
||||
|
||||
if not GH.post_commit_status(
|
||||
name=Settings.READY_FOR_MERGE_STATUS_NAME + f" [{workflow.name}]",
|
||||
@ -354,15 +358,12 @@ def _finish_workflow(workflow, job_name):
|
||||
env.add_info(ResultInfo.GH_STATUS_ERROR)
|
||||
|
||||
if update_final_report:
|
||||
S3.copy_result_to_s3(
|
||||
_ResultS3.copy_result_to_s3(
|
||||
workflow_result,
|
||||
unlock=False,
|
||||
) # no lock - no unlock
|
||||
|
||||
Result.from_fs(job_name).set_status(Result.Status.SUCCESS).set_info(
|
||||
ready_for_merge_description
|
||||
)
|
||||
|
||||
Result.from_fs(job_name).set_status(Result.Status.SUCCESS)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
job_name = sys.argv[1]
|
||||
|
@ -1,12 +1,13 @@
|
||||
import dataclasses
|
||||
import datetime
|
||||
import sys
|
||||
from collections.abc import Container
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, List, Optional
|
||||
from typing import Any, Dict, List, Optional, Union
|
||||
|
||||
from praktika._environment import _Environment
|
||||
from praktika._settings import _Settings
|
||||
from praktika.cache import Cache
|
||||
from praktika.s3 import S3
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import ContextManager, MetaClasses, Shell, Utils
|
||||
|
||||
|
||||
@ -27,10 +28,6 @@ class Result(MetaClasses.Serializable):
|
||||
files (List[str]): A list of file paths or names related to the result.
|
||||
links (List[str]): A list of URLs related to the result (e.g., links to reports or resources).
|
||||
info (str): Additional information about the result. Free-form text.
|
||||
# TODO: rename
|
||||
aux_links (List[str]): A list of auxiliary links that provide additional context for the result.
|
||||
# TODO: remove
|
||||
html_link (str): A direct link to an HTML representation of the result (e.g., a detailed report page).
|
||||
|
||||
Inner Class:
|
||||
Status: Defines possible statuses for the task, such as "success", "failure", etc.
|
||||
@ -52,8 +49,6 @@ class Result(MetaClasses.Serializable):
|
||||
files: List[str] = dataclasses.field(default_factory=list)
|
||||
links: List[str] = dataclasses.field(default_factory=list)
|
||||
info: str = ""
|
||||
aux_links: List[str] = dataclasses.field(default_factory=list)
|
||||
html_link: str = ""
|
||||
|
||||
@staticmethod
|
||||
def create_from(
|
||||
@ -62,14 +57,15 @@ class Result(MetaClasses.Serializable):
|
||||
stopwatch: Utils.Stopwatch = None,
|
||||
status="",
|
||||
files=None,
|
||||
info="",
|
||||
info: Union[List[str], str] = "",
|
||||
with_info_from_results=True,
|
||||
):
|
||||
if isinstance(status, bool):
|
||||
status = Result.Status.SUCCESS if status else Result.Status.FAILED
|
||||
if not results and not status:
|
||||
print("ERROR: Either .results or .status must be provided")
|
||||
raise
|
||||
Utils.raise_with_error(
|
||||
f"Either .results ({results}) or .status ({status}) must be provided"
|
||||
)
|
||||
if not name:
|
||||
name = _Environment.get().JOB_NAME
|
||||
if not name:
|
||||
@ -78,10 +74,10 @@ class Result(MetaClasses.Serializable):
|
||||
result_status = status or Result.Status.SUCCESS
|
||||
infos = []
|
||||
if info:
|
||||
if isinstance(info, Container):
|
||||
infos += info
|
||||
if isinstance(info, str):
|
||||
infos += [info]
|
||||
else:
|
||||
infos.append(info)
|
||||
infos += info
|
||||
if results and not status:
|
||||
for result in results:
|
||||
if result.status not in (Result.Status.SUCCESS, Result.Status.FAILED):
|
||||
@ -112,7 +108,7 @@ class Result(MetaClasses.Serializable):
|
||||
return self.status not in (Result.Status.PENDING, Result.Status.RUNNING)
|
||||
|
||||
def is_running(self):
|
||||
return self.status not in (Result.Status.RUNNING,)
|
||||
return self.status in (Result.Status.RUNNING,)
|
||||
|
||||
def is_ok(self):
|
||||
return self.status in (Result.Status.SKIPPED, Result.Status.SUCCESS)
|
||||
@ -155,7 +151,7 @@ class Result(MetaClasses.Serializable):
|
||||
|
||||
@classmethod
|
||||
def file_name_static(cls, name):
|
||||
return f"{_Settings.TEMP_DIR}/result_{Utils.normalize_string(name)}.json"
|
||||
return f"{Settings.TEMP_DIR}/result_{Utils.normalize_string(name)}.json"
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, obj: Dict[str, Any]) -> "Result":
|
||||
@ -180,6 +176,11 @@ class Result(MetaClasses.Serializable):
|
||||
)
|
||||
return self
|
||||
|
||||
def set_timing(self, stopwatch: Utils.Stopwatch):
|
||||
self.start_time = stopwatch.start_time
|
||||
self.duration = stopwatch.duration
|
||||
return self
|
||||
|
||||
def update_sub_result(self, result: "Result"):
|
||||
assert self.results, "BUG?"
|
||||
for i, result_ in enumerate(self.results):
|
||||
@ -233,7 +234,7 @@ class Result(MetaClasses.Serializable):
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def generate_skipped(cls, name, results=None):
|
||||
def generate_skipped(cls, name, cache_record: Cache.CacheRecord, results=None):
|
||||
return Result(
|
||||
name=name,
|
||||
status=Result.Status.SKIPPED,
|
||||
@ -242,7 +243,7 @@ class Result(MetaClasses.Serializable):
|
||||
results=results or [],
|
||||
files=[],
|
||||
links=[],
|
||||
info="from cache",
|
||||
info=f"from cache: sha [{cache_record.sha}], pr/branch [{cache_record.pr_number or cache_record.branch}]",
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@ -276,7 +277,7 @@ class Result(MetaClasses.Serializable):
|
||||
|
||||
# Set log file path if logging is enabled
|
||||
log_file = (
|
||||
f"{_Settings.TEMP_DIR}/{Utils.normalize_string(name)}.log"
|
||||
f"{Settings.TEMP_DIR}/{Utils.normalize_string(name)}.log"
|
||||
if with_log
|
||||
else None
|
||||
)
|
||||
@ -318,18 +319,35 @@ class Result(MetaClasses.Serializable):
|
||||
files=[log_file] if log_file else None,
|
||||
)
|
||||
|
||||
def finish_job_accordingly(self):
|
||||
def complete_job(self):
|
||||
self.dump()
|
||||
if not self.is_ok():
|
||||
print("ERROR: Job Failed")
|
||||
for result in self.results:
|
||||
if not result.is_ok():
|
||||
print("Failed checks:")
|
||||
print(" | ", result)
|
||||
print(self.to_stdout_formatted())
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("ok")
|
||||
|
||||
def to_stdout_formatted(self, indent="", res=""):
|
||||
if self.is_ok():
|
||||
return res
|
||||
|
||||
res += f"{indent}Task [{self.name}] failed.\n"
|
||||
fail_info = ""
|
||||
sub_indent = indent + " "
|
||||
|
||||
if not self.results:
|
||||
if not self.is_ok():
|
||||
fail_info += f"{sub_indent}{self.name}:\n"
|
||||
for line in self.info.splitlines():
|
||||
fail_info += f"{sub_indent}{sub_indent}{line}\n"
|
||||
return res + fail_info
|
||||
|
||||
for sub_result in self.results:
|
||||
res = sub_result.to_stdout_formatted(sub_indent, res)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
class ResultInfo:
|
||||
SETUP_ENV_JOB_FAILED = (
|
||||
@ -352,3 +370,202 @@ class ResultInfo:
|
||||
)
|
||||
|
||||
S3_ERROR = "S3 call failure"
|
||||
|
||||
|
||||
class _ResultS3:
|
||||
|
||||
@classmethod
|
||||
def copy_result_to_s3(cls, result, unlock=False):
|
||||
result.dump()
|
||||
env = _Environment.get()
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
|
||||
s3_path_full = f"{s3_path}/{Path(result.file_name()).name}"
|
||||
url = S3.copy_file_to_s3(s3_path=s3_path, local_path=result.file_name())
|
||||
# if unlock:
|
||||
# if not cls.unlock(s3_path_full):
|
||||
# print(f"ERROR: File [{s3_path_full}] unlock failure")
|
||||
# assert False # TODO: investigate
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
def copy_result_from_s3(cls, local_path, lock=False):
|
||||
env = _Environment.get()
|
||||
file_name = Path(local_path).name
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/{file_name}"
|
||||
# if lock:
|
||||
# cls.lock(s3_path)
|
||||
if not S3.copy_file_from_s3(s3_path=s3_path, local_path=local_path):
|
||||
print(f"ERROR: failed to cp file [{s3_path}] from s3")
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def copy_result_from_s3_with_version(cls, local_path):
|
||||
env = _Environment.get()
|
||||
file_name = Path(local_path).name
|
||||
local_dir = Path(local_path).parent
|
||||
file_name_pattern = f"{file_name}_*"
|
||||
for file_path in local_dir.glob(file_name_pattern):
|
||||
file_path.unlink()
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/"
|
||||
if not S3.copy_file_from_s3_matching_pattern(
|
||||
s3_path=s3_path, local_path=local_dir, include=file_name_pattern
|
||||
):
|
||||
print(f"ERROR: failed to cp file [{s3_path}] from s3")
|
||||
raise
|
||||
result_files = []
|
||||
for file_path in local_dir.glob(file_name_pattern):
|
||||
result_files.append(file_path)
|
||||
assert result_files, "No result files found"
|
||||
result_files.sort()
|
||||
version = int(result_files[-1].name.split("_")[-1])
|
||||
Shell.check(f"cp {result_files[-1]} {local_path}", strict=True, verbose=True)
|
||||
return version
|
||||
|
||||
@classmethod
|
||||
def copy_result_to_s3_with_version(cls, result, version):
|
||||
result.dump()
|
||||
filename = Path(result.file_name()).name
|
||||
file_name_versioned = f"{filename}_{str(version).zfill(3)}"
|
||||
env = _Environment.get()
|
||||
s3_path_versioned = (
|
||||
f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/{file_name_versioned}"
|
||||
)
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/"
|
||||
if version == 0:
|
||||
S3.clean_s3_directory(s3_path=s3_path)
|
||||
if not S3.put(
|
||||
s3_path=s3_path_versioned,
|
||||
local_path=result.file_name(),
|
||||
if_none_matched=True,
|
||||
):
|
||||
print("Failed to put versioned Result")
|
||||
return False
|
||||
if not S3.put(s3_path=s3_path, local_path=result.file_name()):
|
||||
print("Failed to put non-versioned Result")
|
||||
return True
|
||||
|
||||
# @classmethod
|
||||
# def lock(cls, s3_path, level=0):
|
||||
# env = _Environment.get()
|
||||
# s3_path_lock = s3_path + f".lock"
|
||||
# file_path_lock = f"{Settings.TEMP_DIR}/{Path(s3_path_lock).name}"
|
||||
# assert Shell.check(
|
||||
# f"echo '''{env.JOB_NAME}''' > {file_path_lock}", verbose=True
|
||||
# ), "Never"
|
||||
#
|
||||
# i = 20
|
||||
# meta = S3.head_object(s3_path_lock)
|
||||
# while meta:
|
||||
# locked_by_job = meta.get("Metadata", {"job": ""}).get("job", "")
|
||||
# if locked_by_job:
|
||||
# decoded_bytes = base64.b64decode(locked_by_job)
|
||||
# locked_by_job = decoded_bytes.decode("utf-8")
|
||||
# print(
|
||||
# f"WARNING: Failed to acquire lock, meta [{meta}], job [{locked_by_job}] - wait"
|
||||
# )
|
||||
# i -= 5
|
||||
# if i < 0:
|
||||
# info = f"ERROR: lock acquire failure - unlock forcefully"
|
||||
# print(info)
|
||||
# env.add_info(info)
|
||||
# break
|
||||
# time.sleep(5)
|
||||
#
|
||||
# metadata = {"job": Utils.to_base64(env.JOB_NAME)}
|
||||
# S3.put(
|
||||
# s3_path=s3_path_lock,
|
||||
# local_path=file_path_lock,
|
||||
# metadata=metadata,
|
||||
# if_none_matched=True,
|
||||
# )
|
||||
# time.sleep(1)
|
||||
# obj = S3.head_object(s3_path_lock)
|
||||
# if not obj or not obj.has_tags(tags=metadata):
|
||||
# print(f"WARNING: locked by another job [{obj}]")
|
||||
# env.add_info("S3 lock file failure")
|
||||
# cls.lock(s3_path, level=level + 1)
|
||||
# print("INFO: lock acquired")
|
||||
#
|
||||
# @classmethod
|
||||
# def unlock(cls, s3_path):
|
||||
# s3_path_lock = s3_path + ".lock"
|
||||
# env = _Environment.get()
|
||||
# obj = S3.head_object(s3_path_lock)
|
||||
# if not obj:
|
||||
# print("ERROR: lock file is removed")
|
||||
# assert False # investigate
|
||||
# elif not obj.has_tags({"job": Utils.to_base64(env.JOB_NAME)}):
|
||||
# print("ERROR: lock file was acquired by another job")
|
||||
# assert False # investigate
|
||||
#
|
||||
# if not S3.delete(s3_path_lock):
|
||||
# print(f"ERROR: File [{s3_path_lock}] delete failure")
|
||||
# print("INFO: lock released")
|
||||
# return True
|
||||
|
||||
@classmethod
|
||||
def upload_result_files_to_s3(cls, result):
|
||||
if result.results:
|
||||
for result_ in result.results:
|
||||
cls.upload_result_files_to_s3(result_)
|
||||
for file in result.files:
|
||||
if not Path(file).is_file():
|
||||
print(f"ERROR: Invalid file [{file}] in [{result.name}] - skip upload")
|
||||
result.info += f"\nWARNING: Result file [{file}] was not found"
|
||||
file_link = S3._upload_file_to_s3(file, upload_to_s3=False)
|
||||
else:
|
||||
is_text = False
|
||||
for text_file_suffix in Settings.TEXT_CONTENT_EXTENSIONS:
|
||||
if file.endswith(text_file_suffix):
|
||||
print(
|
||||
f"File [{file}] matches Settings.TEXT_CONTENT_EXTENSIONS [{Settings.TEXT_CONTENT_EXTENSIONS}] - add text attribute for s3 object"
|
||||
)
|
||||
is_text = True
|
||||
break
|
||||
file_link = S3._upload_file_to_s3(
|
||||
file,
|
||||
upload_to_s3=True,
|
||||
text=is_text,
|
||||
s3_subprefix=Utils.normalize_string(result.name),
|
||||
)
|
||||
result.links.append(file_link)
|
||||
if result.files:
|
||||
print(
|
||||
f"Result files [{result.files}] uploaded to s3 [{result.links[-len(result.files):]}] - clean files list"
|
||||
)
|
||||
result.files = []
|
||||
result.dump()
|
||||
|
||||
@classmethod
|
||||
def update_workflow_results(cls, workflow_name, new_info="", new_sub_results=None):
|
||||
assert new_info or new_sub_results
|
||||
|
||||
attempt = 1
|
||||
prev_status = ""
|
||||
new_status = ""
|
||||
done = False
|
||||
while attempt < 10:
|
||||
version = cls.copy_result_from_s3_with_version(
|
||||
Result.file_name_static(workflow_name)
|
||||
)
|
||||
workflow_result = Result.from_fs(workflow_name)
|
||||
prev_status = workflow_result.status
|
||||
if new_info:
|
||||
workflow_result.set_info(new_info)
|
||||
if new_sub_results:
|
||||
if isinstance(new_sub_results, Result):
|
||||
new_sub_results = [new_sub_results]
|
||||
for result_ in new_sub_results:
|
||||
workflow_result.update_sub_result(result_)
|
||||
new_status = workflow_result.status
|
||||
if cls.copy_result_to_s3_with_version(workflow_result, version=version + 1):
|
||||
done = True
|
||||
break
|
||||
print(f"Attempt [{attempt}] to upload workflow result failed")
|
||||
attempt += 1
|
||||
assert done
|
||||
|
||||
if prev_status != new_status:
|
||||
return new_status
|
||||
else:
|
||||
return None
|
||||
|
@ -19,7 +19,7 @@ from praktika.utils import Shell, TeePopen, Utils
|
||||
|
||||
class Runner:
|
||||
@staticmethod
|
||||
def generate_dummy_environment(workflow, job):
|
||||
def generate_local_run_environment(workflow, job, pr=None, branch=None, sha=None):
|
||||
print("WARNING: Generate dummy env for local test")
|
||||
Shell.check(
|
||||
f"mkdir -p {Settings.TEMP_DIR} {Settings.INPUT_DIR} {Settings.OUTPUT_DIR}"
|
||||
@ -28,9 +28,9 @@ class Runner:
|
||||
WORKFLOW_NAME=workflow.name,
|
||||
JOB_NAME=job.name,
|
||||
REPOSITORY="",
|
||||
BRANCH="",
|
||||
SHA="",
|
||||
PR_NUMBER=-1,
|
||||
BRANCH=branch or Settings.MAIN_BRANCH if not pr else "",
|
||||
SHA=sha or Shell.get_output("git rev-parse HEAD"),
|
||||
PR_NUMBER=pr or -1,
|
||||
EVENT_TYPE="",
|
||||
JOB_OUTPUT_STREAM="",
|
||||
EVENT_FILE_PATH="",
|
||||
@ -52,6 +52,7 @@ class Runner:
|
||||
cache_success=[],
|
||||
cache_success_base64=[],
|
||||
cache_artifacts={},
|
||||
cache_jobs={},
|
||||
)
|
||||
for docker in workflow.dockers:
|
||||
workflow_config.digest_dockers[docker.name] = Digest().calc_docker_digest(
|
||||
@ -80,13 +81,12 @@ class Runner:
|
||||
print("Read GH Environment")
|
||||
env = _Environment.from_env()
|
||||
env.JOB_NAME = job.name
|
||||
env.PARAMETER = job.parameter
|
||||
env.dump()
|
||||
print(env)
|
||||
|
||||
return 0
|
||||
|
||||
def _pre_run(self, workflow, job):
|
||||
def _pre_run(self, workflow, job, local_run=False):
|
||||
env = _Environment.get()
|
||||
|
||||
result = Result(
|
||||
@ -96,6 +96,7 @@ class Runner:
|
||||
)
|
||||
result.dump()
|
||||
|
||||
if not local_run:
|
||||
if workflow.enable_report and job.name != Settings.CI_CONFIG_JOB_NAME:
|
||||
print("Update Job and Workflow Report")
|
||||
HtmlRunnerHooks.pre_run(workflow, job)
|
||||
@ -123,28 +124,48 @@ class Runner:
|
||||
|
||||
return 0
|
||||
|
||||
def _run(self, workflow, job, docker="", no_docker=False, param=None):
|
||||
def _run(self, workflow, job, docker="", no_docker=False, param=None, test=""):
|
||||
# re-set envs for local run
|
||||
env = _Environment.get()
|
||||
env.JOB_NAME = job.name
|
||||
env.dump()
|
||||
|
||||
if param:
|
||||
if not isinstance(param, str):
|
||||
Utils.raise_with_error(
|
||||
f"Custom param for local tests must be of type str, got [{type(param)}]"
|
||||
)
|
||||
env = _Environment.get()
|
||||
env.dump()
|
||||
|
||||
if job.run_in_docker and not no_docker:
|
||||
# TODO: add support for any image, including not from ci config (e.g. ubuntu:latest)
|
||||
docker_tag = RunConfig.from_fs(workflow.name).digest_dockers[
|
||||
job.run_in_docker
|
||||
]
|
||||
docker = docker or f"{job.run_in_docker}:{docker_tag}"
|
||||
cmd = f"docker run --rm --user \"$(id -u):$(id -g)\" -e PYTHONPATH='{Settings.DOCKER_WD}:{Settings.DOCKER_WD}/ci' --volume ./:{Settings.DOCKER_WD} --volume {Settings.TEMP_DIR}:{Settings.TEMP_DIR} --workdir={Settings.DOCKER_WD} {docker} {job.command}"
|
||||
job.run_in_docker, docker_settings = (
|
||||
job.run_in_docker.split("+")[0],
|
||||
job.run_in_docker.split("+")[1:],
|
||||
)
|
||||
from_root = "root" in docker_settings
|
||||
settings = [s for s in docker_settings if s.startswith("--")]
|
||||
if ":" in job.run_in_docker:
|
||||
docker_name, docker_tag = job.run_in_docker.split(":")
|
||||
print(
|
||||
f"WARNING: Job [{job.name}] use custom docker image with a tag - praktika won't control docker version"
|
||||
)
|
||||
else:
|
||||
docker_name, docker_tag = (
|
||||
job.run_in_docker,
|
||||
RunConfig.from_fs(workflow.name).digest_dockers[job.run_in_docker],
|
||||
)
|
||||
docker = docker or f"{docker_name}:{docker_tag}"
|
||||
cmd = f"docker run --rm --name praktika {'--user $(id -u):$(id -g)' if not from_root else ''} -e PYTHONPATH='{Settings.DOCKER_WD}:{Settings.DOCKER_WD}/ci' --volume ./:{Settings.DOCKER_WD} --volume {Settings.TEMP_DIR}:{Settings.TEMP_DIR} --workdir={Settings.DOCKER_WD} {' '.join(settings)} {docker} {job.command}"
|
||||
else:
|
||||
cmd = job.command
|
||||
python_path = os.getenv("PYTHONPATH", ":")
|
||||
os.environ["PYTHONPATH"] = f".:{python_path}"
|
||||
|
||||
if param:
|
||||
print(f"Custom --param [{param}] will be passed to job's script")
|
||||
cmd += f" --param {param}"
|
||||
if test:
|
||||
print(f"Custom --test [{test}] will be passed to job's script")
|
||||
cmd += f" --test {test}"
|
||||
print(f"--- Run command [{cmd}]")
|
||||
|
||||
with TeePopen(cmd, timeout=job.timeout) as process:
|
||||
@ -219,13 +240,10 @@ class Runner:
|
||||
print(info)
|
||||
result.set_info(info).set_status(Result.Status.ERROR).dump()
|
||||
|
||||
if not result.is_ok():
|
||||
result.set_files(files=[Settings.RUN_LOG])
|
||||
result.update_duration().dump()
|
||||
|
||||
if result.info and result.status != Result.Status.SUCCESS:
|
||||
# provide job info to workflow level
|
||||
info_errors.append(result.info)
|
||||
|
||||
if run_exit_code == 0:
|
||||
providing_artifacts = []
|
||||
if job.provides and workflow.artifacts:
|
||||
@ -285,14 +303,24 @@ class Runner:
|
||||
return True
|
||||
|
||||
def run(
|
||||
self, workflow, job, docker="", dummy_env=False, no_docker=False, param=None
|
||||
self,
|
||||
workflow,
|
||||
job,
|
||||
docker="",
|
||||
local_run=False,
|
||||
no_docker=False,
|
||||
param=None,
|
||||
test="",
|
||||
pr=None,
|
||||
sha=None,
|
||||
branch=None,
|
||||
):
|
||||
res = True
|
||||
setup_env_code = -10
|
||||
prerun_code = -10
|
||||
run_code = -10
|
||||
|
||||
if res and not dummy_env:
|
||||
if res and not local_run:
|
||||
print(
|
||||
f"\n\n=== Setup env script [{job.name}], workflow [{workflow.name}] ==="
|
||||
)
|
||||
@ -309,13 +337,15 @@ class Runner:
|
||||
traceback.print_exc()
|
||||
print(f"=== Setup env finished ===\n\n")
|
||||
else:
|
||||
self.generate_dummy_environment(workflow, job)
|
||||
self.generate_local_run_environment(
|
||||
workflow, job, pr=pr, branch=branch, sha=sha
|
||||
)
|
||||
|
||||
if res and not dummy_env:
|
||||
if res and (not local_run or pr or sha or branch):
|
||||
res = False
|
||||
print(f"=== Pre run script [{job.name}], workflow [{workflow.name}] ===")
|
||||
try:
|
||||
prerun_code = self._pre_run(workflow, job)
|
||||
prerun_code = self._pre_run(workflow, job, local_run=local_run)
|
||||
res = prerun_code == 0
|
||||
if not res:
|
||||
print(f"ERROR: Pre-run failed with exit code [{prerun_code}]")
|
||||
@ -329,7 +359,12 @@ class Runner:
|
||||
print(f"=== Run script [{job.name}], workflow [{workflow.name}] ===")
|
||||
try:
|
||||
run_code = self._run(
|
||||
workflow, job, docker=docker, no_docker=no_docker, param=param
|
||||
workflow,
|
||||
job,
|
||||
docker=docker,
|
||||
no_docker=no_docker,
|
||||
param=param,
|
||||
test=test,
|
||||
)
|
||||
res = run_code == 0
|
||||
if not res:
|
||||
@ -339,7 +374,7 @@ class Runner:
|
||||
traceback.print_exc()
|
||||
print(f"=== Run scrip finished ===\n\n")
|
||||
|
||||
if not dummy_env:
|
||||
if not local_run:
|
||||
print(f"=== Post run script [{job.name}], workflow [{workflow.name}] ===")
|
||||
self._post_run(workflow, job, setup_env_code, prerun_code, run_code)
|
||||
print(f"=== Post run scrip finished ===")
|
||||
|
@ -15,17 +15,23 @@ class RunConfig(MetaClasses.Serializable):
|
||||
# there are might be issue with special characters in job names if used directly in yaml syntax - create base64 encoded list to avoid this
|
||||
cache_success_base64: List[str]
|
||||
cache_artifacts: Dict[str, Cache.CacheRecord]
|
||||
cache_jobs: Dict[str, Cache.CacheRecord]
|
||||
sha: str
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, obj):
|
||||
cache_artifacts = obj["cache_artifacts"]
|
||||
cache_jobs = obj["cache_jobs"]
|
||||
cache_artifacts_deserialized = {}
|
||||
cache_jobs_deserialized = {}
|
||||
for artifact_name, cache_artifact in cache_artifacts.items():
|
||||
cache_artifacts_deserialized[artifact_name] = Cache.CacheRecord.from_dict(
|
||||
cache_artifact
|
||||
)
|
||||
obj["cache_artifacts"] = cache_artifacts_deserialized
|
||||
for job_name, cache_jobs in cache_jobs.items():
|
||||
cache_jobs_deserialized[job_name] = Cache.CacheRecord.from_dict(cache_jobs)
|
||||
obj["cache_jobs"] = cache_artifacts_deserialized
|
||||
return RunConfig(**obj)
|
||||
|
||||
@classmethod
|
||||
|
@ -1,12 +1,11 @@
|
||||
import dataclasses
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Dict
|
||||
|
||||
from praktika._environment import _Environment
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import Shell, Utils
|
||||
from praktika.utils import Shell
|
||||
|
||||
|
||||
class S3:
|
||||
@ -52,23 +51,22 @@ class S3:
|
||||
cmd += " --content-type text/plain"
|
||||
res = cls.run_command_with_retries(cmd)
|
||||
if not res:
|
||||
raise
|
||||
raise RuntimeError()
|
||||
bucket = s3_path.split("/")[0]
|
||||
endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket]
|
||||
assert endpoint
|
||||
return f"https://{s3_full_path}".replace(bucket, endpoint)
|
||||
|
||||
@classmethod
|
||||
def put(cls, s3_path, local_path, text=False, metadata=None):
|
||||
def put(cls, s3_path, local_path, text=False, metadata=None, if_none_matched=False):
|
||||
assert Path(local_path).exists(), f"Path [{local_path}] does not exist"
|
||||
assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
|
||||
assert Path(
|
||||
local_path
|
||||
).is_file(), f"Path [{local_path}] is not file. Only files are supported"
|
||||
file_name = Path(local_path).name
|
||||
s3_full_path = s3_path
|
||||
if not s3_full_path.endswith(file_name):
|
||||
s3_full_path = f"{s3_path}/{Path(local_path).name}"
|
||||
if s3_full_path.endswith("/"):
|
||||
s3_full_path = f"{s3_path}{Path(local_path).name}"
|
||||
|
||||
s3_full_path = str(s3_full_path).removeprefix("s3://")
|
||||
bucket, key = s3_full_path.split("/", maxsplit=1)
|
||||
@ -76,6 +74,8 @@ class S3:
|
||||
command = (
|
||||
f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}"
|
||||
)
|
||||
if if_none_matched:
|
||||
command += f' --if-none-match "*"'
|
||||
if metadata:
|
||||
for k, v in metadata.items():
|
||||
command += f" --metadata {k}={v}"
|
||||
@ -84,7 +84,7 @@ class S3:
|
||||
if text:
|
||||
cmd += " --content-type text/plain"
|
||||
res = cls.run_command_with_retries(command)
|
||||
assert res
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def run_command_with_retries(cls, command, retries=Settings.MAX_RETRIES_S3):
|
||||
@ -101,6 +101,14 @@ class S3:
|
||||
elif "does not exist" in stderr:
|
||||
print("ERROR: requested file does not exist")
|
||||
break
|
||||
elif "Unknown options" in stderr:
|
||||
print("ERROR: Invalid AWS CLI command or CLI client version:")
|
||||
print(f" | awc error: {stderr}")
|
||||
break
|
||||
elif "PreconditionFailed" in stderr:
|
||||
print("ERROR: AWS API Call Precondition Failed")
|
||||
print(f" | awc error: {stderr}")
|
||||
break
|
||||
if ret_code != 0:
|
||||
print(
|
||||
f"ERROR: aws s3 cp failed, stdout/stderr err: [{stderr}], out [{stdout}]"
|
||||
@ -108,13 +116,6 @@ class S3:
|
||||
res = ret_code == 0
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def get_link(cls, s3_path, local_path):
|
||||
s3_full_path = f"{s3_path}/{Path(local_path).name}"
|
||||
bucket = s3_path.split("/")[0]
|
||||
endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket]
|
||||
return f"https://{s3_full_path}".replace(bucket, endpoint)
|
||||
|
||||
@classmethod
|
||||
def copy_file_from_s3(cls, s3_path, local_path):
|
||||
assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
|
||||
@ -128,6 +129,19 @@ class S3:
|
||||
res = cls.run_command_with_retries(cmd)
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def copy_file_from_s3_matching_pattern(
|
||||
cls, s3_path, local_path, include, exclude="*"
|
||||
):
|
||||
assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
|
||||
assert Path(
|
||||
local_path
|
||||
).is_dir(), f"Path [{local_path}] does not exist or not a directory"
|
||||
assert s3_path.endswith("/"), f"s3 path is invalid [{s3_path}]"
|
||||
cmd = f'aws s3 cp s3://{s3_path} {local_path} --exclude "{exclude}" --include "{include}" --recursive'
|
||||
res = cls.run_command_with_retries(cmd)
|
||||
return res
|
||||
|
||||
@classmethod
|
||||
def head_object(cls, s3_path):
|
||||
s3_path = str(s3_path).removeprefix("s3://")
|
||||
@ -148,103 +162,6 @@ class S3:
|
||||
verbose=True,
|
||||
)
|
||||
|
||||
# TODO: apparently should be placed into separate file to be used only inside praktika
|
||||
# keeping this module clean from importing Settings, Environment and etc, making it easy for use externally
|
||||
@classmethod
|
||||
def copy_result_to_s3(cls, result, unlock=True):
|
||||
result.dump()
|
||||
env = _Environment.get()
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
|
||||
s3_path_full = f"{s3_path}/{Path(result.file_name()).name}"
|
||||
url = S3.copy_file_to_s3(s3_path=s3_path, local_path=result.file_name())
|
||||
if env.PR_NUMBER:
|
||||
print("Duplicate Result for latest commit alias in PR")
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix(latest=True)}"
|
||||
url = S3.copy_file_to_s3(s3_path=s3_path, local_path=result.file_name())
|
||||
if unlock:
|
||||
if not cls.unlock(s3_path_full):
|
||||
print(f"ERROR: File [{s3_path_full}] unlock failure")
|
||||
assert False # TODO: investigate
|
||||
return url
|
||||
|
||||
@classmethod
|
||||
def copy_result_from_s3(cls, local_path, lock=True):
|
||||
env = _Environment.get()
|
||||
file_name = Path(local_path).name
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/{file_name}"
|
||||
if lock:
|
||||
cls.lock(s3_path)
|
||||
if not S3.copy_file_from_s3(s3_path=s3_path, local_path=local_path):
|
||||
print(f"ERROR: failed to cp file [{s3_path}] from s3")
|
||||
raise
|
||||
|
||||
@classmethod
|
||||
def lock(cls, s3_path, level=0):
|
||||
assert level < 3, "Never"
|
||||
env = _Environment.get()
|
||||
s3_path_lock = s3_path + f".lock"
|
||||
file_path_lock = f"{Settings.TEMP_DIR}/{Path(s3_path_lock).name}"
|
||||
assert Shell.check(
|
||||
f"echo '''{env.JOB_NAME}''' > {file_path_lock}", verbose=True
|
||||
), "Never"
|
||||
|
||||
i = 20
|
||||
meta = S3.head_object(s3_path_lock)
|
||||
while meta:
|
||||
print(f"WARNING: Failed to acquire lock, meta [{meta}] - wait")
|
||||
i -= 5
|
||||
if i < 0:
|
||||
info = f"ERROR: lock acquire failure - unlock forcefully"
|
||||
print(info)
|
||||
env.add_info(info)
|
||||
break
|
||||
time.sleep(5)
|
||||
|
||||
metadata = {"job": Utils.to_base64(env.JOB_NAME)}
|
||||
S3.put(
|
||||
s3_path=s3_path_lock,
|
||||
local_path=file_path_lock,
|
||||
metadata=metadata,
|
||||
)
|
||||
time.sleep(1)
|
||||
obj = S3.head_object(s3_path_lock)
|
||||
if not obj or not obj.has_tags(tags=metadata):
|
||||
print(f"WARNING: locked by another job [{obj}]")
|
||||
env.add_info("S3 lock file failure")
|
||||
cls.lock(s3_path, level=level + 1)
|
||||
print("INFO: lock acquired")
|
||||
|
||||
@classmethod
|
||||
def unlock(cls, s3_path):
|
||||
s3_path_lock = s3_path + ".lock"
|
||||
env = _Environment.get()
|
||||
obj = S3.head_object(s3_path_lock)
|
||||
if not obj:
|
||||
print("ERROR: lock file is removed")
|
||||
assert False # investigate
|
||||
elif not obj.has_tags({"job": Utils.to_base64(env.JOB_NAME)}):
|
||||
print("ERROR: lock file was acquired by another job")
|
||||
assert False # investigate
|
||||
|
||||
if not S3.delete(s3_path_lock):
|
||||
print(f"ERROR: File [{s3_path_lock}] delete failure")
|
||||
print("INFO: lock released")
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_result_link(cls, result):
|
||||
env = _Environment.get()
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix(latest=True if env.PR_NUMBER else False)}"
|
||||
return S3.get_link(s3_path=s3_path, local_path=result.file_name())
|
||||
|
||||
@classmethod
|
||||
def clean_latest_result(cls):
|
||||
env = _Environment.get()
|
||||
env.SHA = "latest"
|
||||
assert env.PR_NUMBER
|
||||
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
|
||||
S3.clean_s3_directory(s3_path=s3_path)
|
||||
|
||||
@classmethod
|
||||
def _upload_file_to_s3(
|
||||
cls, local_file_path, upload_to_s3: bool, text: bool = False, s3_subprefix=""
|
||||
@ -260,36 +177,3 @@ class S3:
|
||||
)
|
||||
return html_link
|
||||
return f"file://{Path(local_file_path).absolute()}"
|
||||
|
||||
@classmethod
|
||||
def upload_result_files_to_s3(cls, result):
|
||||
if result.results:
|
||||
for result_ in result.results:
|
||||
cls.upload_result_files_to_s3(result_)
|
||||
for file in result.files:
|
||||
if not Path(file).is_file():
|
||||
print(f"ERROR: Invalid file [{file}] in [{result.name}] - skip upload")
|
||||
result.info += f"\nWARNING: Result file [{file}] was not found"
|
||||
file_link = cls._upload_file_to_s3(file, upload_to_s3=False)
|
||||
else:
|
||||
is_text = False
|
||||
for text_file_suffix in Settings.TEXT_CONTENT_EXTENSIONS:
|
||||
if file.endswith(text_file_suffix):
|
||||
print(
|
||||
f"File [{file}] matches Settings.TEXT_CONTENT_EXTENSIONS [{Settings.TEXT_CONTENT_EXTENSIONS}] - add text attribute for s3 object"
|
||||
)
|
||||
is_text = True
|
||||
break
|
||||
file_link = cls._upload_file_to_s3(
|
||||
file,
|
||||
upload_to_s3=True,
|
||||
text=is_text,
|
||||
s3_subprefix=Utils.normalize_string(result.name),
|
||||
)
|
||||
result.links.append(file_link)
|
||||
if result.files:
|
||||
print(
|
||||
f"Result files [{result.files}] uploaded to s3 [{result.links[-len(result.files):]}] - clean files list"
|
||||
)
|
||||
result.files = []
|
||||
result.dump()
|
||||
|
@ -1,8 +1,152 @@
|
||||
from praktika._settings import _Settings
|
||||
from praktika.mangle import _get_user_settings
|
||||
import dataclasses
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional
|
||||
|
||||
Settings = _Settings()
|
||||
|
||||
user_settings = _get_user_settings()
|
||||
for setting, value in user_settings.items():
|
||||
Settings.__setattr__(setting, value)
|
||||
@dataclasses.dataclass
|
||||
class _Settings:
|
||||
######################################
|
||||
# Pipeline generation settings #
|
||||
######################################
|
||||
MAIN_BRANCH = "main"
|
||||
CI_PATH = "./ci"
|
||||
WORKFLOW_PATH_PREFIX: str = "./.github/workflows"
|
||||
WORKFLOWS_DIRECTORY: str = f"{CI_PATH}/workflows"
|
||||
SETTINGS_DIRECTORY: str = f"{CI_PATH}/settings"
|
||||
CI_CONFIG_JOB_NAME = "Config Workflow"
|
||||
DOCKER_BUILD_JOB_NAME = "Docker Builds"
|
||||
FINISH_WORKFLOW_JOB_NAME = "Finish Workflow"
|
||||
READY_FOR_MERGE_STATUS_NAME = "Ready for Merge"
|
||||
CI_CONFIG_RUNS_ON: Optional[List[str]] = None
|
||||
DOCKER_BUILD_RUNS_ON: Optional[List[str]] = None
|
||||
VALIDATE_FILE_PATHS: bool = True
|
||||
|
||||
######################################
|
||||
# Runtime Settings #
|
||||
######################################
|
||||
MAX_RETRIES_S3 = 3
|
||||
MAX_RETRIES_GH = 3
|
||||
|
||||
######################################
|
||||
# S3 (artifact storage) settings #
|
||||
######################################
|
||||
S3_ARTIFACT_PATH: str = ""
|
||||
|
||||
######################################
|
||||
# CI workspace settings #
|
||||
######################################
|
||||
TEMP_DIR: str = "/tmp/praktika"
|
||||
OUTPUT_DIR: str = f"{TEMP_DIR}/output"
|
||||
INPUT_DIR: str = f"{TEMP_DIR}/input"
|
||||
PYTHON_INTERPRETER: str = "python3"
|
||||
PYTHON_PACKET_MANAGER: str = "pip3"
|
||||
PYTHON_VERSION: str = "3.9"
|
||||
INSTALL_PYTHON_FOR_NATIVE_JOBS: bool = False
|
||||
INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS: str = "./ci/requirements.txt"
|
||||
ENVIRONMENT_VAR_FILE: str = f"{TEMP_DIR}/environment.json"
|
||||
RUN_LOG: str = f"{TEMP_DIR}/praktika_run.log"
|
||||
|
||||
SECRET_GH_APP_ID: str = "GH_APP_ID"
|
||||
SECRET_GH_APP_PEM_KEY: str = "GH_APP_PEM_KEY"
|
||||
|
||||
ENV_SETUP_SCRIPT: str = "/tmp/praktika_setup_env.sh"
|
||||
WORKFLOW_STATUS_FILE: str = f"{TEMP_DIR}/workflow_status.json"
|
||||
|
||||
######################################
|
||||
# CI Cache settings #
|
||||
######################################
|
||||
CACHE_VERSION: int = 1
|
||||
CACHE_DIGEST_LEN: int = 20
|
||||
CACHE_S3_PATH: str = ""
|
||||
CACHE_LOCAL_PATH: str = f"{TEMP_DIR}/ci_cache"
|
||||
|
||||
######################################
|
||||
# Report settings #
|
||||
######################################
|
||||
HTML_S3_PATH: str = ""
|
||||
HTML_PAGE_FILE: str = "./praktika/json.html"
|
||||
TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"])
|
||||
S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None
|
||||
|
||||
DOCKERHUB_USERNAME: str = ""
|
||||
DOCKERHUB_SECRET: str = ""
|
||||
DOCKER_WD: str = "/wd"
|
||||
|
||||
######################################
|
||||
# CI DB Settings #
|
||||
######################################
|
||||
SECRET_CI_DB_URL: str = "CI_DB_URL"
|
||||
SECRET_CI_DB_PASSWORD: str = "CI_DB_PASSWORD"
|
||||
CI_DB_DB_NAME = ""
|
||||
CI_DB_TABLE_NAME = ""
|
||||
CI_DB_INSERT_TIMEOUT_SEC = 5
|
||||
|
||||
DISABLE_MERGE_COMMIT = True
|
||||
|
||||
|
||||
_USER_DEFINED_SETTINGS = [
|
||||
"S3_ARTIFACT_PATH",
|
||||
"CACHE_S3_PATH",
|
||||
"HTML_S3_PATH",
|
||||
"S3_BUCKET_TO_HTTP_ENDPOINT",
|
||||
"TEXT_CONTENT_EXTENSIONS",
|
||||
"TEMP_DIR",
|
||||
"OUTPUT_DIR",
|
||||
"INPUT_DIR",
|
||||
"CI_CONFIG_RUNS_ON",
|
||||
"DOCKER_BUILD_RUNS_ON",
|
||||
"CI_CONFIG_JOB_NAME",
|
||||
"PYTHON_INTERPRETER",
|
||||
"PYTHON_VERSION",
|
||||
"PYTHON_PACKET_MANAGER",
|
||||
"INSTALL_PYTHON_FOR_NATIVE_JOBS",
|
||||
"INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS",
|
||||
"MAX_RETRIES_S3",
|
||||
"MAX_RETRIES_GH",
|
||||
"VALIDATE_FILE_PATHS",
|
||||
"DOCKERHUB_USERNAME",
|
||||
"DOCKERHUB_SECRET",
|
||||
"READY_FOR_MERGE_STATUS_NAME",
|
||||
"SECRET_CI_DB_URL",
|
||||
"SECRET_CI_DB_PASSWORD",
|
||||
"CI_DB_DB_NAME",
|
||||
"CI_DB_TABLE_NAME",
|
||||
"CI_DB_INSERT_TIMEOUT_SEC",
|
||||
"SECRET_GH_APP_PEM_KEY",
|
||||
"SECRET_GH_APP_ID",
|
||||
"MAIN_BRANCH",
|
||||
"DISABLE_MERGE_COMMIT",
|
||||
]
|
||||
|
||||
|
||||
def _get_settings() -> _Settings:
|
||||
res = _Settings()
|
||||
|
||||
directory = Path(_Settings.SETTINGS_DIRECTORY)
|
||||
for py_file in directory.glob("*.py"):
|
||||
module_name = py_file.name.removeprefix(".py")
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
module_name, f"{_Settings.SETTINGS_DIRECTORY}/{module_name}"
|
||||
)
|
||||
assert spec
|
||||
foo = importlib.util.module_from_spec(spec)
|
||||
assert spec.loader
|
||||
spec.loader.exec_module(foo)
|
||||
for setting in _USER_DEFINED_SETTINGS:
|
||||
try:
|
||||
value = getattr(foo, setting)
|
||||
res.__setattr__(setting, value)
|
||||
# print(f"- read user defined setting [{setting} = {value}]")
|
||||
except Exception as e:
|
||||
# print(f"Exception while read user settings: {e}")
|
||||
pass
|
||||
|
||||
return res
|
||||
|
||||
|
||||
class GHRunners:
|
||||
ubuntu = "ubuntu-latest"
|
||||
|
||||
|
||||
Settings = _get_settings()
|
||||
|
@ -17,8 +17,6 @@ from threading import Thread
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, Dict, Iterator, List, Optional, Type, TypeVar, Union
|
||||
|
||||
from praktika._settings import _Settings
|
||||
|
||||
T = TypeVar("T", bound="Serializable")
|
||||
|
||||
|
||||
@ -81,24 +79,25 @@ class MetaClasses:
|
||||
class ContextManager:
|
||||
@staticmethod
|
||||
@contextmanager
|
||||
def cd(to: Optional[Union[Path, str]] = None) -> Iterator[None]:
|
||||
def cd(to: Optional[Union[Path, str]]) -> Iterator[None]:
|
||||
"""
|
||||
changes current working directory to @path or `git root` if @path is None
|
||||
:param to:
|
||||
:return:
|
||||
"""
|
||||
if not to:
|
||||
try:
|
||||
to = Shell.get_output_or_raise("git rev-parse --show-toplevel")
|
||||
except:
|
||||
pass
|
||||
if not to:
|
||||
if Path(_Settings.DOCKER_WD).is_dir():
|
||||
to = _Settings.DOCKER_WD
|
||||
if not to:
|
||||
assert False, "FIX IT"
|
||||
assert to
|
||||
# if not to:
|
||||
# try:
|
||||
# to = Shell.get_output_or_raise("git rev-parse --show-toplevel")
|
||||
# except:
|
||||
# pass
|
||||
# if not to:
|
||||
# if Path(_Settings.DOCKER_WD).is_dir():
|
||||
# to = _Settings.DOCKER_WD
|
||||
# if not to:
|
||||
# assert False, "FIX IT"
|
||||
# assert to
|
||||
old_pwd = os.getcwd()
|
||||
if to:
|
||||
os.chdir(to)
|
||||
try:
|
||||
yield
|
||||
|
@ -4,10 +4,8 @@ from itertools import chain
|
||||
from pathlib import Path
|
||||
|
||||
from praktika import Workflow
|
||||
from praktika._settings import GHRunners
|
||||
from praktika.mangle import _get_workflows
|
||||
from praktika.settings import Settings
|
||||
from praktika.utils import ContextManager
|
||||
from praktika.settings import GHRunners, Settings
|
||||
|
||||
|
||||
class Validator:
|
||||
@ -119,7 +117,6 @@ class Validator:
|
||||
def validate_file_paths_in_run_command(cls, workflow: Workflow.Config) -> None:
|
||||
if not Settings.VALIDATE_FILE_PATHS:
|
||||
return
|
||||
with ContextManager.cd():
|
||||
for job in workflow.jobs:
|
||||
run_command = job.command
|
||||
command_parts = run_command.split(" ")
|
||||
@ -135,7 +132,6 @@ class Validator:
|
||||
def validate_file_paths_in_digest_configs(cls, workflow: Workflow.Config) -> None:
|
||||
if not Settings.VALIDATE_FILE_PATHS:
|
||||
return
|
||||
with ContextManager.cd():
|
||||
for job in workflow.jobs:
|
||||
if not job.digest_config:
|
||||
continue
|
||||
@ -153,7 +149,6 @@ class Validator:
|
||||
|
||||
@classmethod
|
||||
def validate_requirements_txt_files(cls, workflow: Workflow.Config) -> None:
|
||||
with ContextManager.cd():
|
||||
for job in workflow.jobs:
|
||||
if job.job_requirements:
|
||||
if job.job_requirements.python_requirements_txt:
|
||||
@ -171,9 +166,7 @@ class Validator:
|
||||
"\n echo requests==2.32.3 >> ./ci/requirements.txt"
|
||||
)
|
||||
message += "\n echo https://clickhouse-builds.s3.amazonaws.com/packages/praktika-0.1-py3-none-any.whl >> ./ci/requirements.txt"
|
||||
cls.evaluate_check(
|
||||
path.is_file(), message, job.name, workflow.name
|
||||
)
|
||||
cls.evaluate_check(path.is_file(), message, job.name, workflow.name)
|
||||
|
||||
@classmethod
|
||||
def validate_dockers(cls, workflow: Workflow.Config):
|
||||
|
@ -31,6 +31,7 @@ class Workflow:
|
||||
enable_report: bool = False
|
||||
enable_merge_ready_status: bool = False
|
||||
enable_cidb: bool = False
|
||||
enable_merge_commit: bool = False
|
||||
|
||||
def is_event_pull_request(self):
|
||||
return self.event == Workflow.Event.PULL_REQUEST
|
||||
|
@ -80,6 +80,8 @@ jobs:
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{{{ github.head_ref }}}}
|
||||
{JOB_ADDONS}
|
||||
- name: Prepare env script
|
||||
run: |
|
||||
@ -102,7 +104,11 @@ jobs:
|
||||
run: |
|
||||
. /tmp/praktika_setup_env.sh
|
||||
set -o pipefail
|
||||
{PYTHON} -m praktika run --job '''{JOB_NAME}''' --workflow "{WORKFLOW_NAME}" --ci |& tee {RUN_LOG}
|
||||
if command -v ts &> /dev/null; then
|
||||
python3 -m praktika run --job '''{JOB_NAME}''' --workflow "{WORKFLOW_NAME}" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
|
||||
else
|
||||
python3 -m praktika run --job '''{JOB_NAME}''' --workflow "{WORKFLOW_NAME}" --ci |& tee /tmp/praktika/praktika_run.log
|
||||
fi
|
||||
{UPLOADS_GITHUB}\
|
||||
"""
|
||||
|
||||
@ -184,11 +190,9 @@ jobs:
|
||||
False
|
||||
), f"Workflow event not yet supported [{workflow_config.event}]"
|
||||
|
||||
with ContextManager.cd():
|
||||
with open(self._get_workflow_file_name(workflow_config.name), "w") as f:
|
||||
f.write(yaml_workflow_str)
|
||||
|
||||
with ContextManager.cd():
|
||||
Shell.check("git add ./.github/workflows/*.yaml")
|
||||
|
||||
|
||||
|
@ -7,24 +7,33 @@ S3_BUCKET_HTTP_ENDPOINT = "clickhouse-builds.s3.amazonaws.com"
|
||||
class RunnerLabels:
|
||||
CI_SERVICES = "ci_services"
|
||||
CI_SERVICES_EBS = "ci_services_ebs"
|
||||
BUILDER = "builder"
|
||||
BUILDER_AMD = "builder"
|
||||
BUILDER_ARM = "builder-aarch64"
|
||||
FUNC_TESTER_AMD = "func-tester"
|
||||
FUNC_TESTER_ARM = "func-tester-aarch64"
|
||||
|
||||
|
||||
BASE_BRANCH = "master"
|
||||
|
||||
azure_secret = Secret.Config(
|
||||
name="azure_connection_string",
|
||||
type=Secret.Type.AWS_SSM_VAR,
|
||||
)
|
||||
|
||||
SECRETS = [
|
||||
Secret.Config(
|
||||
name="dockerhub_robot_password",
|
||||
type=Secret.Type.AWS_SSM_VAR,
|
||||
),
|
||||
Secret.Config(
|
||||
name="woolenwolf_gh_app.clickhouse-app-id",
|
||||
type=Secret.Type.AWS_SSM_SECRET,
|
||||
),
|
||||
Secret.Config(
|
||||
name="woolenwolf_gh_app.clickhouse-app-key",
|
||||
type=Secret.Type.AWS_SSM_SECRET,
|
||||
),
|
||||
azure_secret,
|
||||
# Secret.Config(
|
||||
# name="woolenwolf_gh_app.clickhouse-app-id",
|
||||
# type=Secret.Type.AWS_SSM_SECRET,
|
||||
# ),
|
||||
# Secret.Config(
|
||||
# name="woolenwolf_gh_app.clickhouse-app-key",
|
||||
# type=Secret.Type.AWS_SSM_SECRET,
|
||||
# ),
|
||||
]
|
||||
|
||||
DOCKERS = [
|
||||
@ -118,18 +127,18 @@ DOCKERS = [
|
||||
# platforms=Docker.Platforms.arm_amd,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/stateless-test",
|
||||
# path="./ci/docker/test/stateless",
|
||||
# platforms=Docker.Platforms.arm_amd,
|
||||
# depends_on=["clickhouse/test-base"],
|
||||
# ),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/stateful-test",
|
||||
# path="./ci/docker/test/stateful",
|
||||
# platforms=Docker.Platforms.arm_amd,
|
||||
# depends_on=["clickhouse/stateless-test"],
|
||||
# ),
|
||||
Docker.Config(
|
||||
name="clickhouse/stateless-test",
|
||||
path="./ci/docker/stateless-test",
|
||||
platforms=Docker.Platforms.arm_amd,
|
||||
depends_on=[],
|
||||
),
|
||||
Docker.Config(
|
||||
name="clickhouse/stateful-test",
|
||||
path="./ci/docker/stateful-test",
|
||||
platforms=Docker.Platforms.arm_amd,
|
||||
depends_on=["clickhouse/stateless-test"],
|
||||
),
|
||||
# Docker.Config(
|
||||
# name="clickhouse/stress-test",
|
||||
# path="./ci/docker/test/stress",
|
||||
@ -230,4 +239,6 @@ DOCKERS = [
|
||||
class JobNames:
|
||||
STYLE_CHECK = "Style Check"
|
||||
FAST_TEST = "Fast test"
|
||||
BUILD_AMD_DEBUG = "Build amd64 debug"
|
||||
BUILD = "Build"
|
||||
STATELESS = "Stateless tests"
|
||||
STATEFUL = "Stateful tests"
|
||||
|
@ -4,6 +4,8 @@ from ci.settings.definitions import (
|
||||
RunnerLabels,
|
||||
)
|
||||
|
||||
MAIN_BRANCH = "master"
|
||||
|
||||
S3_ARTIFACT_PATH = f"{S3_BUCKET_NAME}/artifacts"
|
||||
CI_CONFIG_RUNS_ON = [RunnerLabels.CI_SERVICES]
|
||||
DOCKER_BUILD_RUNS_ON = [RunnerLabels.CI_SERVICES_EBS]
|
||||
|
@ -1,5 +1,3 @@
|
||||
from typing import List
|
||||
|
||||
from praktika import Artifact, Job, Workflow
|
||||
from praktika.settings import Settings
|
||||
|
||||
@ -13,7 +11,10 @@ from ci.settings.definitions import (
|
||||
|
||||
|
||||
class ArtifactNames:
|
||||
ch_debug_binary = "clickhouse_debug_binary"
|
||||
CH_AMD_DEBUG = "CH_AMD_DEBUG"
|
||||
CH_AMD_RELEASE = "CH_AMD_RELEASE"
|
||||
CH_ARM_RELEASE = "CH_ARM_RELEASE"
|
||||
CH_ARM_ASAN = "CH_ARM_ASAN"
|
||||
|
||||
|
||||
style_check_job = Job.Config(
|
||||
@ -25,7 +26,7 @@ style_check_job = Job.Config(
|
||||
|
||||
fast_test_job = Job.Config(
|
||||
name=JobNames.FAST_TEST,
|
||||
runs_on=[RunnerLabels.BUILDER],
|
||||
runs_on=[RunnerLabels.BUILDER_AMD],
|
||||
command="python3 ./ci/jobs/fast_test.py",
|
||||
run_in_docker="clickhouse/fasttest",
|
||||
digest_config=Job.CacheDigestConfig(
|
||||
@ -37,11 +38,13 @@ fast_test_job = Job.Config(
|
||||
),
|
||||
)
|
||||
|
||||
job_build_amd_debug = Job.Config(
|
||||
name=JobNames.BUILD_AMD_DEBUG,
|
||||
runs_on=[RunnerLabels.BUILDER],
|
||||
command="python3 ./ci/jobs/build_clickhouse.py amd_debug",
|
||||
build_jobs = Job.Config(
|
||||
name=JobNames.BUILD,
|
||||
runs_on=["...from params..."],
|
||||
requires=[JobNames.FAST_TEST],
|
||||
command="python3 ./ci/jobs/build_clickhouse.py --build-type {PARAMETER}",
|
||||
run_in_docker="clickhouse/fasttest",
|
||||
timeout=3600 * 2,
|
||||
digest_config=Job.CacheDigestConfig(
|
||||
include_paths=[
|
||||
"./src",
|
||||
@ -54,9 +57,85 @@ job_build_amd_debug = Job.Config(
|
||||
"./docker/packager/packager",
|
||||
"./rust",
|
||||
"./tests/ci/version_helper.py",
|
||||
"./ci/jobs/build_clickhouse.py",
|
||||
],
|
||||
),
|
||||
provides=[ArtifactNames.ch_debug_binary],
|
||||
).parametrize(
|
||||
parameter=["amd_debug", "amd_release", "arm_release", "arm_asan"],
|
||||
provides=[
|
||||
[ArtifactNames.CH_AMD_DEBUG],
|
||||
[ArtifactNames.CH_AMD_RELEASE],
|
||||
[ArtifactNames.CH_ARM_RELEASE],
|
||||
[ArtifactNames.CH_ARM_ASAN],
|
||||
],
|
||||
runs_on=[
|
||||
[RunnerLabels.BUILDER_AMD],
|
||||
[RunnerLabels.BUILDER_AMD],
|
||||
[RunnerLabels.BUILDER_ARM],
|
||||
[RunnerLabels.BUILDER_ARM],
|
||||
],
|
||||
)
|
||||
|
||||
stateless_tests_jobs = Job.Config(
|
||||
name=JobNames.STATELESS,
|
||||
runs_on=[RunnerLabels.BUILDER_AMD],
|
||||
command="python3 ./ci/jobs/functional_stateless_tests.py --test-options {PARAMETER}",
|
||||
# many tests expect to see "/var/lib/clickhouse" in various output lines - add mount for now, consider creating this dir in docker file
|
||||
run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined",
|
||||
digest_config=Job.CacheDigestConfig(
|
||||
include_paths=[
|
||||
"./ci/jobs/functional_stateless_tests.py",
|
||||
],
|
||||
),
|
||||
).parametrize(
|
||||
parameter=[
|
||||
"amd_debug,parallel",
|
||||
"amd_debug,non-parallel",
|
||||
"amd_release,parallel",
|
||||
"amd_release,non-parallel",
|
||||
"arm_asan,parallel",
|
||||
"arm_asan,non-parallel",
|
||||
],
|
||||
runs_on=[
|
||||
[RunnerLabels.BUILDER_AMD],
|
||||
[RunnerLabels.FUNC_TESTER_AMD],
|
||||
[RunnerLabels.BUILDER_AMD],
|
||||
[RunnerLabels.FUNC_TESTER_AMD],
|
||||
[RunnerLabels.BUILDER_ARM],
|
||||
[RunnerLabels.FUNC_TESTER_ARM],
|
||||
],
|
||||
requires=[
|
||||
[ArtifactNames.CH_AMD_DEBUG],
|
||||
[ArtifactNames.CH_AMD_DEBUG],
|
||||
[ArtifactNames.CH_AMD_RELEASE],
|
||||
[ArtifactNames.CH_AMD_RELEASE],
|
||||
[ArtifactNames.CH_ARM_ASAN],
|
||||
[ArtifactNames.CH_ARM_ASAN],
|
||||
],
|
||||
)
|
||||
|
||||
stateful_tests_jobs = Job.Config(
|
||||
name=JobNames.STATEFUL,
|
||||
runs_on=[RunnerLabels.BUILDER_AMD],
|
||||
command="python3 ./ci/jobs/functional_stateful_tests.py --test-options {PARAMETER}",
|
||||
# many tests expect to see "/var/lib/clickhouse"
|
||||
# some tests expect to see "/var/log/clickhouse"
|
||||
run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined",
|
||||
digest_config=Job.CacheDigestConfig(
|
||||
include_paths=[
|
||||
"./ci/jobs/functional_stateful_tests.py",
|
||||
],
|
||||
),
|
||||
).parametrize(
|
||||
parameter=[
|
||||
"amd_debug,parallel",
|
||||
],
|
||||
runs_on=[
|
||||
[RunnerLabels.BUILDER_AMD],
|
||||
],
|
||||
requires=[
|
||||
[ArtifactNames.CH_AMD_DEBUG],
|
||||
],
|
||||
)
|
||||
|
||||
workflow = Workflow.Config(
|
||||
@ -66,14 +145,31 @@ workflow = Workflow.Config(
|
||||
jobs=[
|
||||
style_check_job,
|
||||
fast_test_job,
|
||||
job_build_amd_debug,
|
||||
*build_jobs,
|
||||
*stateless_tests_jobs,
|
||||
*stateful_tests_jobs,
|
||||
],
|
||||
artifacts=[
|
||||
Artifact.Config(
|
||||
name=ArtifactNames.ch_debug_binary,
|
||||
name=ArtifactNames.CH_AMD_DEBUG,
|
||||
type=Artifact.Type.S3,
|
||||
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
|
||||
)
|
||||
),
|
||||
Artifact.Config(
|
||||
name=ArtifactNames.CH_AMD_RELEASE,
|
||||
type=Artifact.Type.S3,
|
||||
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
|
||||
),
|
||||
Artifact.Config(
|
||||
name=ArtifactNames.CH_ARM_RELEASE,
|
||||
type=Artifact.Type.S3,
|
||||
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
|
||||
),
|
||||
Artifact.Config(
|
||||
name=ArtifactNames.CH_ARM_ASAN,
|
||||
type=Artifact.Type.S3,
|
||||
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
|
||||
),
|
||||
],
|
||||
dockers=DOCKERS,
|
||||
secrets=SECRETS,
|
||||
@ -84,11 +180,14 @@ workflow = Workflow.Config(
|
||||
|
||||
WORKFLOWS = [
|
||||
workflow,
|
||||
] # type: List[Workflow.Config]
|
||||
]
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# local job test inside praktika environment
|
||||
from praktika.runner import Runner
|
||||
|
||||
Runner().run(workflow, fast_test_job, docker="fasttest", dummy_env=True)
|
||||
# if __name__ == "__main__":
|
||||
# # local job test inside praktika environment
|
||||
# from praktika.runner import Runner
|
||||
# from praktika.digest import Digest
|
||||
#
|
||||
# print(Digest().calc_job_digest(amd_debug_build_job))
|
||||
#
|
||||
# Runner().run(workflow, fast_test_job, docker="fasttest", local_run=True)
|
||||
|
@ -74,6 +74,7 @@ elseif (ARCH_AARCH64)
|
||||
# introduced as optional, either in v8.2 [7] or in v8.4 [8].
|
||||
# rcpc: Load-Acquire RCpc Register. Better support of release/acquire of atomics. Good for allocators and high contention code.
|
||||
# Optional in v8.2, mandatory in v8.3 [9]. Supported in Graviton >=2, Azure and GCP instances.
|
||||
# bf16: Bfloat16, a half-precision floating point format developed by Google Brain. Optional in v8.2, mandatory in v8.6.
|
||||
#
|
||||
# [1] https://github.com/aws/aws-graviton-getting-started/blob/main/c-c%2B%2B.md
|
||||
# [2] https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10
|
||||
|
6
contrib/CMakeLists.txt
vendored
6
contrib/CMakeLists.txt
vendored
@ -217,7 +217,11 @@ add_contrib (libssh-cmake libssh)
|
||||
|
||||
add_contrib (prometheus-protobufs-cmake prometheus-protobufs prometheus-protobufs-gogo)
|
||||
|
||||
add_contrib(numactl-cmake numactl)
|
||||
add_contrib (numactl-cmake numactl)
|
||||
|
||||
add_contrib (google-cloud-cpp-cmake google-cloud-cpp) # requires grpc, protobuf, absl
|
||||
|
||||
add_contrib (jwt-cpp-cmake jwt-cpp)
|
||||
|
||||
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
|
||||
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
|
||||
|
1
contrib/google-cloud-cpp
vendored
Submodule
1
contrib/google-cloud-cpp
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit 83f30caadb8613fb5c408d8c2fd545291596b53f
|
105
contrib/google-cloud-cpp-cmake/CMakeLists.txt
Normal file
105
contrib/google-cloud-cpp-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,105 @@
|
||||
set(ENABLE_GOOGLE_CLOUD_CPP_DEFAULT OFF)
|
||||
|
||||
if(ENABLE_LIBRARIES AND CLICKHOUSE_CLOUD AND OS_LINUX)
|
||||
set(ENABLE_GOOGLE_CLOUD_CPP_DEFAULT ON)
|
||||
endif()
|
||||
|
||||
option(ENABLE_GOOGLE_CLOUD_CPP "Enable Google Cloud Cpp" ${ENABLE_GOOGLE_CLOUD_CPP_DEFAULT})
|
||||
|
||||
if(NOT ENABLE_GOOGLE_CLOUD_CPP)
|
||||
message(STATUS "Not using Google Cloud Cpp")
|
||||
return()
|
||||
endif()
|
||||
|
||||
if(NOT ENABLE_GRPC)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use Google Cloud Cpp without gRPC")
|
||||
endif()
|
||||
if (NOT ENABLE_PROTOBUF)
|
||||
message( ${RECONFIGURE_MESSAGE_LEVEL} "Can't use Google Cloud Cpp without protobuf")
|
||||
endif()
|
||||
|
||||
# Gather sources and options.
|
||||
set(GOOGLE_CLOUD_CPP_SOURCES)
|
||||
set(GOOGLE_CLOUD_CPP_PUBLIC_INCLUDES)
|
||||
set(GOOGLE_CLOUD_CPP_PRIVATE_INCLUDES)
|
||||
set(GOOGLE_CLOUD_CPP_PRIVATE_LIBS)
|
||||
|
||||
# Directories.
|
||||
SET(GOOGLE_CLOUD_CPP_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-cloud-cpp" )
|
||||
list(APPEND GOOGLE_CLOUD_CPP_PRIVATE_INCLUDES "${GOOGLE_CLOUD_CPP_DIR}")
|
||||
|
||||
# Set the PROJECT_SOURCE_DIR so that all Google Cloud cmake files work
|
||||
set(PROJECT_SOURCE_DIR_BAK ${PROJECT_SOURCE_DIR})
|
||||
set(PROJECT_SOURCE_DIR ${GOOGLE_CLOUD_CPP_DIR})
|
||||
|
||||
list(APPEND CMAKE_MODULE_PATH "${GOOGLE_CLOUD_CPP_DIR}/cmake")
|
||||
|
||||
# Building this target results in all protobufs being compiled.
|
||||
add_custom_target(google-cloud-cpp-protos)
|
||||
|
||||
include("GoogleCloudCppLibrary")
|
||||
|
||||
# Set some variables required for googleapis CMakeLists.txt to work.
|
||||
set(GOOGLE_CLOUD_CPP_ENABLE_GRPC ON)
|
||||
set(PROJECT_VERSION "1")
|
||||
set(PROJECT_VERSION_MAJOR "1")
|
||||
set(PROTO_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src")
|
||||
set(GOOGLE_CLOUD_CPP_GRPC_PLUGIN_EXECUTABLE $<TARGET_FILE:grpc_cpp_plugin>)
|
||||
|
||||
include(GoogleApis.cmake)
|
||||
|
||||
add_library(gRPC::grpc++ ALIAS _ch_contrib_grpc)
|
||||
add_library(gRPC::grpc ALIAS _ch_contrib_grpc)
|
||||
|
||||
# google-cloud-cpp-kms.
|
||||
google_cloud_cpp_add_library_protos(kms)
|
||||
|
||||
include(google_cloud_cpp_common.cmake)
|
||||
include(google_cloud_cpp_grpc_utils.cmake)
|
||||
|
||||
SET(GOOGLE_CLOUD_CPP_KMS_DIR "${GOOGLE_CLOUD_CPP_DIR}/google/cloud/kms")
|
||||
|
||||
file(GLOB GOOGLE_CLOUD_CPP_KMS_SRC
|
||||
"${GOOGLE_CLOUD_CPP_KMS_DIR}/v1/*.cc"
|
||||
"${GOOGLE_CLOUD_CPP_KMS_DIR}/v1/internal/*.cc"
|
||||
"${GOOGLE_CLOUD_CPP_KMS_DIR}/inventory/v1/*.cc"
|
||||
)
|
||||
|
||||
list(APPEND GOOGLE_CLOUD_CPP_SOURCES ${GOOGLE_CLOUD_CPP_KMS_SRC})
|
||||
list(APPEND GOOGLE_CLOUD_CPP_PUBLIC_INCLUDES "${GOOGLE_CLOUD_CPP_DIR}" "${CMAKE_CURRENT_BINARY_DIR}")
|
||||
|
||||
set(GRPC_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/grpc")
|
||||
list(APPEND GOOGLE_CLOUD_CPP_PUBLIC_INCLUDES "${GRPC_INCLUDE_DIR}/include" "${GRPC_INCLUDE_DIR}/spm-cpp-include")
|
||||
|
||||
# Restore the PROJECT_SOURCE_DIR.
|
||||
set(PROJECT_SOURCE_DIR ${PROJECT_SOURCE_DIR_BAK})
|
||||
|
||||
# Link against external libraries.
|
||||
list(APPEND GOOGLE_CLOUD_CPP_PRIVATE_LIBS
|
||||
google_cloud_cpp_common
|
||||
google_cloud_cpp_grpc_utils
|
||||
google_cloud_cpp_kms_protos
|
||||
google_cloud_cpp_cloud_location_locations_protos
|
||||
google_cloud_cpp_iam_v1_iam_policy_protos
|
||||
gRPC::grpc++
|
||||
absl::optional
|
||||
)
|
||||
|
||||
list(APPEND GOOGLE_CLOUD_CPP_PUBLIC_LIBS
|
||||
absl::optional
|
||||
gRPC::grpc++
|
||||
)
|
||||
|
||||
# Add library.
|
||||
add_library(_gcloud ${GOOGLE_CLOUD_CPP_SOURCES})
|
||||
|
||||
target_include_directories(_gcloud SYSTEM PUBLIC ${GOOGLE_CLOUD_CPP_PUBLIC_INCLUDES})
|
||||
target_include_directories(_gcloud SYSTEM PRIVATE ${GOOGLE_CLOUD_CPP_PRIVATE_INCLUDES})
|
||||
target_link_libraries(_gcloud PRIVATE ${GOOGLE_CLOUD_CPP_PRIVATE_LIBS})
|
||||
|
||||
# The library is large - avoid bloat.
|
||||
if (OMIT_HEAVY_DEBUG_SYMBOLS)
|
||||
target_compile_options(_gcloud PRIVATE -g0)
|
||||
endif()
|
||||
|
||||
add_library(ch_contrib::google_cloud_cpp ALIAS _gcloud)
|
469
contrib/google-cloud-cpp-cmake/GoogleApis.cmake
Normal file
469
contrib/google-cloud-cpp-cmake/GoogleApis.cmake
Normal file
@ -0,0 +1,469 @@
|
||||
# ~~~
|
||||
# Copyright 2020 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ~~~
|
||||
|
||||
# File copied from google-cloud-cpp/external/googleapis/CMakeLists.txt with minor modifications.
|
||||
|
||||
if (NOT GOOGLE_CLOUD_CPP_ENABLE_GRPC)
|
||||
return()
|
||||
endif ()
|
||||
|
||||
include(GoogleapisConfig)
|
||||
|
||||
set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL
|
||||
"https://github.com/googleapis/googleapis/archive/${_GOOGLE_CLOUD_CPP_GOOGLEAPIS_COMMIT_SHA}.tar.gz"
|
||||
"https://storage.googleapis.com/cloud-cpp-community-archive/github.com/googleapis/googleapis/archive/${_GOOGLE_CLOUD_CPP_GOOGLEAPIS_COMMIT_SHA}.tar.gz"
|
||||
)
|
||||
set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL_HASH
|
||||
"${_GOOGLE_CLOUD_CPP_GOOGLEAPIS_SHA256}")
|
||||
if (GOOGLE_CLOUD_CPP_OVERRIDE_GOOGLEAPIS_URL)
|
||||
set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL
|
||||
${GOOGLE_CLOUD_CPP_OVERRIDE_GOOGLEAPIS_URL})
|
||||
endif ()
|
||||
if (GOOGLE_CLOUD_CPP_OVERRIDE_GOOGLEAPIS_URL_HASH)
|
||||
set(GOOGLE_CLOUD_CPP_GOOGLEAPIS_URL_HASH
|
||||
"${GOOGLE_CLOUD_CPP_OVERRIDE_GOOGLEAPIS_URL_HASH}")
|
||||
endif ()
|
||||
|
||||
set(EXTERNAL_GOOGLEAPIS_PROTO_FILES
|
||||
# cmake-format: sort
|
||||
"google/api/annotations.proto"
|
||||
"google/api/auth.proto"
|
||||
"google/api/backend.proto"
|
||||
"google/api/billing.proto"
|
||||
"google/api/client.proto"
|
||||
"google/api/config_change.proto"
|
||||
"google/api/consumer.proto"
|
||||
"google/api/context.proto"
|
||||
"google/api/control.proto"
|
||||
"google/api/distribution.proto"
|
||||
"google/api/documentation.proto"
|
||||
"google/api/endpoint.proto"
|
||||
"google/api/error_reason.proto"
|
||||
"google/api/field_behavior.proto"
|
||||
"google/api/field_info.proto"
|
||||
"google/api/http.proto"
|
||||
"google/api/httpbody.proto"
|
||||
"google/api/label.proto"
|
||||
"google/api/launch_stage.proto"
|
||||
"google/api/log.proto"
|
||||
"google/api/logging.proto"
|
||||
"google/api/metric.proto"
|
||||
"google/api/monitored_resource.proto"
|
||||
"google/api/monitoring.proto"
|
||||
"google/api/policy.proto"
|
||||
"google/api/quota.proto"
|
||||
"google/api/resource.proto"
|
||||
"google/api/routing.proto"
|
||||
"google/api/service.proto"
|
||||
"google/api/source_info.proto"
|
||||
"google/api/system_parameter.proto"
|
||||
"google/api/usage.proto"
|
||||
"google/api/visibility.proto"
|
||||
"google/cloud/extended_operations.proto"
|
||||
"google/cloud/location/locations.proto"
|
||||
# orgpolicy/v**1** is used *indirectly* by google/cloud/asset, therefore it
|
||||
# does not appear in protolists/asset.list. In addition, it is not compiled
|
||||
# by any other library. So, added manually.
|
||||
"google/cloud/orgpolicy/v1/orgpolicy.proto"
|
||||
# Some gRPC based authentication is implemented by the IAM Credentials
|
||||
# service.
|
||||
"google/iam/credentials/v1/common.proto"
|
||||
"google/iam/credentials/v1/iamcredentials.proto"
|
||||
# We expose google::iam::v1::Policy in our google::cloud::IAMUpdater
|
||||
"google/iam/v1/iam_policy.proto"
|
||||
"google/iam/v1/options.proto"
|
||||
"google/iam/v1/policy.proto"
|
||||
"google/longrunning/operations.proto"
|
||||
"google/rpc/code.proto"
|
||||
"google/rpc/context/attribute_context.proto"
|
||||
"google/rpc/error_details.proto"
|
||||
"google/rpc/status.proto"
|
||||
"google/type/calendar_period.proto"
|
||||
"google/type/color.proto"
|
||||
"google/type/date.proto"
|
||||
"google/type/datetime.proto"
|
||||
"google/type/dayofweek.proto"
|
||||
"google/type/decimal.proto"
|
||||
"google/type/expr.proto"
|
||||
"google/type/fraction.proto"
|
||||
"google/type/interval.proto"
|
||||
"google/type/latlng.proto"
|
||||
"google/type/localized_text.proto"
|
||||
"google/type/money.proto"
|
||||
"google/type/month.proto"
|
||||
"google/type/phone_number.proto"
|
||||
"google/type/postal_address.proto"
|
||||
"google/type/quaternion.proto"
|
||||
"google/type/timeofday.proto")
|
||||
|
||||
include(GoogleCloudCppCommonOptions)
|
||||
|
||||
# Set EXTERNAL_GOOGLEAPIS_SOURCE in the parent directory, as it is used by all
|
||||
# the generated libraries. The Conan packages (https://conan.io), will need to
|
||||
# patch this value. Setting the value in a single place makes such patching
|
||||
# easier.
|
||||
set(EXTERNAL_GOOGLEAPIS_PREFIX "${PROJECT_BINARY_DIR}/external/googleapis")
|
||||
set(EXTERNAL_GOOGLEAPIS_SOURCE
|
||||
"${EXTERNAL_GOOGLEAPIS_PREFIX}/src/googleapis_download"
|
||||
PARENT_SCOPE)
|
||||
set(EXTERNAL_GOOGLEAPIS_SOURCE
|
||||
"${EXTERNAL_GOOGLEAPIS_PREFIX}/src/googleapis_download")
|
||||
|
||||
# Include the functions to compile proto files and maintain proto libraries.
|
||||
include(CompileProtos)
|
||||
|
||||
set(EXTERNAL_GOOGLEAPIS_BYPRODUCTS)
|
||||
foreach (proto ${EXTERNAL_GOOGLEAPIS_PROTO_FILES})
|
||||
list(APPEND EXTERNAL_GOOGLEAPIS_BYPRODUCTS
|
||||
"${EXTERNAL_GOOGLEAPIS_SOURCE}/${proto}")
|
||||
endforeach ()
|
||||
|
||||
file(GLOB protolists "protolists/*.list")
|
||||
foreach (file IN LISTS protolists)
|
||||
google_cloud_cpp_load_protolist(protos "${file}")
|
||||
foreach (proto IN LISTS protos)
|
||||
list(APPEND EXTERNAL_GOOGLEAPIS_BYPRODUCTS "${proto}")
|
||||
endforeach ()
|
||||
endforeach ()
|
||||
|
||||
include(ExternalProject)
|
||||
|
||||
# -- The build needs protobuf files. The original build scripts download them from a remote server (see target 'googleapis_download').
|
||||
# This is too unreliable in the context of ClickHouse ... we instead ship the downloaded archive with the ClickHouse source and
|
||||
# extract it into the build directory directly.
|
||||
|
||||
# Dummy googleapis_download target. This needs to exist because lots of other targets depend on it
|
||||
# We however trick it a little bit saying this target generates the ${EXTERNAL_GOOGLEAPIS_BYPRODUCTS} BYPRODUCTS when
|
||||
# actually the following section is the one actually providing such BYPRODUCTS.
|
||||
externalproject_add(
|
||||
googleapis_download
|
||||
EXCLUDE_FROM_ALL ON
|
||||
PREFIX "${EXTERNAL_GOOGLEAPIS_PREFIX}"
|
||||
PATCH_COMMAND ""
|
||||
DOWNLOAD_COMMAND ""
|
||||
CONFIGURE_COMMAND ""
|
||||
BUILD_COMMAND ""
|
||||
INSTALL_COMMAND ""
|
||||
BUILD_BYPRODUCTS ${EXTERNAL_GOOGLEAPIS_BYPRODUCTS}
|
||||
LOG_DOWNLOAD OFF)
|
||||
|
||||
# Command that extracts the tarball into the proper dir
|
||||
# Note: The hash must match the Google Cloud Api version, otherwise funny things will happen.
|
||||
# Find the right hash in "strip-prefix" in MODULE.bazel in the subrepository
|
||||
message(STATUS "Extracting googleapis tarball")
|
||||
set(PB_HASH "e60db19f11f94175ac682c5898cce0f77cc508ea")
|
||||
set(PB_ARCHIVE "${PB_HASH}.tar.gz")
|
||||
set(PB_DIR "googleapis-${PB_HASH}")
|
||||
|
||||
file(ARCHIVE_EXTRACT INPUT
|
||||
"${ClickHouse_SOURCE_DIR}/contrib/google-cloud-cpp-cmake/googleapis/${PB_ARCHIVE}"
|
||||
DESTINATION
|
||||
"${EXTERNAL_GOOGLEAPIS_PREFIX}/tmp")
|
||||
|
||||
file(REMOVE_RECURSE "${EXTERNAL_GOOGLEAPIS_SOURCE}")
|
||||
file(RENAME
|
||||
"${EXTERNAL_GOOGLEAPIS_PREFIX}/tmp/${PB_DIR}"
|
||||
"${EXTERNAL_GOOGLEAPIS_SOURCE}"
|
||||
)
|
||||
|
||||
google_cloud_cpp_find_proto_include_dir(PROTO_INCLUDE_DIR)
|
||||
|
||||
google_cloud_cpp_add_protos_property()
|
||||
|
||||
function (external_googleapis_short_name var proto)
|
||||
string(REPLACE "google/" "" short_name "${proto}")
|
||||
string(REPLACE "/" "_" short_name "${short_name}")
|
||||
string(REPLACE ".proto" "_protos" short_name "${short_name}")
|
||||
set("${var}"
|
||||
"${short_name}"
|
||||
PARENT_SCOPE)
|
||||
endfunction ()
|
||||
|
||||
# Create a single source proto library.
|
||||
#
|
||||
# * proto: the filename for the proto source.
|
||||
# * (optional) ARGN: proto libraries the new library depends on.
|
||||
function (external_googleapis_add_library proto)
|
||||
external_googleapis_short_name(short_name "${proto}")
|
||||
google_cloud_cpp_grpcpp_library(
|
||||
google_cloud_cpp_${short_name} "${EXTERNAL_GOOGLEAPIS_SOURCE}/${proto}"
|
||||
PROTO_PATH_DIRECTORIES "${EXTERNAL_GOOGLEAPIS_SOURCE}"
|
||||
"${PROTO_INCLUDE_DIR}")
|
||||
|
||||
external_googleapis_set_version_and_alias("${short_name}")
|
||||
|
||||
set(public_deps)
|
||||
foreach (dep_short_name ${ARGN})
|
||||
list(APPEND public_deps "google-cloud-cpp::${dep_short_name}")
|
||||
endforeach ()
|
||||
list(LENGTH public_deps public_deps_length)
|
||||
if (public_deps_length EQUAL 0)
|
||||
target_link_libraries("google_cloud_cpp_${short_name}")
|
||||
else ()
|
||||
target_link_libraries("google_cloud_cpp_${short_name}"
|
||||
PUBLIC ${public_deps})
|
||||
endif ()
|
||||
endfunction ()
|
||||
|
||||
function (external_googleapis_set_version_and_alias short_name)
|
||||
add_dependencies("google_cloud_cpp_${short_name}" googleapis_download)
|
||||
set_target_properties(
|
||||
"google_cloud_cpp_${short_name}"
|
||||
PROPERTIES EXPORT_NAME google-cloud-cpp::${short_name}
|
||||
VERSION "${PROJECT_VERSION}"
|
||||
SOVERSION ${PROJECT_VERSION_MAJOR})
|
||||
add_library("google-cloud-cpp::${short_name}" ALIAS
|
||||
"google_cloud_cpp_${short_name}")
|
||||
endfunction ()
|
||||
|
||||
if (GOOGLE_CLOUD_CPP_USE_INSTALLED_COMMON)
|
||||
return()
|
||||
endif ()
|
||||
|
||||
# Avoid adding new proto libraries to this list as these libraries are always
|
||||
# installed, regardless of whether or not they are needed. See #8022 for more
|
||||
# details.
|
||||
set(external_googleapis_installed_libraries_list
|
||||
# cmake-format: sort
|
||||
google_cloud_cpp_cloud_common_common_protos
|
||||
google_cloud_cpp_iam_credentials_v1_common_protos
|
||||
google_cloud_cpp_iam_credentials_v1_iamcredentials_protos
|
||||
google_cloud_cpp_iam_v1_iam_policy_protos
|
||||
google_cloud_cpp_iam_v1_options_protos
|
||||
google_cloud_cpp_iam_v1_policy_protos
|
||||
google_cloud_cpp_longrunning_operations_protos)
|
||||
|
||||
# These proto files cannot be added in the foreach() loop because they have
|
||||
# dependencies.
|
||||
set(PROTO_FILES_WITH_DEPENDENCIES
|
||||
# cmake-format: sort
|
||||
"google/api/annotations.proto"
|
||||
"google/api/auth.proto"
|
||||
"google/api/billing.proto"
|
||||
"google/api/client.proto"
|
||||
"google/api/control.proto"
|
||||
"google/api/distribution.proto"
|
||||
"google/api/endpoint.proto"
|
||||
"google/api/log.proto"
|
||||
"google/api/logging.proto"
|
||||
"google/api/metric.proto"
|
||||
"google/api/monitored_resource.proto"
|
||||
"google/api/monitoring.proto"
|
||||
"google/api/quota.proto"
|
||||
"google/api/service.proto"
|
||||
"google/api/usage.proto"
|
||||
"google/cloud/location/locations.proto"
|
||||
"google/rpc/status.proto")
|
||||
|
||||
# For some directories *most* (but not all) the proto files are simple enough
|
||||
# that the libraries can be generated with a foreach() loop.
|
||||
foreach (proto IN LISTS EXTERNAL_GOOGLEAPIS_PROTO_FILES)
|
||||
if (proto MATCHES "^google/api/"
|
||||
OR proto MATCHES "^google/type"
|
||||
OR proto MATCHES "^google/rpc/"
|
||||
OR proto MATCHES "^google/cloud/")
|
||||
external_googleapis_short_name(short_name "${proto}")
|
||||
list(APPEND external_googleapis_installed_libraries_list
|
||||
google_cloud_cpp_${short_name})
|
||||
list(FIND PROTO_FILES_WITH_DEPENDENCIES "${proto}" has_dependency)
|
||||
if (has_dependency EQUAL -1)
|
||||
external_googleapis_add_library("${proto}")
|
||||
endif ()
|
||||
endif ()
|
||||
endforeach ()
|
||||
|
||||
# Out of order because they have dependencies.
|
||||
external_googleapis_add_library("google/api/annotations.proto" api_http_protos)
|
||||
external_googleapis_add_library("google/api/auth.proto" api_annotations_protos)
|
||||
external_googleapis_add_library("google/api/client.proto"
|
||||
api_launch_stage_protos)
|
||||
external_googleapis_add_library("google/api/control.proto" api_policy_protos)
|
||||
external_googleapis_add_library("google/api/metric.proto"
|
||||
api_launch_stage_protos api_label_protos)
|
||||
external_googleapis_add_library("google/api/billing.proto"
|
||||
api_annotations_protos api_metric_protos)
|
||||
external_googleapis_add_library("google/api/distribution.proto"
|
||||
api_annotations_protos)
|
||||
external_googleapis_add_library("google/api/endpoint.proto"
|
||||
api_annotations_protos)
|
||||
external_googleapis_add_library("google/api/log.proto" api_label_protos)
|
||||
external_googleapis_add_library("google/api/logging.proto"
|
||||
api_annotations_protos api_label_protos)
|
||||
external_googleapis_add_library("google/api/monitored_resource.proto"
|
||||
api_launch_stage_protos api_label_protos)
|
||||
external_googleapis_add_library("google/api/monitoring.proto"
|
||||
api_annotations_protos)
|
||||
external_googleapis_add_library("google/api/quota.proto" api_annotations_protos)
|
||||
external_googleapis_add_library("google/api/usage.proto" api_annotations_protos
|
||||
api_visibility_protos)
|
||||
external_googleapis_add_library(
|
||||
"google/api/service.proto"
|
||||
api_annotations_protos
|
||||
api_auth_protos
|
||||
api_backend_protos
|
||||
api_billing_protos
|
||||
api_client_protos
|
||||
api_context_protos
|
||||
api_control_protos
|
||||
api_documentation_protos
|
||||
api_endpoint_protos
|
||||
api_http_protos
|
||||
api_label_protos
|
||||
api_log_protos
|
||||
api_logging_protos
|
||||
api_metric_protos
|
||||
api_monitored_resource_protos
|
||||
api_monitoring_protos
|
||||
api_quota_protos
|
||||
api_resource_protos
|
||||
api_source_info_protos
|
||||
api_system_parameter_protos
|
||||
api_usage_protos)
|
||||
|
||||
external_googleapis_add_library("google/cloud/location/locations.proto"
|
||||
api_annotations_protos api_client_protos)
|
||||
|
||||
external_googleapis_add_library("google/iam/v1/options.proto"
|
||||
api_annotations_protos)
|
||||
external_googleapis_add_library("google/iam/v1/policy.proto"
|
||||
api_annotations_protos type_expr_protos)
|
||||
external_googleapis_add_library("google/rpc/status.proto"
|
||||
rpc_error_details_protos)
|
||||
|
||||
external_googleapis_add_library(
|
||||
"google/longrunning/operations.proto" api_annotations_protos
|
||||
api_client_protos rpc_status_protos)
|
||||
|
||||
external_googleapis_add_library(
|
||||
"google/iam/v1/iam_policy.proto"
|
||||
api_annotations_protos
|
||||
api_client_protos
|
||||
api_field_behavior_protos
|
||||
api_resource_protos
|
||||
iam_v1_options_protos
|
||||
iam_v1_policy_protos)
|
||||
|
||||
external_googleapis_add_library("google/iam/credentials/v1/common.proto"
|
||||
api_field_behavior_protos api_resource_protos)
|
||||
|
||||
external_googleapis_add_library(
|
||||
"google/iam/credentials/v1/iamcredentials.proto" api_annotations_protos
|
||||
api_client_protos iam_credentials_v1_common_protos)
|
||||
|
||||
google_cloud_cpp_load_protolist(cloud_common_list "${GOOGLE_CLOUD_CPP_DIR}/external/googleapis/protolists/common.list")
|
||||
google_cloud_cpp_load_protodeps(cloud_common_deps "${GOOGLE_CLOUD_CPP_DIR}/external/googleapis/protodeps/common.deps")
|
||||
google_cloud_cpp_grpcpp_library(
|
||||
google_cloud_cpp_cloud_common_common_protos ${cloud_common_list}
|
||||
PROTO_PATH_DIRECTORIES "${EXTERNAL_GOOGLEAPIS_SOURCE}"
|
||||
"${PROTO_INCLUDE_DIR}")
|
||||
external_googleapis_set_version_and_alias(cloud_common_common_protos)
|
||||
target_link_libraries(google_cloud_cpp_cloud_common_common_protos
|
||||
PUBLIC ${cloud_common_deps})
|
||||
|
||||
# Install the libraries and headers in the locations determined by
|
||||
# GNUInstallDirs
|
||||
include(GNUInstallDirs)
|
||||
|
||||
install(
|
||||
TARGETS ${external_googleapis_installed_libraries_list}
|
||||
EXPORT googleapis-targets
|
||||
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
COMPONENT google_cloud_cpp_runtime
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
COMPONENT google_cloud_cpp_runtime
|
||||
NAMELINK_COMPONENT google_cloud_cpp_development
|
||||
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
COMPONENT google_cloud_cpp_development)
|
||||
|
||||
foreach (target ${external_googleapis_installed_libraries_list})
|
||||
google_cloud_cpp_install_proto_library_headers("${target}")
|
||||
google_cloud_cpp_install_proto_library_protos(
|
||||
"${target}" "${EXTERNAL_GOOGLEAPIS_SOURCE}")
|
||||
endforeach ()
|
||||
|
||||
# Create and install the pkg-config files.
|
||||
foreach (target ${external_googleapis_installed_libraries_list})
|
||||
external_googleapis_install_pc("${target}")
|
||||
endforeach ()
|
||||
|
||||
# Create and install the googleapis pkg-config file for backwards compatibility.
|
||||
set(GOOGLE_CLOUD_CPP_PC_LIBS "")
|
||||
google_cloud_cpp_set_pkgconfig_paths()
|
||||
set(GOOGLE_CLOUD_CPP_PC_NAME "The Google APIS C++ Proto Library")
|
||||
set(GOOGLE_CLOUD_CPP_PC_DESCRIPTION
|
||||
"Provides C++ APIs to access Google Cloud Platforms.")
|
||||
# This list is for backwards compatibility purposes only. DO NOT add new
|
||||
# libraries to it.
|
||||
string(
|
||||
JOIN
|
||||
" "
|
||||
GOOGLE_CLOUD_CPP_PC_REQUIRES
|
||||
"google_cloud_cpp_bigtable_protos"
|
||||
"google_cloud_cpp_cloud_bigquery_protos"
|
||||
"google_cloud_cpp_iam_protos"
|
||||
"google_cloud_cpp_pubsub_protos"
|
||||
"google_cloud_cpp_storage_protos"
|
||||
"google_cloud_cpp_logging_protos"
|
||||
"google_cloud_cpp_iam_v1_iam_policy_protos"
|
||||
"google_cloud_cpp_iam_v1_options_protos"
|
||||
"google_cloud_cpp_iam_v1_policy_protos"
|
||||
"google_cloud_cpp_longrunning_operations_protos"
|
||||
"google_cloud_cpp_api_auth_protos"
|
||||
"google_cloud_cpp_api_annotations_protos"
|
||||
"google_cloud_cpp_api_client_protos"
|
||||
"google_cloud_cpp_api_field_behavior_protos"
|
||||
"google_cloud_cpp_api_http_protos"
|
||||
"google_cloud_cpp_rpc_status_protos"
|
||||
"google_cloud_cpp_rpc_error_details_protos"
|
||||
"google_cloud_cpp_type_expr_protos"
|
||||
"grpc++"
|
||||
"grpc"
|
||||
"openssl"
|
||||
"protobuf"
|
||||
"zlib"
|
||||
"libcares")
|
||||
set(GOOGLE_CLOUD_CPP_PC_LIBS "")
|
||||
google_cloud_cpp_set_pkgconfig_paths()
|
||||
configure_file("${PROJECT_SOURCE_DIR}/cmake/templates/config.pc.in"
|
||||
"googleapis.pc" @ONLY)
|
||||
install(
|
||||
FILES "${CMAKE_CURRENT_BINARY_DIR}/googleapis.pc"
|
||||
DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig"
|
||||
COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# Create and install the CMake configuration files.
|
||||
# include(CMakePackageConfigHelpers)
|
||||
|
||||
# configure_file("${CMAKE_CURRENT_LIST_DIR}/config.cmake.in"
|
||||
# "google_cloud_cpp_googleapis-config.cmake" @ONLY)
|
||||
# write_basic_package_version_file(
|
||||
# "google_cloud_cpp_googleapis-config-version.cmake"
|
||||
# VERSION ${PROJECT_VERSION}
|
||||
# COMPATIBILITY ExactVersion)
|
||||
|
||||
# Export the CMake targets to make it easy to create configuration files.
|
||||
# install(
|
||||
# EXPORT googleapis-targets
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_googleapis"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
# install(
|
||||
# FILES
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_googleapis-config.cmake"
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_googleapis-config-version.cmake"
|
||||
# "${PROJECT_SOURCE_DIR}/cmake/FindgRPC.cmake"
|
||||
# "${PROJECT_SOURCE_DIR}/cmake/CompileProtos.cmake"
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_googleapis"
|
||||
# COMPONENT google_cloud_cpp_development)
|
447
contrib/google-cloud-cpp-cmake/google_cloud_cpp_common.cmake
Normal file
447
contrib/google-cloud-cpp-cmake/google_cloud_cpp_common.cmake
Normal file
@ -0,0 +1,447 @@
|
||||
# ~~~
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ~~~
|
||||
|
||||
# File copied from google-cloud-cpp/google-cloud-cpp/google_cloud_cpp_common.cmake with minor modifications.
|
||||
|
||||
set(GOOGLE_CLOUD_CPP_COMMON_DIR "${GOOGLE_CLOUD_CPP_DIR}/google/cloud")
|
||||
|
||||
# Generate the version information from the CMake values.
|
||||
# configure_file(${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/version_info.h.in
|
||||
# ${CMAKE_CURRENT_SOURCE_DIR}/internal/version_info.h)
|
||||
|
||||
# Create the file that captures build information. Having access to the compiler
|
||||
# and build flags at runtime allows us to print better benchmark results.
|
||||
string(TOUPPER "${CMAKE_BUILD_TYPE}" GOOGLE_CLOUD_CPP_BUILD_TYPE_UPPER)
|
||||
configure_file(${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/build_info.cc.in internal/build_info.cc)
|
||||
|
||||
# the client library
|
||||
add_library(
|
||||
google_cloud_cpp_common # cmake-format: sort
|
||||
${CMAKE_CURRENT_BINARY_DIR}/internal/build_info.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/access_token.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/access_token.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/backoff_policy.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/common_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/credentials.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/credentials.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/experimental_tag.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/future.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/future_generic.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/future_void.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/idempotency.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/absl_str_cat_quiet.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/absl_str_join_quiet.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/absl_str_replace_quiet.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/algorithm.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/api_client_header.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/api_client_header.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/attributes.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/auth_header_error.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/auth_header_error.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/backoff_policy.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/backoff_policy.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/base64_transforms.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/base64_transforms.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/big_endian.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/build_info.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/call_context.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/clock.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/compiler_info.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/compiler_info.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/compute_engine_util.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/compute_engine_util.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/credentials_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/credentials_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_future_status.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_future_status.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_string.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_string.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/detect_gcp.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/detect_gcp_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/detect_gcp_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/diagnostics_pop.inc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/diagnostics_push.inc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/disable_deprecation_warnings.inc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/disable_msvc_crt_secure_warnings.inc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/error_context.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/error_context.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/filesystem.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/filesystem.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/format_time_point.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/format_time_point.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/future_base.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/future_coroutines.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/future_fwd.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/future_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/future_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/future_then_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/future_then_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/getenv.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/getenv.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/group_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/invocation_id_generator.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/invocation_id_generator.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/invoke_result.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/ios_flags_saver.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/log_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/log_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/make_status.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/make_status.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/noexcept_action.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/noexcept_action.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/non_constructible.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/opentelemetry.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/opentelemetry.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/opentelemetry_context.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/opentelemetry_context.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/pagination_range.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/parse_rfc3339.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/parse_rfc3339.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/populate_common_options.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/populate_common_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/port_platform.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/random.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/random.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/retry_info.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/retry_loop_helpers.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/retry_loop_helpers.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/retry_policy_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/retry_policy_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/service_endpoint.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/service_endpoint.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/sha256_hash.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/sha256_hash.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/sha256_hmac.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/sha256_hmac.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/sha256_type.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/status_payload_keys.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/status_payload_keys.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/status_utils.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/status_utils.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/strerror.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/strerror.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/subject_token.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/subject_token.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/throw_delegate.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/throw_delegate.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/timer_queue.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/timer_queue.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/trace_propagator.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/trace_propagator.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/traced_stream_range.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/tuple.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/type_list.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/type_traits.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/url_encode.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/url_encode.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/user_agent_prefix.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/user_agent_prefix.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/utility.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/version_info.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/kms_key_name.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/kms_key_name.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/location.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/location.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/log.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/log.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/no_await_tag.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/opentelemetry_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/optional.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/options.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/polling_policy.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/project.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/project.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/retry_policy.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/rpc_metadata.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/status.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/status.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/status_or.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/stream_range.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/terminate_handler.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/terminate_handler.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/tracing_options.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/tracing_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/universe_domain_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/version.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/version.h)
|
||||
target_link_libraries(
|
||||
google_cloud_cpp_common
|
||||
PUBLIC absl::base
|
||||
absl::memory
|
||||
absl::optional
|
||||
absl::span
|
||||
absl::str_format
|
||||
absl::time
|
||||
absl::variant
|
||||
Threads::Threads)
|
||||
if (WIN32)
|
||||
target_compile_definitions(google_cloud_cpp_common
|
||||
PRIVATE WIN32_LEAN_AND_MEAN)
|
||||
target_link_libraries(google_cloud_cpp_common PUBLIC bcrypt)
|
||||
else ()
|
||||
target_link_libraries(google_cloud_cpp_common PUBLIC OpenSSL::Crypto ch_contrib::re2)
|
||||
endif ()
|
||||
|
||||
google_cloud_cpp_add_common_options(google_cloud_cpp_common)
|
||||
target_include_directories(
|
||||
google_cloud_cpp_common PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
|
||||
$<INSTALL_INTERFACE:include>)
|
||||
|
||||
# We're putting generated code into ${PROJECT_BINARY_DIR} (e.g. compiled
|
||||
# protobufs or build info), so we need it on the include path, however we don't
|
||||
# want it checked by linters so we mark it as SYSTEM.
|
||||
target_include_directories(google_cloud_cpp_common SYSTEM
|
||||
PUBLIC $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>)
|
||||
target_compile_options(google_cloud_cpp_common
|
||||
PUBLIC ${GOOGLE_CLOUD_CPP_EXCEPTIONS_FLAG})
|
||||
|
||||
set_target_properties(
|
||||
google_cloud_cpp_common
|
||||
PROPERTIES EXPORT_NAME "google-cloud-cpp::common"
|
||||
VERSION ${PROJECT_VERSION}
|
||||
SOVERSION ${PROJECT_VERSION_MAJOR})
|
||||
add_library(google-cloud-cpp::common ALIAS google_cloud_cpp_common)
|
||||
|
||||
#create_bazel_config(google_cloud_cpp_common YEAR 2018)
|
||||
|
||||
# # Export the CMake targets to make it easy to create configuration files.
|
||||
# install(
|
||||
# EXPORT google_cloud_cpp_common-targets
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_common"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# # Install the libraries and headers in the locations determined by
|
||||
# # GNUInstallDirs
|
||||
# install(
|
||||
# TARGETS google_cloud_cpp_common
|
||||
# EXPORT google_cloud_cpp_common-targets
|
||||
# RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
# COMPONENT google_cloud_cpp_runtime
|
||||
# LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
# COMPONENT google_cloud_cpp_runtime
|
||||
# NAMELINK_COMPONENT google_cloud_cpp_development
|
||||
# ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
#google_cloud_cpp_install_headers(google_cloud_cpp_common include/google/cloud)
|
||||
|
||||
# google_cloud_cpp_add_pkgconfig(
|
||||
# "common"
|
||||
# "Google Cloud C++ Client Library Common Components"
|
||||
# "Common Components used by the Google Cloud C++ Client Libraries."
|
||||
# "absl_optional"
|
||||
# "absl_span"
|
||||
# "absl_strings"
|
||||
# "absl_time"
|
||||
# "absl_time_zone"
|
||||
# "absl_variant"
|
||||
# "${GOOGLE_CLOUD_CPP_OPENTELEMETRY_API}"
|
||||
# NON_WIN32_REQUIRES
|
||||
# openssl
|
||||
# WIN32_LIBS
|
||||
# bcrypt)
|
||||
|
||||
# Create and install the CMake configuration files.
|
||||
# configure_file("config.cmake.in" "google_cloud_cpp_common-config.cmake" @ONLY)
|
||||
# write_basic_package_version_file(
|
||||
# "google_cloud_cpp_common-config-version.cmake"
|
||||
# VERSION ${PROJECT_VERSION}
|
||||
# COMPATIBILITY ExactVersion)
|
||||
|
||||
# install(
|
||||
# FILES
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_common-config.cmake"
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_common-config-version.cmake"
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_common"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# if (GOOGLE_CLOUD_CPP_WITH_MOCKS)
|
||||
# # Create a header-only library for the mocks. We use a CMake `INTERFACE`
|
||||
# # library for these, a regular library would not work on macOS (where the
|
||||
# # library needs at least one .o file).
|
||||
# add_library(google_cloud_cpp_mocks INTERFACE)
|
||||
# set(google_cloud_cpp_mocks_hdrs
|
||||
# # cmake-format: sort
|
||||
# mocks/current_options.h mocks/mock_async_streaming_read_write_rpc.h
|
||||
# mocks/mock_stream_range.h)
|
||||
# export_list_to_bazel("google_cloud_cpp_mocks.bzl"
|
||||
# "google_cloud_cpp_mocks_hdrs" YEAR "2022")
|
||||
# target_link_libraries(
|
||||
# google_cloud_cpp_mocks INTERFACE google-cloud-cpp::common GTest::gmock
|
||||
# GTest::gtest)
|
||||
# set_target_properties(google_cloud_cpp_mocks
|
||||
# PROPERTIES EXPORT_NAME google-cloud-cpp::mocks)
|
||||
# target_include_directories(
|
||||
# google_cloud_cpp_mocks
|
||||
# INTERFACE $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
|
||||
# $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}>
|
||||
# $<INSTALL_INTERFACE:include>)
|
||||
# target_compile_options(google_cloud_cpp_mocks
|
||||
# INTERFACE ${GOOGLE_CLOUD_CPP_EXCEPTIONS_FLAG})
|
||||
# add_library(google-cloud-cpp::mocks ALIAS google_cloud_cpp_mocks)
|
||||
|
||||
# install(
|
||||
# FILES ${google_cloud_cpp_mocks_hdrs}
|
||||
# DESTINATION "include/google/cloud/mocks"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# # Export the CMake targets to make it easy to create configuration files.
|
||||
# install(
|
||||
# EXPORT google_cloud_cpp_mocks-targets
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_mocks"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# install(
|
||||
# TARGETS google_cloud_cpp_mocks
|
||||
# EXPORT google_cloud_cpp_mocks-targets
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# google_cloud_cpp_add_pkgconfig(
|
||||
# "mocks" "Google Cloud C++ Testing Library"
|
||||
# "Helpers for testing the Google Cloud C++ Client Libraries"
|
||||
# "google_cloud_cpp_common" "gmock")
|
||||
|
||||
# # Create and install the CMake configuration files.
|
||||
# configure_file("mocks-config.cmake.in"
|
||||
# "google_cloud_cpp_mocks-config.cmake" @ONLY)
|
||||
# write_basic_package_version_file(
|
||||
# "google_cloud_cpp_mocks-config-version.cmake"
|
||||
# VERSION ${PROJECT_VERSION}
|
||||
# COMPATIBILITY ExactVersion)
|
||||
|
||||
# install(
|
||||
# FILES
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_mocks-config.cmake"
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_mocks-config-version.cmake"
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_mocks"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
# endif ()
|
||||
|
||||
# if (BUILD_TESTING)
|
||||
# include(FindBenchmarkWithWorkarounds)
|
||||
|
||||
# set(google_cloud_cpp_common_unit_tests
|
||||
# # cmake-format: sort
|
||||
# access_token_test.cc
|
||||
# common_options_test.cc
|
||||
# future_coroutines_test.cc
|
||||
# future_generic_test.cc
|
||||
# future_generic_then_test.cc
|
||||
# future_void_test.cc
|
||||
# future_void_then_test.cc
|
||||
# internal/algorithm_test.cc
|
||||
# internal/api_client_header_test.cc
|
||||
# internal/backoff_policy_test.cc
|
||||
# internal/base64_transforms_test.cc
|
||||
# internal/big_endian_test.cc
|
||||
# internal/call_context_test.cc
|
||||
# internal/clock_test.cc
|
||||
# internal/compiler_info_test.cc
|
||||
# internal/compute_engine_util_test.cc
|
||||
# internal/credentials_impl_test.cc
|
||||
# internal/debug_future_status_test.cc
|
||||
# internal/debug_string_test.cc
|
||||
# internal/detect_gcp_test.cc
|
||||
# internal/error_context_test.cc
|
||||
# internal/filesystem_test.cc
|
||||
# internal/format_time_point_test.cc
|
||||
# internal/future_impl_test.cc
|
||||
# internal/future_then_impl_test.cc
|
||||
# internal/group_options_test.cc
|
||||
# internal/invocation_id_generator_test.cc
|
||||
# internal/invoke_result_test.cc
|
||||
# internal/log_impl_test.cc
|
||||
# internal/make_status_test.cc
|
||||
# internal/noexcept_action_test.cc
|
||||
# internal/opentelemetry_context_test.cc
|
||||
# internal/opentelemetry_test.cc
|
||||
# internal/pagination_range_test.cc
|
||||
# internal/parse_rfc3339_test.cc
|
||||
# internal/populate_common_options_test.cc
|
||||
# internal/random_test.cc
|
||||
# internal/retry_loop_helpers_test.cc
|
||||
# internal/retry_policy_impl_test.cc
|
||||
# internal/service_endpoint_test.cc
|
||||
# internal/sha256_hash_test.cc
|
||||
# internal/sha256_hmac_test.cc
|
||||
# internal/status_payload_keys_test.cc
|
||||
# internal/status_utils_test.cc
|
||||
# internal/strerror_test.cc
|
||||
# internal/subject_token_test.cc
|
||||
# internal/throw_delegate_test.cc
|
||||
# internal/timer_queue_test.cc
|
||||
# internal/trace_propagator_test.cc
|
||||
# internal/traced_stream_range_test.cc
|
||||
# internal/tuple_test.cc
|
||||
# internal/type_list_test.cc
|
||||
# internal/url_encode_test.cc
|
||||
# internal/user_agent_prefix_test.cc
|
||||
# internal/utility_test.cc
|
||||
# kms_key_name_test.cc
|
||||
# location_test.cc
|
||||
# log_test.cc
|
||||
# mocks/current_options_test.cc
|
||||
# mocks/mock_stream_range_test.cc
|
||||
# options_test.cc
|
||||
# polling_policy_test.cc
|
||||
# project_test.cc
|
||||
# status_or_test.cc
|
||||
# status_test.cc
|
||||
# stream_range_test.cc
|
||||
# terminate_handler_test.cc
|
||||
# tracing_options_test.cc)
|
||||
|
||||
# # Export the list of unit tests so the Bazel BUILD file can pick it up.
|
||||
# export_list_to_bazel("google_cloud_cpp_common_unit_tests.bzl"
|
||||
# "google_cloud_cpp_common_unit_tests" YEAR "2018")
|
||||
|
||||
# foreach (fname ${google_cloud_cpp_common_unit_tests})
|
||||
# google_cloud_cpp_add_executable(target "common" "${fname}")
|
||||
# target_link_libraries(
|
||||
# ${target}
|
||||
# PRIVATE google_cloud_cpp_testing
|
||||
# google-cloud-cpp::common
|
||||
# google-cloud-cpp::mocks
|
||||
# absl::variant
|
||||
# GTest::gmock_main
|
||||
# GTest::gmock
|
||||
# GTest::gtest)
|
||||
# google_cloud_cpp_add_common_options(${target})
|
||||
# add_test(NAME ${target} COMMAND ${target})
|
||||
# endforeach ()
|
||||
|
||||
# set(google_cloud_cpp_common_benchmarks # cmake-format: sort
|
||||
# options_benchmark.cc)
|
||||
|
||||
# # Export the list of benchmarks to a .bzl file so we do not need to maintain
|
||||
# # the list in two places.
|
||||
# export_list_to_bazel("google_cloud_cpp_common_benchmarks.bzl"
|
||||
# "google_cloud_cpp_common_benchmarks" YEAR "2020")
|
||||
|
||||
# # Generate a target for each benchmark.
|
||||
# foreach (fname ${google_cloud_cpp_common_benchmarks})
|
||||
# google_cloud_cpp_add_executable(target "common" "${fname}")
|
||||
# add_test(NAME ${target} COMMAND ${target})
|
||||
# target_link_libraries(${target} PRIVATE google-cloud-cpp::common
|
||||
# benchmark::benchmark_main)
|
||||
# google_cloud_cpp_add_common_options(${target})
|
||||
# endforeach ()
|
||||
# endif ()
|
||||
|
||||
# if (BUILD_TESTING AND GOOGLE_CLOUD_CPP_ENABLE_CXX_EXCEPTIONS)
|
||||
# google_cloud_cpp_add_samples_relative("common" "samples/")
|
||||
# endif ()
|
350
contrib/google-cloud-cpp-cmake/google_cloud_cpp_grpc_utils.cmake
Normal file
350
contrib/google-cloud-cpp-cmake/google_cloud_cpp_grpc_utils.cmake
Normal file
@ -0,0 +1,350 @@
|
||||
# ~~~
|
||||
# Copyright 2022 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# https://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ~~~
|
||||
|
||||
# File copied from google-cloud-cpp/google-cloud-cpp/google_cloud_cpp_grpc_utils.cmake with minor modifications.
|
||||
|
||||
set(GOOGLE_CLOUD_CPP_COMMON_DIR "${GOOGLE_CLOUD_CPP_DIR}/google/cloud")
|
||||
|
||||
# the library
|
||||
add_library(
|
||||
google_cloud_cpp_grpc_utils # cmake-format: sort
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/async_operation.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/async_streaming_read_write_rpc.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/background_threads.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/completion_queue.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/completion_queue.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/connection_options.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/connection_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_error_delegate.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_error_delegate.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_options.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_utils/async_operation.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_utils/completion_queue.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_utils/grpc_error_delegate.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/grpc_utils/version.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/iam_updater.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_connection_ready.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_connection_ready.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_long_running_operation.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_polling_loop.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_polling_loop.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_read_stream_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_read_write_stream_auth.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_read_write_stream_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_read_write_stream_logging.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_read_write_stream_timeout.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_read_write_stream_tracing.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_retry_loop.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_retry_unary_rpc.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_rpc_details.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_read_rpc.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_read_rpc_auth.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_read_rpc_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_read_rpc_logging.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_read_rpc_timeout.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_read_rpc_tracing.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_write_rpc.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_write_rpc_auth.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_write_rpc_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_write_rpc_logging.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_write_rpc_timeout.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/async_streaming_write_rpc_tracing.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/background_threads_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/background_threads_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/completion_queue_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_string_protobuf.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_string_protobuf.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_string_status.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/debug_string_status.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/default_completion_queue_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/default_completion_queue_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/extract_long_running_result.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/extract_long_running_result.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_access_token_authentication.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_access_token_authentication.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_api_key_authentication.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_api_key_authentication.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_async_access_token_cache.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_async_access_token_cache.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_channel_credentials_authentication.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_channel_credentials_authentication.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_impersonate_service_account.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_impersonate_service_account.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_metadata_view.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_opentelemetry.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_opentelemetry.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_request_metadata.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_request_metadata.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_service_account_authentication.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/grpc_service_account_authentication.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/log_wrapper.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/log_wrapper.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/minimal_iam_credentials_stub.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/minimal_iam_credentials_stub.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/populate_grpc_options.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/populate_grpc_options.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/resumable_streaming_read_rpc.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/retry_loop.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/routing_matcher.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/setup_context.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_read_rpc.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_read_rpc.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_read_rpc_logging.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_read_rpc_tracing.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_write_rpc.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_write_rpc_impl.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_write_rpc_impl.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_write_rpc_logging.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/streaming_write_rpc_tracing.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/time_utils.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/time_utils.h
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/unified_grpc_credentials.cc
|
||||
${GOOGLE_CLOUD_CPP_COMMON_DIR}/internal/unified_grpc_credentials.h)
|
||||
target_link_libraries(
|
||||
google_cloud_cpp_grpc_utils
|
||||
PUBLIC absl::function_ref
|
||||
absl::memory
|
||||
absl::time
|
||||
absl::variant
|
||||
google-cloud-cpp::iam_credentials_v1_iamcredentials_protos
|
||||
google-cloud-cpp::iam_v1_policy_protos
|
||||
google-cloud-cpp::longrunning_operations_protos
|
||||
google-cloud-cpp::iam_v1_iam_policy_protos
|
||||
google-cloud-cpp::rpc_error_details_protos
|
||||
google-cloud-cpp::rpc_status_protos
|
||||
google-cloud-cpp::common
|
||||
gRPC::grpc++
|
||||
gRPC::grpc)
|
||||
google_cloud_cpp_add_common_options(google_cloud_cpp_grpc_utils)
|
||||
target_include_directories(
|
||||
google_cloud_cpp_grpc_utils PUBLIC $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}>
|
||||
$<INSTALL_INTERFACE:include>)
|
||||
target_compile_options(google_cloud_cpp_grpc_utils
|
||||
PUBLIC ${GOOGLE_CLOUD_CPP_EXCEPTIONS_FLAG})
|
||||
set_target_properties(
|
||||
google_cloud_cpp_grpc_utils
|
||||
PROPERTIES EXPORT_NAME "google-cloud-cpp::grpc_utils"
|
||||
VERSION ${PROJECT_VERSION}
|
||||
SOVERSION ${PROJECT_VERSION_MAJOR})
|
||||
add_library(google-cloud-cpp::grpc_utils ALIAS google_cloud_cpp_grpc_utils)
|
||||
|
||||
#create_bazel_config(google_cloud_cpp_grpc_utils YEAR 2019)
|
||||
|
||||
# # Install the libraries and headers in the locations determined by
|
||||
# # GNUInstallDirs
|
||||
# install(
|
||||
# TARGETS
|
||||
# EXPORT grpc_utils-targets
|
||||
# RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
# LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
# ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# # Export the CMake targets to make it easy to create configuration files.
|
||||
# install(
|
||||
# EXPORT grpc_utils-targets
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_grpc_utils"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# install(
|
||||
# TARGETS google_cloud_cpp_grpc_utils
|
||||
# EXPORT grpc_utils-targets
|
||||
# RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
||||
# COMPONENT google_cloud_cpp_runtime
|
||||
# LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
# COMPONENT google_cloud_cpp_runtime
|
||||
# NAMELINK_COMPONENT google_cloud_cpp_development
|
||||
# ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# google_cloud_cpp_install_headers(google_cloud_cpp_grpc_utils
|
||||
# include/google/cloud)
|
||||
|
||||
# google_cloud_cpp_add_pkgconfig(
|
||||
# grpc_utils
|
||||
# "gRPC Utilities for the Google Cloud C++ Client Library"
|
||||
# "Provides gRPC Utilities for the Google Cloud C++ Client Library."
|
||||
# "google_cloud_cpp_common"
|
||||
# "google_cloud_cpp_iam_credentials_v1_iamcredentials_protos"
|
||||
# "google_cloud_cpp_iam_v1_policy_protos"
|
||||
# "google_cloud_cpp_iam_v1_iam_policy_protos"
|
||||
# "google_cloud_cpp_longrunning_operations_protos"
|
||||
# "google_cloud_cpp_rpc_status_protos"
|
||||
# "absl_function_ref"
|
||||
# "absl_strings"
|
||||
# "absl_time"
|
||||
# "absl_time_zone"
|
||||
# "absl_variant"
|
||||
# "openssl")
|
||||
|
||||
# # Create and install the CMake configuration files.
|
||||
# configure_file("grpc_utils/config.cmake.in"
|
||||
# "google_cloud_cpp_grpc_utils-config.cmake" @ONLY)
|
||||
# write_basic_package_version_file(
|
||||
# "google_cloud_cpp_grpc_utils-config-version.cmake"
|
||||
# VERSION ${PROJECT_VERSION}
|
||||
# COMPATIBILITY ExactVersion)
|
||||
|
||||
# install(
|
||||
# FILES
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_grpc_utils-config.cmake"
|
||||
# "${CMAKE_CURRENT_BINARY_DIR}/google_cloud_cpp_grpc_utils-config-version.cmake"
|
||||
# DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/google_cloud_cpp_grpc_utils"
|
||||
# COMPONENT google_cloud_cpp_development)
|
||||
|
||||
# function (google_cloud_cpp_grpc_utils_add_test fname labels)
|
||||
# google_cloud_cpp_add_executable(target "common" "${fname}")
|
||||
# target_link_libraries(
|
||||
# ${target}
|
||||
# PRIVATE google-cloud-cpp::grpc_utils
|
||||
# google_cloud_cpp_testing_grpc
|
||||
# google_cloud_cpp_testing
|
||||
# google-cloud-cpp::common
|
||||
# absl::variant
|
||||
# GTest::gmock_main
|
||||
# GTest::gmock
|
||||
# GTest::gtest
|
||||
# gRPC::grpc++
|
||||
# gRPC::grpc)
|
||||
# google_cloud_cpp_add_common_options(${target})
|
||||
# add_test(NAME ${target} COMMAND ${target})
|
||||
# set_tests_properties(${target} PROPERTIES LABELS "${labels}")
|
||||
# endfunction ()
|
||||
|
||||
# if (BUILD_TESTING)
|
||||
# include(FindBenchmarkWithWorkarounds)
|
||||
|
||||
# # List the unit tests, then setup the targets and dependencies.
|
||||
# set(google_cloud_cpp_grpc_utils_unit_tests
|
||||
# # cmake-format: sort
|
||||
# completion_queue_test.cc
|
||||
# connection_options_test.cc
|
||||
# grpc_error_delegate_test.cc
|
||||
# grpc_options_test.cc
|
||||
# internal/async_connection_ready_test.cc
|
||||
# internal/async_long_running_operation_test.cc
|
||||
# internal/async_polling_loop_test.cc
|
||||
# internal/async_read_write_stream_auth_test.cc
|
||||
# internal/async_read_write_stream_impl_test.cc
|
||||
# internal/async_read_write_stream_logging_test.cc
|
||||
# internal/async_read_write_stream_timeout_test.cc
|
||||
# internal/async_read_write_stream_tracing_test.cc
|
||||
# internal/async_retry_loop_test.cc
|
||||
# internal/async_retry_unary_rpc_test.cc
|
||||
# internal/async_streaming_read_rpc_auth_test.cc
|
||||
# internal/async_streaming_read_rpc_impl_test.cc
|
||||
# internal/async_streaming_read_rpc_logging_test.cc
|
||||
# internal/async_streaming_read_rpc_timeout_test.cc
|
||||
# internal/async_streaming_read_rpc_tracing_test.cc
|
||||
# internal/async_streaming_write_rpc_auth_test.cc
|
||||
# internal/async_streaming_write_rpc_impl_test.cc
|
||||
# internal/async_streaming_write_rpc_logging_test.cc
|
||||
# internal/async_streaming_write_rpc_timeout_test.cc
|
||||
# internal/async_streaming_write_rpc_tracing_test.cc
|
||||
# internal/background_threads_impl_test.cc
|
||||
# internal/debug_string_protobuf_test.cc
|
||||
# internal/debug_string_status_test.cc
|
||||
# internal/extract_long_running_result_test.cc
|
||||
# internal/grpc_access_token_authentication_test.cc
|
||||
# internal/grpc_async_access_token_cache_test.cc
|
||||
# internal/grpc_channel_credentials_authentication_test.cc
|
||||
# internal/grpc_opentelemetry_test.cc
|
||||
# internal/grpc_request_metadata_test.cc
|
||||
# internal/grpc_service_account_authentication_test.cc
|
||||
# internal/log_wrapper_test.cc
|
||||
# internal/minimal_iam_credentials_stub_test.cc
|
||||
# internal/populate_grpc_options_test.cc
|
||||
# internal/resumable_streaming_read_rpc_test.cc
|
||||
# internal/retry_loop_test.cc
|
||||
# internal/routing_matcher_test.cc
|
||||
# internal/streaming_read_rpc_logging_test.cc
|
||||
# internal/streaming_read_rpc_test.cc
|
||||
# internal/streaming_read_rpc_tracing_test.cc
|
||||
# internal/streaming_write_rpc_logging_test.cc
|
||||
# internal/streaming_write_rpc_test.cc
|
||||
# internal/streaming_write_rpc_tracing_test.cc
|
||||
# internal/time_utils_test.cc
|
||||
# internal/unified_grpc_credentials_test.cc)
|
||||
|
||||
# # List the unit tests, then setup the targets and dependencies.
|
||||
# set(google_cloud_cpp_grpc_utils_integration_tests
|
||||
# # cmake-format: sort
|
||||
# internal/grpc_impersonate_service_account_integration_test.cc)
|
||||
|
||||
# # Export the list of unit and integration tests so the Bazel BUILD file can
|
||||
# # pick them up.
|
||||
# export_list_to_bazel("google_cloud_cpp_grpc_utils_unit_tests.bzl"
|
||||
# "google_cloud_cpp_grpc_utils_unit_tests" YEAR "2019")
|
||||
# export_list_to_bazel(
|
||||
# "google_cloud_cpp_grpc_utils_integration_tests.bzl"
|
||||
# "google_cloud_cpp_grpc_utils_integration_tests" YEAR "2021")
|
||||
|
||||
# foreach (fname ${google_cloud_cpp_grpc_utils_unit_tests})
|
||||
# google_cloud_cpp_grpc_utils_add_test("${fname}" "")
|
||||
# endforeach ()
|
||||
|
||||
# # TODO(#12485) - remove dependency on bigtable in this integration test.
|
||||
# if (NOT bigtable IN_LIST GOOGLE_CLOUD_CPP_ENABLE)
|
||||
# list(REMOVE_ITEM google_cloud_cpp_grpc_utils_integration_tests
|
||||
# "internal/grpc_impersonate_service_account_integration_test.cc")
|
||||
# endif ()
|
||||
|
||||
# foreach (fname ${google_cloud_cpp_grpc_utils_integration_tests})
|
||||
# google_cloud_cpp_add_executable(target "common" "${fname}")
|
||||
# target_link_libraries(
|
||||
# ${target}
|
||||
# PRIVATE google-cloud-cpp::grpc_utils
|
||||
# google_cloud_cpp_testing_grpc
|
||||
# google_cloud_cpp_testing
|
||||
# google-cloud-cpp::common
|
||||
# google-cloud-cpp::iam_credentials_v1_iamcredentials_protos
|
||||
# absl::variant
|
||||
# GTest::gmock_main
|
||||
# GTest::gmock
|
||||
# GTest::gtest
|
||||
# gRPC::grpc++
|
||||
# gRPC::grpc)
|
||||
# google_cloud_cpp_add_common_options(${target})
|
||||
# add_test(NAME ${target} COMMAND ${target})
|
||||
# set_tests_properties(${target} PROPERTIES LABELS
|
||||
# "integration-test-production")
|
||||
# # TODO(12485) - remove dep on bigtable_protos
|
||||
# if (bigtable IN_LIST GOOGLE_CLOUD_CPP_ENABLE)
|
||||
# target_link_libraries(${target}
|
||||
# PRIVATE google-cloud-cpp::bigtable_protos)
|
||||
# endif ()
|
||||
# endforeach ()
|
||||
|
||||
# set(google_cloud_cpp_grpc_utils_benchmarks # cmake-format: sortable
|
||||
# completion_queue_benchmark.cc)
|
||||
|
||||
# # Export the list of benchmarks to a .bzl file so we do not need to maintain
|
||||
# # the list in two places.
|
||||
# export_list_to_bazel("google_cloud_cpp_grpc_utils_benchmarks.bzl"
|
||||
# "google_cloud_cpp_grpc_utils_benchmarks" YEAR "2020")
|
||||
|
||||
# # Generate a target for each benchmark.
|
||||
# foreach (fname ${google_cloud_cpp_grpc_utils_benchmarks})
|
||||
# google_cloud_cpp_add_executable(target "common" "${fname}")
|
||||
# add_test(NAME ${target} COMMAND ${target})
|
||||
# target_link_libraries(
|
||||
# ${target}
|
||||
# PRIVATE google-cloud-cpp::grpc_utils google-cloud-cpp::common
|
||||
# benchmark::benchmark_main)
|
||||
# google_cloud_cpp_add_common_options(${target})
|
||||
# endforeach ()
|
||||
# endif ()
|
Binary file not shown.
1
contrib/jwt-cpp
vendored
Submodule
1
contrib/jwt-cpp
vendored
Submodule
@ -0,0 +1 @@
|
||||
Subproject commit a6927cb8140858c34e05d1a954626b9849fbcdfc
|
23
contrib/jwt-cpp-cmake/CMakeLists.txt
Normal file
23
contrib/jwt-cpp-cmake/CMakeLists.txt
Normal file
@ -0,0 +1,23 @@
|
||||
set(ENABLE_JWT_CPP_DEFAULT OFF)
|
||||
if(ENABLE_LIBRARIES AND CLICKHOUSE_CLOUD)
|
||||
set(ENABLE_JWT_CPP_DEFAULT ON)
|
||||
endif()
|
||||
|
||||
option(ENABLE_JWT_CPP "Enable jwt-cpp library" ${ENABLE_JWT_CPP_DEFAULT})
|
||||
|
||||
if (NOT ENABLE_JWT_CPP)
|
||||
message(STATUS "Not using jwt-cpp")
|
||||
return()
|
||||
endif()
|
||||
|
||||
if(ENABLE_JWT_CPP)
|
||||
if(NOT TARGET OpenSSL::Crypto)
|
||||
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use jwt-cpp without OpenSSL")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
set (JWT_CPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jwt-cpp/include")
|
||||
|
||||
add_library (_jwt-cpp INTERFACE)
|
||||
target_include_directories(_jwt-cpp SYSTEM BEFORE INTERFACE ${JWT_CPP_INCLUDE_DIR})
|
||||
add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp)
|
2
contrib/update-submodules.sh
vendored
2
contrib/update-submodules.sh
vendored
@ -24,7 +24,7 @@ git config --file .gitmodules --get-regexp '.*path' | sed 's/[^ ]* //' | xargs -
|
||||
# We don't want to depend on any third-party CMake files.
|
||||
# To check it, find and delete them.
|
||||
grep -o -P '"contrib/[^"]+"' .gitmodules |
|
||||
grep -v -P 'contrib/(llvm-project|google-protobuf|grpc|abseil-cpp|corrosion|aws-crt-cpp)' |
|
||||
grep -v -P 'contrib/(llvm-project|google-protobuf|grpc|abseil-cpp|corrosion|aws-crt-cpp|google-cloud-cpp)' |
|
||||
xargs -I@ find @ \
|
||||
-'(' -name 'CMakeLists.txt' -or -name '*.cmake' -')' -and -not -name '*.h.cmake' \
|
||||
-delete
|
||||
|
@ -31,14 +31,14 @@ COPY entrypoint.sh /entrypoint.sh
|
||||
ARG TARGETARCH
|
||||
RUN arch=${TARGETARCH:-amd64} \
|
||||
&& case $arch in \
|
||||
amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.31.so /lib64/ld-linux-x86-64.so.2 ;; \
|
||||
arm64) ln -sf /lib/ld-2.31.so /lib/ld-linux-aarch64.so.1 ;; \
|
||||
amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.35.so /lib64/ld-linux-x86-64.so.2 ;; \
|
||||
arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
|
||||
esac
|
||||
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.10.1.2812"
|
||||
ARG VERSION="24.10.2.80"
|
||||
ARG PACKAGES="clickhouse-keeper"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
@ -86,7 +86,8 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
ARG DEFAULT_CONFIG_DIR="/etc/clickhouse-keeper"
|
||||
ARG DEFAULT_DATA_DIR="/var/lib/clickhouse-keeper"
|
||||
ARG DEFAULT_LOG_DIR="/var/log/clickhouse-keeper"
|
||||
RUN mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \
|
||||
RUN clickhouse-keeper --version \
|
||||
&& mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \
|
||||
&& chown clickhouse:clickhouse "${DEFAULT_DATA_DIR}" \
|
||||
&& chown root:clickhouse "${DEFAULT_LOG_DIR}" \
|
||||
&& chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}"
|
||||
|
@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
|
||||
# lts / testing / prestable / etc
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
|
||||
ARG VERSION="24.10.1.2812"
|
||||
ARG VERSION="24.10.2.80"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
ARG DIRECT_DOWNLOAD_URLS=""
|
||||
|
||||
|
@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
|
||||
|
||||
ARG REPO_CHANNEL="stable"
|
||||
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
|
||||
ARG VERSION="24.10.1.2812"
|
||||
ARG VERSION="24.10.2.80"
|
||||
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
|
||||
|
||||
#docker-official-library:off
|
||||
|
61
docs/changelogs/v24.10.2.80-stable.md
Normal file
61
docs/changelogs/v24.10.2.80-stable.md
Normal file
@ -0,0 +1,61 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.10.2.80-stable (96b80057159) FIXME as compared to v24.10.1.2812-stable (9cd0a3738d5)
|
||||
|
||||
#### Backward Incompatible Change
|
||||
* Backported in [#71363](https://github.com/ClickHouse/ClickHouse/issues/71363): Fix possible error `No such file or directory` due to unescaped special symbols in files for JSON subcolumns. [#71182](https://github.com/ClickHouse/ClickHouse/pull/71182) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
|
||||
#### Performance Improvement
|
||||
* Backported in [#71852](https://github.com/ClickHouse/ClickHouse/issues/71852): Improve the performance and accuracy of system.query_metric_log collection interval by reducing the critical region. [#71473](https://github.com/ClickHouse/ClickHouse/pull/71473) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#71495](https://github.com/ClickHouse/ClickHouse/issues/71495): Enable `parallel_replicas_local_plan` by default. Building a full-fledged local plan on the query initiator improves parallel replicas performance with less resource consumption, provides opportunities to apply more query optimizations. [#70171](https://github.com/ClickHouse/ClickHouse/pull/70171) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Backported in [#71985](https://github.com/ClickHouse/ClickHouse/issues/71985): Fixes RIGHT / FULL joins in queries with parallel replicas. Now, RIGHT joins can be executed with parallel replicas (right table reading is distributed). FULL joins can't be parallelized among nodes, - executed locally. [#71162](https://github.com/ClickHouse/ClickHouse/pull/71162) ([Igor Nikonov](https://github.com/devcrafter)).
|
||||
* Backported in [#71670](https://github.com/ClickHouse/ClickHouse/issues/71670): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71940](https://github.com/ClickHouse/ClickHouse/issues/71940): Update `HostResolver` 3 times in a `history` period. [#71863](https://github.com/ClickHouse/ClickHouse/pull/71863) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
* Backported in [#71922](https://github.com/ClickHouse/ClickHouse/issues/71922): Allow_reorder_prewhere_conditions is on by default with old compatibility settings. [#71867](https://github.com/ClickHouse/ClickHouse/pull/71867) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#71588](https://github.com/ClickHouse/ClickHouse/issues/71588): Fix mismatched aggreage function name of quantileExactWeightedInterpolated. The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/69619. cc @Algunenano. [#71168](https://github.com/ClickHouse/ClickHouse/pull/71168) ([李扬](https://github.com/taiyang-li)).
|
||||
* Backported in [#71357](https://github.com/ClickHouse/ClickHouse/issues/71357): Fix bad_weak_ptr exception with Dynamic in functions comparison. [#71183](https://github.com/ClickHouse/ClickHouse/pull/71183) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71467](https://github.com/ClickHouse/ClickHouse/issues/71467): Fix bug of memory usage increase if enable_filesystem_cache=1, but disk in storage configuration did not have any cache configuration. [#71261](https://github.com/ClickHouse/ClickHouse/pull/71261) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#71355](https://github.com/ClickHouse/ClickHouse/issues/71355): Fix possible error "Cannot read all data" erros during deserialization of LowCardinality dictionary from Dynamic column. [#71299](https://github.com/ClickHouse/ClickHouse/pull/71299) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71324](https://github.com/ClickHouse/ClickHouse/issues/71324): Fix incomplete cleanup of parallel output format in the client. [#71304](https://github.com/ClickHouse/ClickHouse/pull/71304) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71466](https://github.com/ClickHouse/ClickHouse/issues/71466): Added missing unescaping in named collections. Without fix clickhouse-server can't start. [#71308](https://github.com/ClickHouse/ClickHouse/pull/71308) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Backported in [#71393](https://github.com/ClickHouse/ClickHouse/issues/71393): Fix inconsistent AST formatting when granting wrong wildcard grants [#71309](https://github.com/ClickHouse/ClickHouse/issues/71309). [#71332](https://github.com/ClickHouse/ClickHouse/pull/71332) ([pufit](https://github.com/pufit)).
|
||||
* Backported in [#71379](https://github.com/ClickHouse/ClickHouse/issues/71379): Add try/catch to data parts destructors to avoid terminate. [#71364](https://github.com/ClickHouse/ClickHouse/pull/71364) ([alesapin](https://github.com/alesapin)).
|
||||
* Backported in [#71751](https://github.com/ClickHouse/ClickHouse/issues/71751): Check suspicious and experimental types in JSON type hints. [#71369](https://github.com/ClickHouse/ClickHouse/pull/71369) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71451](https://github.com/ClickHouse/ClickHouse/issues/71451): Start memory worker thread on non-Linux OS too (fixes [#71051](https://github.com/ClickHouse/ClickHouse/issues/71051)). [#71384](https://github.com/ClickHouse/ClickHouse/pull/71384) ([Alexandre Snarskii](https://github.com/snar)).
|
||||
* Backported in [#71608](https://github.com/ClickHouse/ClickHouse/issues/71608): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71493](https://github.com/ClickHouse/ClickHouse/issues/71493): Fix crash in `mongodb` table function when passing wrong arguments (e.g. `NULL`). [#71426](https://github.com/ClickHouse/ClickHouse/pull/71426) ([Vladimir Cherkasov](https://github.com/vdimir)).
|
||||
* Backported in [#71815](https://github.com/ClickHouse/ClickHouse/issues/71815): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71521](https://github.com/ClickHouse/ClickHouse/issues/71521): Fix possible error `Argument for function must be constant` (old analyzer) in case when arrayJoin can apparently appear in `WHERE` condition. Regression after https://github.com/ClickHouse/ClickHouse/pull/65414. [#71476](https://github.com/ClickHouse/ClickHouse/pull/71476) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#71555](https://github.com/ClickHouse/ClickHouse/issues/71555): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71618](https://github.com/ClickHouse/ClickHouse/issues/71618): Analyzer fix when query inside materialized view uses IN with CTE. Closes [#65598](https://github.com/ClickHouse/ClickHouse/issues/65598). [#71538](https://github.com/ClickHouse/ClickHouse/pull/71538) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#71570](https://github.com/ClickHouse/ClickHouse/issues/71570): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71646](https://github.com/ClickHouse/ClickHouse/issues/71646): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#71880](https://github.com/ClickHouse/ClickHouse/issues/71880): Fix LOGICAL_ERROR when doing ALTER with empty tuple. This fixes [#71647](https://github.com/ClickHouse/ClickHouse/issues/71647). [#71679](https://github.com/ClickHouse/ClickHouse/pull/71679) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#71741](https://github.com/ClickHouse/ClickHouse/issues/71741): Don't transform constant set in predicates over partition columns in case of NOT IN operator. [#71695](https://github.com/ClickHouse/ClickHouse/pull/71695) ([Eduard Karacharov](https://github.com/korowa)).
|
||||
* Backported in [#72012](https://github.com/ClickHouse/ClickHouse/issues/72012): Fix exception for toDayOfWeek on WHERE condition with primary key of DateTime64 type. [#71849](https://github.com/ClickHouse/ClickHouse/pull/71849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
|
||||
* Backported in [#71897](https://github.com/ClickHouse/ClickHouse/issues/71897): Fixed filling of defaults after parsing into sparse columns. [#71854](https://github.com/ClickHouse/ClickHouse/pull/71854) ([Anton Popov](https://github.com/CurtizJ)).
|
||||
* Backported in [#71955](https://github.com/ClickHouse/ClickHouse/issues/71955): Fix data race between the progress indicator and the progress table in clickhouse-client. This issue is visible when FROM INFILE is used. Intercept keystrokes during INSERT queries to toggle progress table display. [#71901](https://github.com/ClickHouse/ClickHouse/pull/71901) ([Julia Kartseva](https://github.com/jkartseva)).
|
||||
* Backported in [#72006](https://github.com/ClickHouse/ClickHouse/issues/72006): Fix a crash in clickhouse-client syntax highlighting. Closes [#71864](https://github.com/ClickHouse/ClickHouse/issues/71864). [#71949](https://github.com/ClickHouse/ClickHouse/pull/71949) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#71692](https://github.com/ClickHouse/ClickHouse/issues/71692): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#71387](https://github.com/ClickHouse/ClickHouse/issues/71387): Remove bad test `test_system_replicated_fetches`. [#71071](https://github.com/ClickHouse/ClickHouse/pull/71071) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
|
||||
* Backported in [#71586](https://github.com/ClickHouse/ClickHouse/issues/71586): Fix `WITH TOTALS` in subquery with parallel replicas. [#71224](https://github.com/ClickHouse/ClickHouse/pull/71224) ([Nikita Taranov](https://github.com/nickitat)).
|
||||
* Backported in [#71437](https://github.com/ClickHouse/ClickHouse/issues/71437): Ignore `No such key` exceptions in some cases. [#71236](https://github.com/ClickHouse/ClickHouse/pull/71236) ([Antonio Andelic](https://github.com/antonio2368)).
|
||||
* Backported in [#71629](https://github.com/ClickHouse/ClickHouse/issues/71629): Fix compatibility with refreshable materialized views created by old clickhouse servers. [#71556](https://github.com/ClickHouse/ClickHouse/pull/71556) ([Michael Kolupaev](https://github.com/al13n321)).
|
||||
* Backported in [#71805](https://github.com/ClickHouse/ClickHouse/issues/71805): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71832](https://github.com/ClickHouse/ClickHouse/issues/71832): Closes [#71780](https://github.com/ClickHouse/ClickHouse/issues/71780). [#71818](https://github.com/ClickHouse/ClickHouse/pull/71818) ([Kseniia Sumarokova](https://github.com/kssenii)).
|
||||
* Backported in [#71840](https://github.com/ClickHouse/ClickHouse/issues/71840): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
31
docs/changelogs/v24.3.14.35-lts.md
Normal file
31
docs/changelogs/v24.3.14.35-lts.md
Normal file
@ -0,0 +1,31 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.3.14.35-lts (cfa4e62b775) FIXME as compared to v24.3.13.40-lts (7acabd77389)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#71711](https://github.com/ClickHouse/ClickHouse/issues/71711): CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
|
||||
* Backported in [#71662](https://github.com/ClickHouse/ClickHouse/issues/71662): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#65755](https://github.com/ClickHouse/ClickHouse/issues/65755): Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#71600](https://github.com/ClickHouse/ClickHouse/issues/71600): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71842](https://github.com/ClickHouse/ClickHouse/issues/71842): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71562](https://github.com/ClickHouse/ClickHouse/issues/71562): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71731](https://github.com/ClickHouse/ClickHouse/issues/71731): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#71697](https://github.com/ClickHouse/ClickHouse/issues/71697): Vendor in rust dependencies. [#62297](https://github.com/ClickHouse/ClickHouse/pull/62297) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71688](https://github.com/ClickHouse/ClickHouse/issues/71688): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#71808](https://github.com/ClickHouse/ClickHouse/issues/71808): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71834](https://github.com/ClickHouse/ClickHouse/issues/71834): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Fix bitShift test after backport. [#71861](https://github.com/ClickHouse/ClickHouse/pull/71861) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Revert "Merge pull request [#71861](https://github.com/ClickHouse/ClickHouse/issues/71861) from pamarcos/fix-bitshift-test". [#71871](https://github.com/ClickHouse/ClickHouse/pull/71871) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
|
37
docs/changelogs/v24.8.7.41-lts.md
Normal file
37
docs/changelogs/v24.8.7.41-lts.md
Normal file
@ -0,0 +1,37 @@
|
||||
---
|
||||
sidebar_position: 1
|
||||
sidebar_label: 2024
|
||||
---
|
||||
|
||||
# 2024 Changelog
|
||||
|
||||
### ClickHouse release v24.8.7.41-lts (e28553d4f2b) FIXME as compared to v24.8.6.70-lts (ddb8c219771)
|
||||
|
||||
#### Improvement
|
||||
* Backported in [#71713](https://github.com/ClickHouse/ClickHouse/issues/71713): CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
|
||||
* Backported in [#71666](https://github.com/ClickHouse/ClickHouse/issues/71666): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71936](https://github.com/ClickHouse/ClickHouse/issues/71936): Update `HostResolver` 3 times in a `history` period. [#71863](https://github.com/ClickHouse/ClickHouse/pull/71863) ([Sema Checherinda](https://github.com/CheSema)).
|
||||
|
||||
#### Bug Fix (user-visible misbehavior in an official stable release)
|
||||
* Backported in [#71486](https://github.com/ClickHouse/ClickHouse/issues/71486): Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
|
||||
* Backported in [#71462](https://github.com/ClickHouse/ClickHouse/issues/71462): Added missing unescaping in named collections. Without fix clickhouse-server can't start. [#71308](https://github.com/ClickHouse/ClickHouse/pull/71308) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
|
||||
* Backported in [#71747](https://github.com/ClickHouse/ClickHouse/issues/71747): Check suspicious and experimental types in JSON type hints. [#71369](https://github.com/ClickHouse/ClickHouse/pull/71369) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71604](https://github.com/ClickHouse/ClickHouse/issues/71604): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
|
||||
* Backported in [#71826](https://github.com/ClickHouse/ClickHouse/issues/71826): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71517](https://github.com/ClickHouse/ClickHouse/issues/71517): Fix possible error `Argument for function must be constant` (old analyzer) in case when arrayJoin can apparently appear in `WHERE` condition. Regression after https://github.com/ClickHouse/ClickHouse/pull/65414. [#71476](https://github.com/ClickHouse/ClickHouse/pull/71476) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
|
||||
* Backported in [#71551](https://github.com/ClickHouse/ClickHouse/issues/71551): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71614](https://github.com/ClickHouse/ClickHouse/issues/71614): Analyzer fix when query inside materialized view uses IN with CTE. Closes [#65598](https://github.com/ClickHouse/ClickHouse/issues/65598). [#71538](https://github.com/ClickHouse/ClickHouse/pull/71538) ([Maksim Kita](https://github.com/kitaisreal)).
|
||||
* Backported in [#71566](https://github.com/ClickHouse/ClickHouse/issues/71566): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
|
||||
* Backported in [#71727](https://github.com/ClickHouse/ClickHouse/issues/71727): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
|
||||
* Backported in [#71876](https://github.com/ClickHouse/ClickHouse/issues/71876): Fix LOGICAL_ERROR when doing ALTER with empty tuple. This fixes [#71647](https://github.com/ClickHouse/ClickHouse/issues/71647). [#71679](https://github.com/ClickHouse/ClickHouse/pull/71679) ([Amos Bird](https://github.com/amosbird)).
|
||||
* Backported in [#71737](https://github.com/ClickHouse/ClickHouse/issues/71737): Don't transform constant set in predicates over partition columns in case of NOT IN operator. [#71695](https://github.com/ClickHouse/ClickHouse/pull/71695) ([Eduard Karacharov](https://github.com/korowa)).
|
||||
* Backported in [#72002](https://github.com/ClickHouse/ClickHouse/issues/72002): Fix a crash in clickhouse-client syntax highlighting. Closes [#71864](https://github.com/ClickHouse/ClickHouse/issues/71864). [#71949](https://github.com/ClickHouse/ClickHouse/pull/71949) ([Nikolay Degterinsky](https://github.com/evillique)).
|
||||
|
||||
#### Build/Testing/Packaging Improvement
|
||||
* Backported in [#71690](https://github.com/ClickHouse/ClickHouse/issues/71690): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
||||
#### NOT FOR CHANGELOG / INSIGNIFICANT
|
||||
|
||||
* Backported in [#71801](https://github.com/ClickHouse/ClickHouse/issues/71801): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
* Backported in [#71836](https://github.com/ClickHouse/ClickHouse/issues/71836): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
|
||||
|
88
docs/changelogs/v24.9.3.128-stable.md
Normal file
88
docs/changelogs/v24.9.3.128-stable.md
Normal file
File diff suppressed because one or more lines are too long
@ -258,7 +258,7 @@ CREATE TABLE table_with_asterisk (name String, value UInt32)
|
||||
|
||||
- [s3_truncate_on_insert](/docs/en/operations/settings/settings.md#s3_truncate_on_insert) - allows to truncate file before insert into it. Disabled by default.
|
||||
- [s3_create_new_file_on_insert](/docs/en/operations/settings/settings.md#s3_create_new_file_on_insert) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [s3_skip_empty_files](/docs/en/operations/settings/settings.md#s3_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
|
||||
- [s3_skip_empty_files](/docs/en/operations/settings/settings.md#s3_skip_empty_files) - allows to skip empty files while reading. Enabled by default.
|
||||
|
||||
## S3-related Settings {#settings}
|
||||
|
||||
|
@ -122,7 +122,7 @@ Default value: `0`.
|
||||
|
||||
### s3queue_polling_min_timeout_ms {#polling_min_timeout_ms}
|
||||
|
||||
Minimal timeout before next polling (in milliseconds).
|
||||
Specifies the minimum time, in milliseconds, that ClickHouse waits before making the next polling attempt.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -132,7 +132,7 @@ Default value: `1000`.
|
||||
|
||||
### s3queue_polling_max_timeout_ms {#polling_max_timeout_ms}
|
||||
|
||||
Maximum timeout before next polling (in milliseconds).
|
||||
Defines the maximum time, in milliseconds, that ClickHouse waits before initiating the next polling attempt.
|
||||
|
||||
Possible values:
|
||||
|
||||
@ -142,7 +142,7 @@ Default value: `10000`.
|
||||
|
||||
### s3queue_polling_backoff_ms {#polling_backoff_ms}
|
||||
|
||||
Polling backoff (in milliseconds).
|
||||
Determines the additional wait time added to the previous polling interval when no new files are found. The next poll occurs after the sum of the previous interval and this backoff value, or the maximum interval, whichever is lower.
|
||||
|
||||
Possible values:
|
||||
|
||||
|
@ -10,6 +10,11 @@ The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-fa
|
||||
|
||||
You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views.
|
||||
|
||||
You can see an example of how to use the AggregatingMergeTree and Aggregate functions in the below video:
|
||||
<div class='vimeo-container'>
|
||||
<iframe width="1030" height="579" src="https://www.youtube.com/embed/pryhI4F_zqQ" title="Aggregation States in ClickHouse" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
|
||||
</div>
|
||||
|
||||
The engine processes all columns with the following types:
|
||||
|
||||
## [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md)
|
||||
|
@ -684,8 +684,7 @@ If you perform the `SELECT` query between merges, you may get expired data. To a
|
||||
|
||||
**See Also**
|
||||
|
||||
- [ttl_only_drop_parts](/docs/en/operations/settings/settings.md/#ttl_only_drop_parts) setting
|
||||
|
||||
- [ttl_only_drop_parts](/docs/en/operations/settings/merge-tree-settings#ttl_only_drop_parts) setting
|
||||
|
||||
## Disk types
|
||||
|
||||
|
@ -16,7 +16,7 @@ You have four options for getting up and running with ClickHouse:
|
||||
- **[ClickHouse Cloud](https://clickhouse.com/cloud/):** The official ClickHouse as a service, - built by, maintained and supported by the creators of ClickHouse
|
||||
- **[Quick Install](#quick-install):** an easy-to-download binary for testing and developing with ClickHouse
|
||||
- **[Production Deployments](#available-installation-options):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, modern ARM (ARMv8.2-A up), or PowerPC64LE CPU architecture
|
||||
- **[Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/):** use the official Docker image in Docker Hub
|
||||
- **[Docker Image](https://hub.docker.com/_/clickhouse):** use the official Docker image in Docker Hub
|
||||
|
||||
## ClickHouse Cloud
|
||||
|
||||
|
@ -1643,6 +1643,7 @@ You can specify the log format that will be outputted in the console log. Curren
|
||||
|
||||
```json
|
||||
{
|
||||
"date_time_utc": "2024-11-06T09:06:09Z",
|
||||
"date_time": "1650918987.180175",
|
||||
"thread_name": "#1",
|
||||
"thread_id": "254545",
|
||||
@ -3285,3 +3286,17 @@ Use the legacy MongoDB integration implementation. Deprecated.
|
||||
Type: Bool
|
||||
|
||||
Default value: `true`.
|
||||
|
||||
## allowed_feature_tier
|
||||
|
||||
Controls if the user can change settings related to the different feature tiers.
|
||||
0 - Changes to any setting are allowed (experimental, beta, production).
|
||||
1 - Only changes to beta and production feature settings are allowed. Changes to experimental settings are rejected.
|
||||
2 - Only changes to production settings are allowed. Changes to experimental or beta settings are rejected.
|
||||
|
||||
This is equivalent to setting a readonly constraint on all EXPERIMENTAL / BETA features.
|
||||
```
|
||||
|
||||
Type: UInt32
|
||||
|
||||
Default value: `0` (all settings can be changed).
|
||||
|
@ -78,6 +78,16 @@ If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled.
|
||||
|
||||
Default value: `10 * 1024 * 1024 * 1024` bytes.
|
||||
|
||||
## ttl_only_drop_parts
|
||||
|
||||
Controls whether data parts are fully dropped in MergeTree tables when all rows in that part have expired according to their `TTL` settings.
|
||||
|
||||
When `ttl_only_drop_parts` is disabled (by default), only the rows that have expired based on their TTL settings are removed.
|
||||
|
||||
When `ttl_only_drop_parts` is enabled, the entire part is dropped if all rows in that part have expired according to their `TTL` settings.
|
||||
|
||||
Default value: 0.
|
||||
|
||||
## merge_with_ttl_timeout
|
||||
|
||||
Minimum delay in seconds before repeating a merge with delete TTL.
|
||||
@ -1095,3 +1105,13 @@ Possible values:
|
||||
Default value: 0.0
|
||||
|
||||
Note that if both `min_free_disk_ratio_to_perform_insert` and `min_free_disk_bytes_to_perform_insert` are specified, ClickHouse will count on the value that will allow to perform inserts on a bigger amount of free memory.
|
||||
|
||||
## cache_populated_by_fetch
|
||||
|
||||
A Cloud only setting.
|
||||
|
||||
When `cache_populated_by_fetch` is disabled (the default setting), new data parts are loaded into the cache only when a query is run that requires those parts.
|
||||
|
||||
If enabled, `cache_populated_by_fetch` will instead cause all nodes to load new data parts from storage into their cache without requiring a query to trigger such an action.
|
||||
|
||||
Default value: 0.
|
@ -211,7 +211,7 @@ Number of threads in the server of the replicas communication protocol (without
|
||||
|
||||
The difference in time the thread for calculation of the asynchronous metrics was scheduled to wake up and the time it was in fact, woken up. A proxy-indicator of overall system latency and responsiveness.
|
||||
|
||||
### LoadAverage_*N*
|
||||
### LoadAverage*N*
|
||||
|
||||
The whole system load, averaged with exponential smoothing over 1 minute. The load represents the number of threads across all the processes (the scheduling entities of the OS kernel), that are currently running by CPU or waiting for IO, or ready to run but not being scheduled at this point of time. This number includes all the processes, not only clickhouse-server. The number can be greater than the number of CPU cores, if the system is overloaded, and many processes are ready to run but waiting for CPU or IO.
|
||||
|
||||
|
@ -75,7 +75,7 @@ FROM t_null_big
|
||||
└────────────────────┴─────────────────────┘
|
||||
```
|
||||
|
||||
Also you can use [Tuple](/docs/en/sql-reference/data-types/tuple.md) to work around NULL skipping behavior. The a `Tuple` that contains only a `NULL` value is not `NULL`, so the aggregate functions won't skip that row because of that `NULL` value.
|
||||
Also you can use [Tuple](/docs/en/sql-reference/data-types/tuple.md) to work around NULL skipping behavior. A `Tuple` that contains only a `NULL` value is not `NULL`, so the aggregate functions won't skip that row because of that `NULL` value.
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
@ -110,7 +110,7 @@ GROUP BY v
|
||||
└──────┴─────────┴──────────┘
|
||||
```
|
||||
|
||||
And here is an example of of first_value with `RESPECT NULLS` where we can see that NULL inputs are respected and it will return the first value read, whether it's NULL or not:
|
||||
And here is an example of first_value with `RESPECT NULLS` where we can see that NULL inputs are respected and it will return the first value read, whether it's NULL or not:
|
||||
|
||||
```sql
|
||||
SELECT
|
||||
|
@ -5,7 +5,15 @@ sidebar_position: 102
|
||||
|
||||
# any
|
||||
|
||||
Selects the first encountered value of a column, ignoring any `NULL` values.
|
||||
Selects the first encountered value of a column.
|
||||
|
||||
:::warning
|
||||
As a query can be executed in arbitrary order, the result of this function is non-deterministic.
|
||||
If you need an arbitrary but deterministic result, use functions [`min`](../reference/min.md) or [`max`](../reference/max.md).
|
||||
:::
|
||||
|
||||
By default, the function never returns NULL, i.e. ignores NULL values in the input column.
|
||||
However, if the function is used with the `RESPECT NULLS` modifier, it returns the first value reads no matter if NULL or not.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -13,46 +21,51 @@ Selects the first encountered value of a column, ignoring any `NULL` values.
|
||||
any(column) [RESPECT NULLS]
|
||||
```
|
||||
|
||||
Aliases: `any_value`, [`first_value`](../reference/first_value.md).
|
||||
Aliases `any(column)` (without `RESPECT NULLS`)
|
||||
- `any_value`
|
||||
- [`first_value`](../reference/first_value.md).
|
||||
|
||||
Alias for `any(column) RESPECT NULLS`
|
||||
- `anyRespectNulls`, `any_respect_nulls`
|
||||
- `firstValueRespectNulls`, `first_value_respect_nulls`
|
||||
- `anyValueRespectNulls`, `any_value_respect_nulls`
|
||||
|
||||
**Parameters**
|
||||
- `column`: The column name.
|
||||
|
||||
**Returned value**
|
||||
|
||||
:::note
|
||||
Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the first value passed, regardless of whether it is `NULL` or not.
|
||||
:::
|
||||
The first value encountered.
|
||||
|
||||
:::note
|
||||
The return type of the function is the same as the input, except for LowCardinality which is discarded. This means that given no rows as input it will return the default value of that type (0 for integers, or Null for a Nullable() column). You might use the `-OrNull` [combinator](../../../sql-reference/aggregate-functions/combinators.md) ) to modify this behaviour.
|
||||
:::
|
||||
|
||||
:::warning
|
||||
The query can be executed in any order and even in a different order each time, so the result of this function is indeterminate.
|
||||
To get a determinate result, you can use the [`min`](../reference/min.md) or [`max`](../reference/max.md) function instead of `any`.
|
||||
The return type of the function is the same as the input, except for LowCardinality which is discarded.
|
||||
This means that given no rows as input it will return the default value of that type (0 for integers, or Null for a Nullable() column).
|
||||
You might use the `-OrNull` [combinator](../../../sql-reference/aggregate-functions/combinators.md) ) to modify this behaviour.
|
||||
:::
|
||||
|
||||
**Implementation details**
|
||||
|
||||
In some cases, you can rely on the order of execution. This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`.
|
||||
In some cases, you can rely on the order of execution.
|
||||
This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`.
|
||||
|
||||
When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
|
||||
When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions.
|
||||
In other words, each column selected from the table must be used either in keys or inside aggregate functions.
|
||||
To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
|
||||
|
||||
**Example**
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
CREATE TABLE any_nulls (city Nullable(String)) ENGINE=Log;
|
||||
CREATE TABLE tab (city Nullable(String)) ENGINE=Memory;
|
||||
|
||||
INSERT INTO any_nulls (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL);
|
||||
INSERT INTO tab (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL);
|
||||
|
||||
SELECT any(city) FROM any_nulls;
|
||||
SELECT any(city), anyRespectNulls(city) FROM tab;
|
||||
```
|
||||
|
||||
```response
|
||||
┌─any(city)─┐
|
||||
│ Amsterdam │
|
||||
└───────────┘
|
||||
┌─any(city)─┬─anyRespectNulls(city)─┐
|
||||
│ Amsterdam │ ᴺᵁᴸᴸ │
|
||||
└───────────┴───────────────────────┘
|
||||
```
|
||||
|
@ -5,7 +5,15 @@ sidebar_position: 105
|
||||
|
||||
# anyLast
|
||||
|
||||
Selects the last value encountered, ignoring any `NULL` values by default. The result is just as indeterminate as for the [any](../../../sql-reference/aggregate-functions/reference/any.md) function.
|
||||
Selects the last encountered value of a column.
|
||||
|
||||
:::warning
|
||||
As a query can be executed in arbitrary order, the result of this function is non-deterministic.
|
||||
If you need an arbitrary but deterministic result, use functions [`min`](../reference/min.md) or [`max`](../reference/max.md).
|
||||
:::
|
||||
|
||||
By default, the function never returns NULL, i.e. ignores NULL values in the input column.
|
||||
However, if the function is used with the `RESPECT NULLS` modifier, it returns the first value reads no matter if NULL or not.
|
||||
|
||||
**Syntax**
|
||||
|
||||
@ -13,13 +21,16 @@ Selects the last value encountered, ignoring any `NULL` values by default. The r
|
||||
anyLast(column) [RESPECT NULLS]
|
||||
```
|
||||
|
||||
Alias `anyLast(column)` (without `RESPECT NULLS`)
|
||||
- [`last_value`](../reference/last_value.md).
|
||||
|
||||
Aliases for `anyLast(column) RESPECT NULLS`
|
||||
- `anyLastRespectNulls`, `anyLast_respect_nulls`
|
||||
- `lastValueRespectNulls`, `last_value_respect_nulls`
|
||||
|
||||
**Parameters**
|
||||
- `column`: The column name.
|
||||
|
||||
:::note
|
||||
Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the last value passed, regardless of whether it is `NULL` or not.
|
||||
:::
|
||||
|
||||
**Returned value**
|
||||
|
||||
- The last value encountered.
|
||||
@ -29,15 +40,15 @@ Supports the `RESPECT NULLS` modifier after the function name. Using this modifi
|
||||
Query:
|
||||
|
||||
```sql
|
||||
CREATE TABLE any_last_nulls (city Nullable(String)) ENGINE=Log;
|
||||
CREATE TABLE tab (city Nullable(String)) ENGINE=Memory;
|
||||
|
||||
INSERT INTO any_last_nulls (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL);
|
||||
INSERT INTO tab (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL);
|
||||
|
||||
SELECT anyLast(city) FROM any_last_nulls;
|
||||
SELECT anyLast(city), anyLastRespectNulls(city) FROM tab;
|
||||
```
|
||||
|
||||
```response
|
||||
┌─anyLast(city)─┐
|
||||
│ Valencia │
|
||||
└───────────────┘
|
||||
┌─anyLast(city)─┬─anyLastRespectNulls(city)─┐
|
||||
│ Valencia │ ᴺᵁᴸᴸ │
|
||||
└───────────────┴───────────────────────────┘
|
||||
```
|
||||
|
@ -4489,9 +4489,9 @@ Using replacement fields, you can define a pattern for the resulting string.
|
||||
| k | clockhour of day (1~24) | number | 24 |
|
||||
| m | minute of hour | number | 30 |
|
||||
| s | second of minute | number | 55 |
|
||||
| S | fraction of second (not supported yet) | number | 978 |
|
||||
| z | time zone (short name not supported yet) | text | Pacific Standard Time; PST |
|
||||
| Z | time zone offset/id (not supported yet) | zone | -0800; -08:00; America/Los_Angeles |
|
||||
| S | fraction of second | number | 978 |
|
||||
| z | time zone | text | Eastern Standard Time; EST |
|
||||
| Z | time zone offset | zone | -0800; -0812 |
|
||||
| ' | escape for text | delimiter | |
|
||||
| '' | single quote | literal | ' |
|
||||
|
||||
|
@ -6791,7 +6791,7 @@ parseDateTime(str[, format[, timezone]])
|
||||
|
||||
**Returned value(s)**
|
||||
|
||||
Returns DateTime values parsed from input string according to a MySQL style format string.
|
||||
Return a [DateTime](../data-types/datetime.md) value parsed from the input string according to a MySQL-style format string.
|
||||
|
||||
**Supported format specifiers**
|
||||
|
||||
@ -6840,7 +6840,7 @@ parseDateTimeInJodaSyntax(str[, format[, timezone]])
|
||||
|
||||
**Returned value(s)**
|
||||
|
||||
Returns DateTime values parsed from input string according to a Joda style format.
|
||||
Return a [DateTime](../data-types/datetime.md) value parsed from the input string according to a Joda-style format string.
|
||||
|
||||
**Supported format specifiers**
|
||||
|
||||
@ -6867,9 +6867,55 @@ Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that
|
||||
|
||||
Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that it returns `NULL` when it encounters a date format that cannot be processed.
|
||||
|
||||
## parseDateTime64
|
||||
|
||||
Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datetime64.md) according to a [MySQL format string](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-format).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
parseDateTime64(str[, format[, timezone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `str` — The String to be parsed.
|
||||
- `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s.%f` if not specified.
|
||||
- `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional.
|
||||
|
||||
**Returned value(s)**
|
||||
|
||||
Return a [DateTime64](../data-types/datetime64.md) value parsed from the input string according to a MySQL-style format string.
|
||||
The precision of the returned value is 6.
|
||||
|
||||
## parseDateTime64OrZero
|
||||
|
||||
Same as for [parseDateTime64](#parsedatetime64) except that it returns zero date when it encounters a date format that cannot be processed.
|
||||
|
||||
## parseDateTime64OrNull
|
||||
|
||||
Same as for [parseDateTime64](#parsedatetime64) except that it returns `NULL` when it encounters a date format that cannot be processed.
|
||||
|
||||
## parseDateTime64InJodaSyntax
|
||||
|
||||
Similar to [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax). Differently, it returns a value of type [DateTime64](../data-types/datetime64.md).
|
||||
Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datetime64.md) according to a [Joda format string](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html).
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
parseDateTime64InJodaSyntax(str[, format[, timezone]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `str` — The String to be parsed.
|
||||
- `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified.
|
||||
- `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional.
|
||||
|
||||
**Returned value(s)**
|
||||
|
||||
Return a [DateTime64](../data-types/datetime64.md) value parsed from the input string according to a Joda-style format string.
|
||||
The precision of the returned value equal to the number of `S` placeholders in the format string (but at most 6).
|
||||
|
||||
## parseDateTime64InJodaSyntaxOrZero
|
||||
|
||||
|
46
docs/en/sql-reference/statements/check-grant.md
Normal file
46
docs/en/sql-reference/statements/check-grant.md
Normal file
@ -0,0 +1,46 @@
|
||||
---
|
||||
slug: /en/sql-reference/statements/check-grant
|
||||
sidebar_position: 56
|
||||
sidebar_label: CHECK GRANT
|
||||
title: "CHECK GRANT Statement"
|
||||
---
|
||||
|
||||
The `CHECK GRANT` query is used to check whether the current user/role has been granted a specific privilege.
|
||||
|
||||
## Syntax
|
||||
|
||||
The basic syntax of the query is as follows:
|
||||
|
||||
```sql
|
||||
CHECK GRANT privilege[(column_name [,...])] [,...] ON {db.table[*]|db[*].*|*.*|table[*]|*}
|
||||
```
|
||||
|
||||
- `privilege` — Type of privilege.
|
||||
|
||||
## Examples
|
||||
|
||||
If the user used to be granted the privilege, the response`check_grant` will be `1`. Otherwise, the response `check_grant` will be `0`.
|
||||
|
||||
If `table_1.col1` exists and current user is granted by privilege `SELECT`/`SELECT(con)` or role(with privilege), the response is `1`.
|
||||
```sql
|
||||
CHECK GRANT SELECT(col1) ON table_1;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─result─┐
|
||||
│ 1 │
|
||||
└────────┘
|
||||
```
|
||||
If `table_2.col2` doesn't exists, or current user is not granted by privilege `SELECT`/`SELECT(con)` or role(with privilege), the response is `0`.
|
||||
```sql
|
||||
CHECK GRANT SELECT(col2) ON table_2;
|
||||
```
|
||||
|
||||
```text
|
||||
┌─result─┐
|
||||
│ 0 │
|
||||
└────────┘
|
||||
```
|
||||
|
||||
## Wildcard
|
||||
Specifying privileges you can use asterisk (`*`) instead of a table or a database name. Please check [WILDCARD GRANTS](../../sql-reference/statements/grant.md#wildcard-grants) for wildcard rules.
|
@ -161,6 +161,8 @@ Settings:
|
||||
- `actions` — Prints detailed information about step actions. Default: 0.
|
||||
- `json` — Prints query plan steps as a row in [JSON](../../interfaces/formats.md#json) format. Default: 0. It is recommended to use [TSVRaw](../../interfaces/formats.md#tabseparatedraw) format to avoid unnecessary escaping.
|
||||
|
||||
When `json=1` step names will contain an additional suffix with unique step identifier.
|
||||
|
||||
Example:
|
||||
|
||||
```sql
|
||||
@ -194,30 +196,25 @@ EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw;
|
||||
{
|
||||
"Plan": {
|
||||
"Node Type": "Union",
|
||||
"Node Id": "Union_10",
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "Expression",
|
||||
"Node Id": "Expression_13",
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "SettingQuotaAndLimits",
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "ReadFromStorage"
|
||||
}
|
||||
]
|
||||
"Node Type": "ReadFromStorage",
|
||||
"Node Id": "ReadFromStorage_0"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"Node Type": "Expression",
|
||||
"Node Id": "Expression_16",
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "SettingQuotaAndLimits",
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "ReadFromStorage"
|
||||
}
|
||||
]
|
||||
"Node Type": "ReadFromStorage",
|
||||
"Node Id": "ReadFromStorage_4"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -249,6 +246,7 @@ EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy;
|
||||
{
|
||||
"Plan": {
|
||||
"Node Type": "Expression",
|
||||
"Node Id": "Expression_5",
|
||||
"Header": [
|
||||
{
|
||||
"Name": "1",
|
||||
@ -259,18 +257,10 @@ EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy;
|
||||
"Type": "UInt16"
|
||||
}
|
||||
],
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "SettingQuotaAndLimits",
|
||||
"Header": [
|
||||
{
|
||||
"Name": "dummy",
|
||||
"Type": "UInt8"
|
||||
}
|
||||
],
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "ReadFromStorage",
|
||||
"Node Id": "ReadFromStorage_0",
|
||||
"Header": [
|
||||
{
|
||||
"Name": "dummy",
|
||||
@ -280,8 +270,6 @@ EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy;
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
```
|
||||
@ -351,17 +339,31 @@ EXPLAIN json = 1, actions = 1, description = 0 SELECT 1 FORMAT TSVRaw;
|
||||
{
|
||||
"Plan": {
|
||||
"Node Type": "Expression",
|
||||
"Node Id": "Expression_5",
|
||||
"Expression": {
|
||||
"Inputs": [],
|
||||
"Inputs": [
|
||||
{
|
||||
"Name": "dummy",
|
||||
"Type": "UInt8"
|
||||
}
|
||||
],
|
||||
"Actions": [
|
||||
{
|
||||
"Node Type": "Column",
|
||||
"Node Type": "INPUT",
|
||||
"Result Type": "UInt8",
|
||||
"Result Type": "Column",
|
||||
"Result Name": "dummy",
|
||||
"Arguments": [0],
|
||||
"Removed Arguments": [0],
|
||||
"Result": 0
|
||||
},
|
||||
{
|
||||
"Node Type": "COLUMN",
|
||||
"Result Type": "UInt8",
|
||||
"Result Name": "1",
|
||||
"Column": "Const(UInt8)",
|
||||
"Arguments": [],
|
||||
"Removed Arguments": [],
|
||||
"Result": 0
|
||||
"Result": 1
|
||||
}
|
||||
],
|
||||
"Outputs": [
|
||||
@ -370,17 +372,12 @@ EXPLAIN json = 1, actions = 1, description = 0 SELECT 1 FORMAT TSVRaw;
|
||||
"Type": "UInt8"
|
||||
}
|
||||
],
|
||||
"Positions": [0],
|
||||
"Project Input": true
|
||||
"Positions": [1]
|
||||
},
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "SettingQuotaAndLimits",
|
||||
"Plans": [
|
||||
{
|
||||
"Node Type": "ReadFromStorage"
|
||||
}
|
||||
]
|
||||
"Node Type": "ReadFromStorage",
|
||||
"Node Id": "ReadFromStorage_0"
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -396,6 +393,8 @@ Settings:
|
||||
- `graph` — Prints a graph described in the [DOT](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) graph description language. Default: 0.
|
||||
- `compact` — Prints graph in compact mode if `graph` setting is enabled. Default: 1.
|
||||
|
||||
When `compact=0` and `graph=1` processor names will contain an additional suffix with unique processor identifier.
|
||||
|
||||
Example:
|
||||
|
||||
```sql
|
||||
|
@ -5,9 +5,14 @@ sidebar_label: EXCEPT
|
||||
|
||||
# EXCEPT Clause
|
||||
|
||||
The `EXCEPT` clause returns only those rows that result from the first query without the second. The queries must match the number of columns, order, and type. The result of `EXCEPT` can contain duplicate rows.
|
||||
The `EXCEPT` clause returns only those rows that result from the first query without the second.
|
||||
|
||||
Multiple `EXCEPT` statements are executed left to right if parenthesis are not specified. The `EXCEPT` operator has the same priority as the `UNION` clause and lower priority than the `INTERSECT` clause.
|
||||
- Both queries must have the same number of columns in the same order and data type.
|
||||
- The result of `EXCEPT` can contain duplicate rows. Use `EXCEPT DISTINCT` if this is not desirable.
|
||||
- Multiple `EXCEPT` statements are executed from left to right if parentheses are not specified.
|
||||
- The `EXCEPT` operator has the same priority as the `UNION` clause and lower priority than the `INTERSECT` clause.
|
||||
|
||||
## Syntax
|
||||
|
||||
``` sql
|
||||
SELECT column1 [, column2 ]
|
||||
@ -19,18 +24,33 @@ EXCEPT
|
||||
SELECT column1 [, column2 ]
|
||||
FROM table2
|
||||
[WHERE condition]
|
||||
|
||||
```
|
||||
The condition could be any expression based on your requirements.
|
||||
|
||||
Additionally, `EXCEPT()` can be used to exclude columns from a result in the same table, as is possible with BigQuery (Google Cloud), using the following syntax:
|
||||
|
||||
```sql
|
||||
SELECT column1 [, column2 ] EXCEPT (column3 [, column4])
|
||||
FROM table1
|
||||
[WHERE condition]
|
||||
```
|
||||
|
||||
## Examples
|
||||
|
||||
The examples in this section demonstrate usage of the `EXCEPT` clause.
|
||||
|
||||
### Filtering Numbers Using the `EXCEPT` Clause
|
||||
|
||||
Here is a simple example that returns the numbers 1 to 10 that are _not_ a part of the numbers 3 to 8:
|
||||
|
||||
Query:
|
||||
|
||||
``` sql
|
||||
SELECT number FROM numbers(1,10) EXCEPT SELECT number FROM numbers(3,6);
|
||||
SELECT number
|
||||
FROM numbers(1, 10)
|
||||
EXCEPT
|
||||
SELECT number
|
||||
FROM numbers(3, 6)
|
||||
```
|
||||
|
||||
Result:
|
||||
@ -44,7 +64,53 @@ Result:
|
||||
└────────┘
|
||||
```
|
||||
|
||||
`EXCEPT` and `INTERSECT` can often be used interchangeably with different Boolean logic, and they are both useful if you have two tables that share a common column (or columns). For example, suppose we have a few million rows of historical cryptocurrency data that contains trade prices and volume:
|
||||
### Excluding Specific Columns Using `EXCEPT()`
|
||||
|
||||
`EXCEPT()` can be used to quickly exclude columns from a result. For instance if we want to select all columns from a table, except a few select columns as shown in the example below:
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
SHOW COLUMNS IN system.settings
|
||||
|
||||
SELECT * EXCEPT (default, alias_for, readonly, description)
|
||||
FROM system.settings
|
||||
LIMIT 5
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─field───────┬─type─────────────────────────────────────────────────────────────────────┬─null─┬─key─┬─default─┬─extra─┐
|
||||
1. │ alias_for │ String │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
2. │ changed │ UInt8 │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
3. │ default │ String │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
4. │ description │ String │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
5. │ is_obsolete │ UInt8 │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
6. │ max │ Nullable(String) │ YES │ │ ᴺᵁᴸᴸ │ │
|
||||
7. │ min │ Nullable(String) │ YES │ │ ᴺᵁᴸᴸ │ │
|
||||
8. │ name │ String │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
9. │ readonly │ UInt8 │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
10. │ tier │ Enum8('Production' = 0, 'Obsolete' = 4, 'Experimental' = 8, 'Beta' = 12) │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
11. │ type │ String │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
12. │ value │ String │ NO │ │ ᴺᵁᴸᴸ │ │
|
||||
└─────────────┴──────────────────────────────────────────────────────────────────────────┴──────┴─────┴─────────┴───────┘
|
||||
|
||||
┌─name────────────────────┬─value──────┬─changed─┬─min──┬─max──┬─type────┬─is_obsolete─┬─tier───────┐
|
||||
1. │ dialect │ clickhouse │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dialect │ 0 │ Production │
|
||||
2. │ min_compress_block_size │ 65536 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
|
||||
3. │ max_compress_block_size │ 1048576 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
|
||||
4. │ max_block_size │ 65409 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
|
||||
5. │ max_insert_block_size │ 1048449 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
|
||||
└─────────────────────────┴────────────┴─────────┴──────┴──────┴─────────┴─────────────┴────────────┘
|
||||
```
|
||||
|
||||
### Using `EXCEPT` and `INTERSECT` with Cryptocurrency Data
|
||||
|
||||
`EXCEPT` and `INTERSECT` can often be used interchangeably with different Boolean logic, and they are both useful if you have two tables that share a common column (or columns).
|
||||
For example, suppose we have a few million rows of historical cryptocurrency data that contains trade prices and volume:
|
||||
|
||||
Query:
|
||||
|
||||
```sql
|
||||
CREATE TABLE crypto_prices
|
||||
@ -72,6 +138,8 @@ ORDER BY trade_date DESC
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
Result:
|
||||
|
||||
```response
|
||||
┌─trade_date─┬─crypto_name─┬──────volume─┬────price─┬───market_cap─┬──change_1_day─┐
|
||||
│ 2020-11-02 │ Bitcoin │ 30771456000 │ 13550.49 │ 251119860000 │ -0.013585099 │
|
||||
@ -127,7 +195,7 @@ Result:
|
||||
|
||||
This means of the four cryptocurrencies we own, only Bitcoin has never dropped below $10 (based on the limited data we have here in this example).
|
||||
|
||||
## EXCEPT DISTINCT
|
||||
### Using `EXCEPT DISTINCT`
|
||||
|
||||
Notice in the previous query we had multiple Bitcoin holdings in the result. You can add `DISTINCT` to `EXCEPT` to eliminate duplicate rows from the result:
|
||||
|
||||
@ -146,7 +214,6 @@ Result:
|
||||
└─────────────┘
|
||||
```
|
||||
|
||||
|
||||
**See Also**
|
||||
|
||||
- [UNION](union.md#union-clause)
|
||||
|
@ -49,4 +49,4 @@ LIMIT 2
|
||||
**See Also**
|
||||
|
||||
- [DeltaLake engine](/docs/en/engines/table-engines/integrations/deltalake.md)
|
||||
|
||||
- [DeltaLake cluster table function](/docs/en/sql-reference/table-functions/deltalakeCluster.md)
|
||||
|
30
docs/en/sql-reference/table-functions/deltalakeCluster.md
Normal file
30
docs/en/sql-reference/table-functions/deltalakeCluster.md
Normal file
@ -0,0 +1,30 @@
|
||||
---
|
||||
slug: /en/sql-reference/table-functions/deltalakeCluster
|
||||
sidebar_position: 46
|
||||
sidebar_label: deltaLakeCluster
|
||||
title: "deltaLakeCluster Table Function"
|
||||
---
|
||||
This is an extension to the [deltaLake](/docs/en/sql-reference/table-functions/deltalake.md) table function.
|
||||
|
||||
Allows processing files from [Delta Lake](https://github.com/delta-io/delta) tables in Amazon S3 in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
deltaLakeCluster(cluster_name, url [,aws_access_key_id, aws_secret_access_key] [,format] [,structure] [,compression])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers.
|
||||
|
||||
- Description of all other arguments coincides with description of arguments in equivalent [deltaLake](/docs/en/sql-reference/table-functions/deltalake.md) table function.
|
||||
|
||||
**Returned value**
|
||||
|
||||
A table with the specified structure for reading data from cluster in the specified Delta Lake table in S3.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [deltaLake engine](/docs/en/engines/table-engines/integrations/deltalake.md)
|
||||
- [deltaLake table function](/docs/en/sql-reference/table-functions/deltalake.md)
|
@ -29,4 +29,4 @@ A table with the specified structure for reading data in the specified Hudi tabl
|
||||
**See Also**
|
||||
|
||||
- [Hudi engine](/docs/en/engines/table-engines/integrations/hudi.md)
|
||||
|
||||
- [Hudi cluster table function](/docs/en/sql-reference/table-functions/hudiCluster.md)
|
||||
|
30
docs/en/sql-reference/table-functions/hudiCluster.md
Normal file
30
docs/en/sql-reference/table-functions/hudiCluster.md
Normal file
@ -0,0 +1,30 @@
|
||||
---
|
||||
slug: /en/sql-reference/table-functions/hudiCluster
|
||||
sidebar_position: 86
|
||||
sidebar_label: hudiCluster
|
||||
title: "hudiCluster Table Function"
|
||||
---
|
||||
This is an extension to the [hudi](/docs/en/sql-reference/table-functions/hudi.md) table function.
|
||||
|
||||
Allows processing files from Apache [Hudi](https://hudi.apache.org/) tables in Amazon S3 in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
hudiCluster(cluster_name, url [,aws_access_key_id, aws_secret_access_key] [,format] [,structure] [,compression])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers.
|
||||
|
||||
- Description of all other arguments coincides with description of arguments in equivalent [hudi](/docs/en/sql-reference/table-functions/hudi.md) table function.
|
||||
|
||||
**Returned value**
|
||||
|
||||
A table with the specified structure for reading data from cluster in the specified Hudi table in S3.
|
||||
|
||||
**See Also**
|
||||
|
||||
- [Hudi engine](/docs/en/engines/table-engines/integrations/hudi.md)
|
||||
- [Hudi table function](/docs/en/sql-reference/table-functions/hudi.md)
|
@ -72,3 +72,4 @@ Table function `iceberg` is an alias to `icebergS3` now.
|
||||
**See Also**
|
||||
|
||||
- [Iceberg engine](/docs/en/engines/table-engines/integrations/iceberg.md)
|
||||
- [Iceberg cluster table function](/docs/en/sql-reference/table-functions/icebergCluster.md)
|
||||
|
43
docs/en/sql-reference/table-functions/icebergCluster.md
Normal file
43
docs/en/sql-reference/table-functions/icebergCluster.md
Normal file
@ -0,0 +1,43 @@
|
||||
---
|
||||
slug: /en/sql-reference/table-functions/icebergCluster
|
||||
sidebar_position: 91
|
||||
sidebar_label: icebergCluster
|
||||
title: "icebergCluster Table Function"
|
||||
---
|
||||
This is an extension to the [iceberg](/docs/en/sql-reference/table-functions/iceberg.md) table function.
|
||||
|
||||
Allows processing files from Apache [Iceberg](https://iceberg.apache.org/) in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished.
|
||||
|
||||
**Syntax**
|
||||
|
||||
``` sql
|
||||
icebergS3Cluster(cluster_name, url [, NOSIGN | access_key_id, secret_access_key, [session_token]] [,format] [,compression_method])
|
||||
icebergS3Cluster(cluster_name, named_collection[, option=value [,..]])
|
||||
|
||||
icebergAzureCluster(cluster_name, connection_string|storage_account_url, container_name, blobpath, [,account_name], [,account_key] [,format] [,compression_method])
|
||||
icebergAzureCluster(cluster_name, named_collection[, option=value [,..]])
|
||||
|
||||
icebergHDFSCluster(cluster_name, path_to_table, [,format] [,compression_method])
|
||||
icebergHDFSCluster(cluster_name, named_collection[, option=value [,..]])
|
||||
```
|
||||
|
||||
**Arguments**
|
||||
|
||||
- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers.
|
||||
|
||||
- Description of all other arguments coincides with description of arguments in equivalent [iceberg](/docs/en/sql-reference/table-functions/iceberg.md) table function.
|
||||
|
||||
**Returned value**
|
||||
|
||||
A table with the specified structure for reading data from cluster in the specified Iceberg table.
|
||||
|
||||
**Examples**
|
||||
|
||||
```sql
|
||||
SELECT * FROM icebergS3Cluster('cluster_simple', 'http://test.s3.amazonaws.com/clickhouse-bucket/test_table', 'test', 'test')
|
||||
```
|
||||
|
||||
**See Also**
|
||||
|
||||
- [Iceberg engine](/docs/en/engines/table-engines/integrations/iceberg.md)
|
||||
- [Iceberg table function](/docs/en/sql-reference/table-functions/iceberg.md)
|
@ -317,7 +317,7 @@ SELECT * from s3('s3://data/path/date=*/country=*/code=*/*.parquet') where _date
|
||||
|
||||
- [s3_truncate_on_insert](/docs/en/operations/settings/settings.md#s3_truncate_on_insert) - allows to truncate file before insert into it. Disabled by default.
|
||||
- [s3_create_new_file_on_insert](/docs/en/operations/settings/settings.md#s3_create_new_file_on_insert) - allows to create a new file on each insert if format has suffix. Disabled by default.
|
||||
- [s3_skip_empty_files](/docs/en/operations/settings/settings.md#s3_skip_empty_files) - allows to skip empty files while reading. Disabled by default.
|
||||
- [s3_skip_empty_files](/docs/en/operations/settings/settings.md#s3_skip_empty_files) - allows to skip empty files while reading. Enabled by default.
|
||||
|
||||
**See Also**
|
||||
|
||||
|
@ -15,7 +15,7 @@ first_value (column_name) [[RESPECT NULLS] | [IGNORE NULLS]]
|
||||
OVER ([[PARTITION BY grouping_column] [ORDER BY sorting_column]
|
||||
[ROWS or RANGE expression_to_bound_rows_withing_the_group]] | [window_name])
|
||||
FROM table_name
|
||||
WINDOW window_name as ([[PARTITION BY grouping_column] [ORDER BY sorting_column])
|
||||
WINDOW window_name as ([PARTITION BY grouping_column] [ORDER BY sorting_column])
|
||||
```
|
||||
|
||||
Alias: `any`.
|
||||
@ -23,6 +23,8 @@ Alias: `any`.
|
||||
:::note
|
||||
Using the optional modifier `RESPECT NULLS` after `first_value(column_name)` will ensure that `NULL` arguments are not skipped.
|
||||
See [NULL processing](../aggregate-functions/index.md/#null-processing) for more information.
|
||||
|
||||
Alias: `firstValueRespectNulls`
|
||||
:::
|
||||
|
||||
For more detail on window function syntax see: [Window Functions - Syntax](./index.md/#syntax).
|
||||
@ -48,7 +50,7 @@ CREATE TABLE salaries
|
||||
)
|
||||
Engine = Memory;
|
||||
|
||||
INSERT INTO salaries FORMAT Values
|
||||
INSERT INTO salaries FORMAT VALUES
|
||||
('Port Elizabeth Barbarians', 'Gary Chen', 196000, 'F'),
|
||||
('New Coreystad Archdukes', 'Charles Juarez', 190000, 'F'),
|
||||
('Port Elizabeth Barbarians', 'Michael Stanley', 100000, 'D'),
|
||||
|
@ -23,6 +23,8 @@ Alias: `anyLast`.
|
||||
:::note
|
||||
Using the optional modifier `RESPECT NULLS` after `first_value(column_name)` will ensure that `NULL` arguments are not skipped.
|
||||
See [NULL processing](../aggregate-functions/index.md/#null-processing) for more information.
|
||||
|
||||
Alias: `lastValueRespectNulls`
|
||||
:::
|
||||
|
||||
For more detail on window function syntax see: [Window Functions - Syntax](./index.md/#syntax).
|
||||
@ -33,7 +35,7 @@ For more detail on window function syntax see: [Window Functions - Syntax](./ind
|
||||
|
||||
**Example**
|
||||
|
||||
In this example the `last_value` function is used to find the highest paid footballer from a fictional dataset of salaries of Premier League football players.
|
||||
In this example the `last_value` function is used to find the lowest paid footballer from a fictional dataset of salaries of Premier League football players.
|
||||
|
||||
Query:
|
||||
|
||||
@ -48,7 +50,7 @@ CREATE TABLE salaries
|
||||
)
|
||||
Engine = Memory;
|
||||
|
||||
INSERT INTO salaries FORMAT Values
|
||||
INSERT INTO salaries FORMAT VALUES
|
||||
('Port Elizabeth Barbarians', 'Gary Chen', 196000, 'F'),
|
||||
('New Coreystad Archdukes', 'Charles Juarez', 190000, 'F'),
|
||||
('Port Elizabeth Barbarians', 'Michael Stanley', 100000, 'D'),
|
||||
|
8
docs/ja/_placeholders/api/_invitations-api-reference.md
Normal file
8
docs/ja/_placeholders/api/_invitations-api-reference.md
Normal file
@ -0,0 +1,8 @@
|
||||
---
|
||||
sidebar_label: 招待
|
||||
title: 招待
|
||||
---
|
||||
|
||||
## すべての招待を一覧表示
|
||||
|
||||
このファイルは、ビルドプロセス中に `clickhouseapi.js` によって生成されます。内容を変更する必要がある場合は、`clickhouseapi.js` を編集してください。
|
9
docs/ja/_placeholders/api/_keys-api-reference.md
Normal file
9
docs/ja/_placeholders/api/_keys-api-reference.md
Normal file
@ -0,0 +1,9 @@
|
||||
---
|
||||
sidebar_label: キー
|
||||
title: キー
|
||||
---
|
||||
|
||||
## すべてのキーのリストを取得する
|
||||
|
||||
このファイルは、ビルドプロセス中に `clickhouseapi.js` によって生成されます。
|
||||
内容を変更する必要がある場合は、`clickhouseapi.js` を編集してください。
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user