From 92a6d30d3a54243c7d899fe3bb529ee9311a828f Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Thu, 13 Feb 2020 17:34:23 -0400 Subject: [PATCH 01/12] Added performance test for min_group_by / order_by_limit1 --- .../performance/mingroupby-orderbylimit1.xml | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 dbms/tests/performance/mingroupby-orderbylimit1.xml diff --git a/dbms/tests/performance/mingroupby-orderbylimit1.xml b/dbms/tests/performance/mingroupby-orderbylimit1.xml new file mode 100644 index 00000000000..1d3a7f6df18 --- /dev/null +++ b/dbms/tests/performance/mingroupby-orderbylimit1.xml @@ -0,0 +1,51 @@ + + once + + + + 30000 + + + + + 1 + + + + + aggregationscale + + 11111111 + 11111 + + + + tableorder + + tuple() + (key,value) + + + + + + create table mingroupby_orderbylimit1_{aggregationscale}_{tableorder} + Engine=MergeTree order by {tableorder} AS + select toUInt64( number % {aggregationscale} ) key, toUInt64(1) value + from numbers(10000000); + + + + SELECT key, min(value) + FROM mingroupby_orderbylimit1_{aggregationscale}_{tableorder} + group by key format Null; + + + + SELECT key, value + FROM mingroupby_orderbylimit1_{aggregationscale}_{tableorder} + order by key, value limit 1 by key format Null; + + + DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale}_{tableorder} + From 4857d43b6a130bd1c02c40b60a7878aba6727e69 Mon Sep 17 00:00:00 2001 From: Denis Zhuravlev Date: Thu, 13 Feb 2020 23:00:18 -0400 Subject: [PATCH 02/12] Update mingroupby-orderbylimit1.xml fix for table name --- .../performance/mingroupby-orderbylimit1.xml | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/dbms/tests/performance/mingroupby-orderbylimit1.xml b/dbms/tests/performance/mingroupby-orderbylimit1.xml index 1d3a7f6df18..37b153185f8 100644 --- a/dbms/tests/performance/mingroupby-orderbylimit1.xml +++ b/dbms/tests/performance/mingroupby-orderbylimit1.xml @@ -19,33 +19,47 @@ 11111 - - tableorder - - tuple() - (key,value) - - - create table mingroupby_orderbylimit1_{aggregationscale}_{tableorder} - Engine=MergeTree order by {tableorder} AS + create table mingroupby_orderbylimit1_{aggregationscale}_tuple + Engine=MergeTree order by tuple() AS select toUInt64( number % {aggregationscale} ) key, toUInt64(1) value from numbers(10000000); - - + + + create table mingroupby_orderbylimit1_{aggregationscale}_key_value + Engine=MergeTree order by (key,value) AS + select toUInt64( number % {aggregationscale} ) key, toUInt64(1) value + from numbers(10000000); + + + SELECT key, min(value) - FROM mingroupby_orderbylimit1_{aggregationscale}_{tableorder} + FROM mingroupby_orderbylimit1_{aggregationscale}_tuple group by key format Null; - - + + + SELECT key, min(value) + FROM mingroupby_orderbylimit1_{aggregationscale}_key_value + group by key format Null; + + + SELECT key, value - FROM mingroupby_orderbylimit1_{aggregationscale}_{tableorder} + FROM mingroupby_orderbylimit1_{aggregationscale}_tuple order by key, value limit 1 by key format Null; - DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale}_{tableorder} + + SELECT key, value + FROM mingroupby_orderbylimit1_{aggregationscale}_key_value + order by key, value limit 1 by key format Null; + + + DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale}_tuple + DROP TABLE IF EXISTS mingroupby_orderbylimit1_{aggregationscale}_key_value + From 242a1a85d4a9693b082693a9e52ac454815d953b Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 14 Feb 2020 12:34:18 +0300 Subject: [PATCH 03/12] adjust .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index d955392fe97..1e6bb1716ec 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ /docs/build /docs/publish /docs/edit +/docs/website /docs/tools/venv/ /docs/en/single.md /docs/ru/single.md From 0244e1b5edd008054ce980bcd7a90f8632a25cd9 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 14 Feb 2020 12:34:57 +0300 Subject: [PATCH 04/12] [docs] minify by default is not needed anymore --- docs/tools/build.py | 1 + docs/tools/website.py | 44 ++++++++++++++++++++++--------------------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/docs/tools/build.py b/docs/tools/build.py index e3953921175..b12f738a6e3 100755 --- a/docs/tools/build.py +++ b/docs/tools/build.py @@ -294,6 +294,7 @@ if __name__ == '__main__': arg_parser.add_argument('--skip-single-page', action='store_true') arg_parser.add_argument('--skip-pdf', action='store_true') arg_parser.add_argument('--skip-website', action='store_true') + arg_parser.add_argument('--minify', action='store_true') arg_parser.add_argument('--save-raw-single-page', type=str) arg_parser.add_argument('--verbose', action='store_true') diff --git a/docs/tools/website.py b/docs/tools/website.py index 1b5c580a7de..de971662a5a 100644 --- a/docs/tools/website.py +++ b/docs/tools/website.py @@ -21,25 +21,27 @@ def build_website(args): ) ) -def minify_website(args): - for root, _, filenames in os.walk(args.output_dir): - for filename in filenames: - path = os.path.join(root, filename) - if not ( - filename.endswith('.html') or - filename.endswith('.css') or - filename.endswith('.js') - ): - continue - logging.info('Minifying %s', path) - with open(path, 'rb') as f: - content = f.read().decode('utf-8') - if filename.endswith('.html'): - content = htmlmin.minify(content, remove_empty_space=False) - elif filename.endswith('.css'): - content = cssmin.cssmin(content) - elif filename.endswith('.js'): - content = jsmin.jsmin(content) - with open(path, 'wb') as f: - f.write(content.encode('utf-8')) +def minify_website(args): + if args.minify: + for root, _, filenames in os.walk(args.output_dir): + for filename in filenames: + path = os.path.join(root, filename) + if not ( + filename.endswith('.html') or + filename.endswith('.css') or + filename.endswith('.js') + ): + continue + + logging.info('Minifying %s', path) + with open(path, 'rb') as f: + content = f.read().decode('utf-8') + if filename.endswith('.html'): + content = htmlmin.minify(content, remove_empty_space=False) + elif filename.endswith('.css'): + content = cssmin.cssmin(content) + elif filename.endswith('.js'): + content = jsmin.jsmin(content) + with open(path, 'wb') as f: + f.write(content.encode('utf-8')) From 5dc64838076bf8669f05cfd42e9b61f0d941b701 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 14 Feb 2020 13:20:22 +0300 Subject: [PATCH 05/12] website font tuning --- .../assets/stylesheets/custom.css | 74 +--------------- website/images/clickhouse-black.svg | 1 + website/images/clickhouse-white.svg | 1 + website/index.css | 88 +++---------------- website/index.html | 10 +-- 5 files changed, 22 insertions(+), 152 deletions(-) create mode 100644 website/images/clickhouse-black.svg create mode 100644 website/images/clickhouse-white.svg diff --git a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css index 21d6ab80b38..1642d6eea54 100644 --- a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css +++ b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css @@ -1,75 +1,7 @@ -@font-face { - font-family: 'Yandex Sans Text Web'; - src: url(https://yastatic.net/adv-www/_/yy5JveR58JFkc97waf-xp0i6_jM.eot); - src: url(https://yastatic.net/adv-www/_/yy5JveR58JFkc97waf-xp0i6_jM.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/CYblzLEXzCqQIvrYs7QKQe2omRk.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/pUcnOdRwl83MvPPzrNomhyletnA.woff) format('woff'), - url(https://yastatic.net/adv-www/_/vNFEmXOcGYKJ4AAidUprHWoXrLU.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/0w7OcWZM_QLP8x-LQUXFOgXO6dE.svg#YandexSansTextWeb-Bold) format('svg'); - font-weight: 700; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Text Web'; - src: url(https://yastatic.net/adv-www/_/LI6l3L2RqcgxBe2pXmuUha37czQ.eot); - src: url(https://yastatic.net/adv-www/_/LI6l3L2RqcgxBe2pXmuUha37czQ.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/z3MYElcut0R2MF_Iw1RDNrstgYs.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/1jvKJ_-hCXl3s7gmFl-y_-UHTaI.woff) format('woff'), - url(https://yastatic.net/adv-www/_/9nzjfpCR2QHvK1EzHpDEIoVFGuY.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/gwyBTpxSwkFCF1looxqs6JokKls.svg#YandexSansTextWeb-Regular) format('svg'); - font-weight: 400; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Text Web'; - src: url(https://yastatic.net/adv-www/_/ayAFYoY8swgBLhq_I56tKj2JftU.eot); - src: url(https://yastatic.net/adv-www/_/ayAFYoY8swgBLhq_I56tKj2JftU.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/lGQcYklLVV0hyvz1HFmFsUTj8_0.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/f0AAJ9GJ4iiwEmhG-7PWMHk6vUY.woff) format('woff'), - url(https://yastatic.net/adv-www/_/4UDe4nlVvgEJ-VmLWNVq3SxCsA.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/EKLr1STNokPqxLAQa_RyN82pL98.svg#YandexSansTextWeb-Light) format('svg'); - font-weight: 300; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Display Web'; - src: url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot); - src: url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'), - url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg'); - font-weight: 400; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Display Web'; - src: url(https://yastatic.net/adv-www/_/g8_MyyKVquSZ3xEL6tarK__V9Vw.eot); - src: url(https://yastatic.net/adv-www/_/g8_MyyKVquSZ3xEL6tarK__V9Vw.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/LGiRvlfqQHlWR9YKLhsw5e7KGNA.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/40vXwNl4eYYMgteIVgLP49dwmfc.woff) format('woff'), - url(https://yastatic.net/adv-www/_/X6zG5x_wO8-AtwJ-vDLJcKC5228.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/ZKhaR0m08c8CRRL77GtFKoHcLYA.svg#YandexSansDisplayWeb-Light) format('svg'); - font-weight: 300; - font-style: normal; - font-stretch: normal -} - body { - font: 300 14pt/200% 'Yandex Sans Text Web', Arial, sans-serif; + font: 300 14pt/200% -apple-system, BlinkMacSystemFont, Segoe UI, Helvetica, Arial, sans-serif, Apple Color Emoji,Segoe UI Emoji; } -body.md-lang-zh { - font: 400 14pt/200% 'Yandex Sans Text Web', -apple-system, BlinkMacSystemFont, "Segoe UI", Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol"; -} a:link, a:visited { color: #08f; @@ -96,14 +28,13 @@ a:hover, a:active { } .md-typeset pre { - font: 13px/18px monospace, "Courier New"; + font: 13px/18px SFMono-Regular, Consolas, Liberation Mono, Menlo, monospace; display: block; padding: 1rem 3rem 1rem 1rem; overflow: scroll; } h1, h2, h3, .md-logo { - font-family: 'Yandex Sans Display Web', Arial, sans-serif; color: #000 !important; } @@ -117,7 +48,6 @@ h1, h2, h3, .md-logo { .md-header-nav__title { font-size: 3rem; - font-family: 'Yandex Sans Display Web', Arial, sans-serif; } .md-content__icon:hover { diff --git a/website/images/clickhouse-black.svg b/website/images/clickhouse-black.svg new file mode 100644 index 00000000000..10c41aed68f --- /dev/null +++ b/website/images/clickhouse-black.svg @@ -0,0 +1 @@ +ClickHouse \ No newline at end of file diff --git a/website/images/clickhouse-white.svg b/website/images/clickhouse-white.svg new file mode 100644 index 00000000000..08ef457d8b2 --- /dev/null +++ b/website/images/clickhouse-white.svg @@ -0,0 +1 @@ +ClickHouse white \ No newline at end of file diff --git a/website/index.css b/website/index.css index b303ac7ffbe..b73a8f52795 100644 --- a/website/index.css +++ b/website/index.css @@ -1,71 +1,6 @@ -@font-face { - font-family: 'Yandex Sans Text Web'; - src: url(https://yastatic.net/adv-www/_/yy5JveR58JFkc97waf-xp0i6_jM.eot); - src: url(https://yastatic.net/adv-www/_/yy5JveR58JFkc97waf-xp0i6_jM.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/CYblzLEXzCqQIvrYs7QKQe2omRk.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/pUcnOdRwl83MvPPzrNomhyletnA.woff) format('woff'), - url(https://yastatic.net/adv-www/_/vNFEmXOcGYKJ4AAidUprHWoXrLU.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/0w7OcWZM_QLP8x-LQUXFOgXO6dE.svg#YandexSansTextWeb-Bold) format('svg'); - font-weight: 700; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Text Web'; - src: url(https://yastatic.net/adv-www/_/LI6l3L2RqcgxBe2pXmuUha37czQ.eot); - src: url(https://yastatic.net/adv-www/_/LI6l3L2RqcgxBe2pXmuUha37czQ.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/z3MYElcut0R2MF_Iw1RDNrstgYs.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/1jvKJ_-hCXl3s7gmFl-y_-UHTaI.woff) format('woff'), - url(https://yastatic.net/adv-www/_/9nzjfpCR2QHvK1EzHpDEIoVFGuY.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/gwyBTpxSwkFCF1looxqs6JokKls.svg#YandexSansTextWeb-Regular) format('svg'); - font-weight: 400; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Text Web'; - src: url(https://yastatic.net/adv-www/_/ayAFYoY8swgBLhq_I56tKj2JftU.eot); - src: url(https://yastatic.net/adv-www/_/ayAFYoY8swgBLhq_I56tKj2JftU.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/lGQcYklLVV0hyvz1HFmFsUTj8_0.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/f0AAJ9GJ4iiwEmhG-7PWMHk6vUY.woff) format('woff'), - url(https://yastatic.net/adv-www/_/4UDe4nlVvgEJ-VmLWNVq3SxCsA.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/EKLr1STNokPqxLAQa_RyN82pL98.svg#YandexSansTextWeb-Light) format('svg'); - font-weight: 300; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Display Web'; - src: url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot); - src: url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'), - url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg'); - font-weight: 400; - font-style: normal; - font-stretch: normal -} - -@font-face { - font-family: 'Yandex Sans Display Web'; - src: url(https://yastatic.net/adv-www/_/g8_MyyKVquSZ3xEL6tarK__V9Vw.eot); - src: url(https://yastatic.net/adv-www/_/g8_MyyKVquSZ3xEL6tarK__V9Vw.eot?#iefix) format('embedded-opentype'), - url(https://yastatic.net/adv-www/_/LGiRvlfqQHlWR9YKLhsw5e7KGNA.woff2) format('woff2'), - url(https://yastatic.net/adv-www/_/40vXwNl4eYYMgteIVgLP49dwmfc.woff) format('woff'), - url(https://yastatic.net/adv-www/_/X6zG5x_wO8-AtwJ-vDLJcKC5228.ttf) format('truetype'), - url(https://yastatic.net/adv-www/_/ZKhaR0m08c8CRRL77GtFKoHcLYA.svg#YandexSansDisplayWeb-Light) format('svg'); - font-weight: 300; - font-style: normal; - font-stretch: normal -} - body { background: #fff; - font: 300 14pt/200% 'Yandex Sans Text Web', Arial, sans-serif; + font: 300 14pt/200% -apple-system, BlinkMacSystemFont, Segoe UI, Helvetica, Arial, sans-serif, Apple Color Emoji,Segoe UI Emoji; margin: 0; padding: 0; } @@ -93,8 +28,12 @@ body { font-size: 110%; } +#logo-text { + width: 200px; + height: 44px; +} + #main-title { - font: 400 42pt/90% 'Yandex Sans Display Web', Arial, sans-serif; margin: 0; } @@ -134,7 +73,7 @@ body { h2 { margin: 47px 0 23px; - font: 400 200%/133% 'Yandex Sans Display Web', Arial, sans-serif; + font-size: 200%; } a:link, a:visited { @@ -148,8 +87,8 @@ a:hover, a:active { } #top-menu { - margin: 12px 0 0 0; - font: 400 18pt 'Yandex Sans Display Web', Arial, sans-serif; + margin: 8px 0 0 0; + font-size: 18pt; float: right; } @@ -171,11 +110,10 @@ a:hover, a:active { .index_item:active, .index_item:visited { color: #ededed; - font: 300 100% 'Yandex Sans Display Web', Arial, sans-serif; } #short-description { - font: 300 125%/150% 'Yandex Sans Display Web', Arial, sans-serif; + font-size: 125%; margin: 0 1em 1.75em 0; text-align: left; } @@ -215,7 +153,7 @@ a:hover, a:active { #announcement { margin: 0 0 60px 0; padding: 20px 0; - font: 400 125%/133% 'Yandex Sans Display Web',Arial,sans-serif; + font-size: 125%; } .announcement-link { color: #000!important; @@ -268,7 +206,7 @@ a:hover, a:active { } pre { - font: 13px/18px monospace, "Courier New"; + font: 13px/18px SFMono-Regular, Consolas, Liberation Mono, Menlo, monospace; display: block; border-left: 5px solid #ffdb4d; padding: 5px 10px; @@ -326,7 +264,7 @@ img { } #tutorial_title { - font: normal 100px 'Yandex Sans Display Web', Arial, sans-serif; + font-size: 100px; margin-top: 25px; margin-bottom: 0; text-align: center; diff --git a/website/index.html b/website/index.html index a4a45a00aeb..aed161076fc 100644 --- a/website/index.html +++ b/website/index.html @@ -47,7 +47,7 @@ - ClickHouse + ClickHouse @@ -450,7 +450,7 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh official Docker images of ClickHouse, this is not the only option though. - Alternatively, you can easily get a running ClickHouse instance or cluster at + Alternatively, you can easily get a running ClickHouse instance or cluster at Yandex Managed Service for ClickHouse.

@@ -529,15 +529,15 @@ sudo clickhouse-client-$LATEST_VERSION/install/doinst.sh line = line.split(']('); var tail = line[1].split(') '); result.push( - '' + line[0].replace('* [', '').replace('ClickHouse Meetup in ', '') + + '' + line[0].replace('* [', '').replace('ClickHouse Meetup in ', '') + ' ' + tail[1].slice(0, -1) ); } }); if (result.length) { if (result.length == 1) { - result = 'Upcoming Meetup: ' + result[0]; + result = 'Upcoming Meetup: ' + result[0]; } else { result = 'Upcoming Meetups: ' + result.join(', '); var offset = result.lastIndexOf(', '); From bcb1ff68bbb027b2149663c7d666488824f13822 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 14 Feb 2020 13:28:19 +0300 Subject: [PATCH 06/12] Less docker/qloud in release.sh --- docs/tools/release.sh | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/docs/tools/release.sh b/docs/tools/release.sh index 45f44167951..e7c966ae5fd 100755 --- a/docs/tools/release.sh +++ b/docs/tools/release.sh @@ -4,7 +4,6 @@ set -ex BASE_DIR=$(dirname $(readlink -f $0)) BUILD_DIR="${BASE_DIR}/../build" PUBLISH_DIR="${BASE_DIR}/../publish" -IMAGE="clickhouse/website" GIT_TEST_URI="git@github.com:ClickHouse/clickhouse-test.github.io.git" GIT_PROD_URI="git@github.com:ClickHouse/clickhouse.github.io.git" @@ -14,8 +13,6 @@ then else TAG="$1" fi -FULL_NAME="${IMAGE}:${TAG}" -REMOTE_NAME="registry.yandex.net/${FULL_NAME}" DOCKER_HASH="$2" if [[ -z "$1" ]] then @@ -37,10 +34,7 @@ then git commit -a -m "add new release at $(date)" git push origin master cd "${BUILD_DIR}" - docker build -t "${FULL_NAME}" "${BUILD_DIR}" - docker tag "${FULL_NAME}" "${REMOTE_NAME}" - DOCKER_HASH=$(docker push "${REMOTE_NAME}" | tail -1 | awk '{print $3;}') - docker rmi "${FULL_NAME}" + DOCKER_HASH=$(head -c 16 < /dev/urandom | xxd -p) else rm -rf "${BUILD_DIR}" || true rm -rf "${PUBLISH_DIR}" || true @@ -67,8 +61,4 @@ then else QLOUD_ENV="${QLOUD_PROJECT}.prod" fi -QLOUD_COMPONENT="${QLOUD_ENV}.nginx" -QLOUD_VERSION=$(curl -v -H "Authorization: OAuth ${QLOUD_TOKEN}" "${QLOUD_ENDPOINT}/environment/status/${QLOUD_ENV}" | python -c "import json; import sys; print json.loads(sys.stdin.read()).get('version')") -curl -v -H "Authorization: OAuth ${QLOUD_TOKEN}" -H "Content-Type: application/json" --data "{\"repository\": \"${REMOTE_NAME}\", \"hash\": \"${DOCKER_HASH}\"}" "${QLOUD_ENDPOINT}/component/${QLOUD_COMPONENT}/${QLOUD_VERSION}/deploy" > /dev/null - echo ">>> Successfully deployed ${TAG} ${DOCKER_HASH} to ${QLOUD_ENV} <<<" From c2f92c1c54d70269c9976875619c63ff76cf1bf7 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 14 Feb 2020 13:38:31 +0300 Subject: [PATCH 07/12] blue to orange --- .../assets/stylesheets/custom.css | 9 +++------ website/index.css | 11 +++++------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css index 1642d6eea54..f6f0fbf3bfd 100644 --- a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css +++ b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css @@ -2,9 +2,8 @@ body { font: 300 14pt/200% -apple-system, BlinkMacSystemFont, Segoe UI, Helvetica, Arial, sans-serif, Apple Color Emoji,Segoe UI Emoji; } - a:link, a:visited { - color: #08f; + color: #f14600; text-decoration: none; } @@ -16,14 +15,12 @@ a:link, a:visited { color: #888 !important; } - .md-nav__link:hover, .md-nav__link:active { - color: #08f !important; + color: #f14600 !important; text-decoration: none; } a:hover, a:active { - color: #f00; text-decoration: underline; } @@ -52,7 +49,7 @@ h1, h2, h3, .md-logo { .md-content__icon:hover { text-decoration: none !important; - color: #08f !important; + color: #f14600 !important; } .md-search-result__link { diff --git a/website/index.css b/website/index.css index b73a8f52795..bfe0d49bd76 100644 --- a/website/index.css +++ b/website/index.css @@ -77,12 +77,11 @@ h2 { } a:link, a:visited { - color: #08f; + color: #f14600; text-decoration: none; } a:hover, a:active { - color: #f00; text-decoration: underline; } @@ -251,9 +250,9 @@ img { } .distributive_not_selected { - color: #08f; + color: #f14600; cursor: pointer; - border-bottom: 1px solid #08f; + border-bottom: 1px solid #f14600; } #tutorial_logo { @@ -287,8 +286,8 @@ img { } .spoiler_title { - color: #08f; - border-bottom: 1px dotted #08f; + color: #f14600; + border-bottom: 1px dotted #f14600; } .spoiler_title:hover { From 8265b2e467203ac5e96a81ea8667eb7d82386291 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 14 Feb 2020 13:44:24 +0300 Subject: [PATCH 08/12] =?UTF-8?q?fix=20=C2=B6=20color?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tools/mkdocs-material-theme/assets/stylesheets/custom.css | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css index f6f0fbf3bfd..40e9eddc9c0 100644 --- a/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css +++ b/docs/tools/mkdocs-material-theme/assets/stylesheets/custom.css @@ -156,3 +156,7 @@ h1, h2, h3, .md-logo { overflow-x: hidden; overflow-y: auto; } + +.md-typeset a:active, .md-typeset a:hover { + color: #f14600 !important; +} From d1d28cda34aad18f1bf5d7a3a1db09a0f7fb2262 Mon Sep 17 00:00:00 2001 From: Ivan Blinkov Date: Fri, 14 Feb 2020 14:03:50 +0300 Subject: [PATCH 09/12] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ab78ebfe987..db24ebe710f 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.yandex) +[![ClickHouse — open source distributed column-oriented DBMS](https://github.com/ClickHouse/ClickHouse/raw/master/website/images/logo-400x240.png)](https://clickhouse.tech) ClickHouse is an open-source column-oriented database management system that allows generating analytical data reports in real time. ## Useful Links -* [Official website](https://clickhouse.yandex/) has quick high-level overview of ClickHouse on main page. -* [Tutorial](https://clickhouse.yandex/tutorial.html) shows how to set up and query small ClickHouse cluster. -* [Documentation](https://clickhouse.yandex/docs/en/) provides more in-depth information. +* [Official website](https://clickhouse.tech/) has quick high-level overview of ClickHouse on main page. +* [Tutorial](https://clickhouse.tech/docs/en/getting_started/tutorial/) shows how to set up and query small ClickHouse cluster. +* [Documentation](https://clickhouse.tech/docs/en/) provides more in-depth information. * [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format. * [Blog](https://clickhouse.yandex/blog/en/) contains various ClickHouse-related articles, as well as announces and reports about events. -* [Contacts](https://clickhouse.yandex/#contacts) can help to get your questions answered if there are any. +* [Contacts](https://clickhouse.tech/#contacts) can help to get your questions answered if there are any. * You can also [fill this form](https://forms.yandex.com/surveys/meet-yandex-clickhouse-team/) to meet Yandex ClickHouse team in person. ## Upcoming Events From 551cd4776b039d3f3d5cfd5d444b3ec998edf5bb Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 14 Feb 2020 15:14:33 +0300 Subject: [PATCH 10/12] Move the documentation draft out of public docs. --- .../test/performance-comparison}/performance_comparison.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {docs/en/development => docker/test/performance-comparison}/performance_comparison.md (100%) diff --git a/docs/en/development/performance_comparison.md b/docker/test/performance-comparison/performance_comparison.md similarity index 100% rename from docs/en/development/performance_comparison.md rename to docker/test/performance-comparison/performance_comparison.md From 3c754113f126130f20e5ccfd1fa1a7dff2756096 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 14 Feb 2020 15:55:47 +0300 Subject: [PATCH 11/12] performance comparison --- docker/test/performance-comparison/Dockerfile | 1 + docker/test/performance-comparison/compare.sh | 71 +++++++++++++++++-- docker/test/performance-comparison/eqmed.sql | 2 +- docker/test/performance-comparison/perf.py | 23 ++++++ 4 files changed, 90 insertions(+), 7 deletions(-) diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index b733a127af1..2710c386ac7 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update \ python3 \ python3-dev \ python3-pip \ + rsync \ tree \ tzdata \ vim \ diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index c8eeec16962..58707be68d3 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -45,7 +45,10 @@ function configure sed -i 's/9000/9001/g' left/config/config.xml sed -i 's/9000/9002/g' right/config/config.xml - cat > right/config/config.d/zz-perf-test-tweaks.xml < right/config/config.d/zz-perf-test-tweaks-config.xml < true @@ -59,7 +62,20 @@ function configure EOF - cp right/config/config.d/zz-perf-test-tweaks.xml left/config/config.d/zz-perf-test-tweaks.xml + cat > right/config/users.d/zz-perf-test-tweaks-users.xml < + + + 10000000 + 0 + 1 + + + +EOF + + cp right/config/config.d/zz-perf-test-tweaks-config.xml left/config/config.d/zz-perf-test-tweaks-config.xml + cp right/config/users.d/zz-perf-test-tweaks-users.xml left/config/users.d/zz-perf-test-tweaks-users.xml rm left/config/config.d/metric_log.xml ||: rm left/config/config.d/text_log.xml ||: @@ -81,6 +97,13 @@ EOF left/clickhouse client --port 9001 --query "create database test" ||: left/clickhouse client --port 9001 --query "rename table datasets.hits_v1 to test.hits" ||: + + while killall clickhouse ; do echo . ; sleep 1 ; done + echo all killed + + # Remove logs etc, because they will be updated, and sharing them between + # servers with hardlink might cause unpredictable behavior. + rm db0/data/system/* -rf ||: } function restart @@ -125,11 +148,36 @@ function run_tests rm -v test-times.tsv ||: + # Why the ugly cut: + # 1) can't make --out-format='%n' work for deleted files, it outputs things + # like "deleted 1.xml"; + # 2) the output is not tab separated, but at least it's fixed width, so I + # cut by characters. + changed_files=$(rsync --dry-run --dirs --checksum --delete --itemize-changes left/performance/ right/performance/ | cut -c13-) + # FIXME remove some broken long tests rm right/performance/{IPv4,IPv6,modulo,parse_engine_file,number_formatting_formats,select_format}.xml ||: + test_files=$(ls right/performance/*) + + # FIXME a quick crutch to bring the run time down for the flappy tests -- + # run only those that have changed. Only on my prs for now. + if grep Kuzmenkov right-commit.txt + then + if [ "PR_TO_TEST" != "0" ] + then + test_files=$(cd right/performance && readlink -e $changed_files) + fi + fi + + # Run only explicitly specified tests, if any + if [ -v CHPC_TEST_GLOB ] + then + test_files=$(ls right/performance/${CHPC_TEST_GLOB}.xml) + fi + # Run the tests - for test in right/performance/${CHPC_TEST_GLOB:-*}.xml + for test in $test_files do test_name=$(basename $test ".xml") echo test $test_name @@ -138,8 +186,19 @@ function run_tests { time "$script_dir/perf.py" "$test" > "$test_name-raw.tsv" 2> "$test_name-err.log" ; } 2>&1 >/dev/null | grep -v ^+ >> "wall-clock-times.tsv" || continue grep ^query "$test_name-raw.tsv" | cut -f2- > "$test_name-queries.tsv" grep ^client-time "$test_name-raw.tsv" | cut -f2- > "$test_name-client-time.tsv" - right/clickhouse local --file "$test_name-queries.tsv" --structure 'query text, run int, version UInt32, time float' --query "$(cat $script_dir/eqmed.sql)" > "$test_name-report.tsv" + # this may be slow, run it in background + right/clickhouse local --file "$test_name-queries.tsv" --structure 'query text, run int, version UInt32, time float' --query "$(cat $script_dir/eqmed.sql)" > "$test_name-report.tsv" & done + + wait + + # Collect the profiles + left/clickhouse client --port 9001 --query "select * from system.trace_log format TSVWithNamesAndTypes" > left-trace-log.tsv ||: & + left/clickhouse client --port 9001 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > left-addresses.tsv ||: & + right/clickhouse client --port 9002 --query "select * from system.trace_log format TSVWithNamesAndTypes" > right-trace-log.tsv ||: & + right/clickhouse client --port 9002 --query "select arrayJoin(trace) addr, concat(splitByChar('/', addressToLine(addr))[-1], '#', demangle(addressToSymbol(addr)) ) name from system.trace_log group by addr format TSVWithNamesAndTypes" > right-addresses.tsv ||: & + + wait } # Analyze results @@ -156,7 +215,7 @@ create table queries engine Memory as select -- remove them altogether because we want to be able to detect regressions, -- but the right way to do this is not yet clear. not short and abs(diff) < 0.05 and rd[3] > 0.05 as unstable, - not short and abs(diff) > 0.05 and abs(diff) > rd[3] as changed, + not short and abs(diff) > 0.10 and abs(diff) > rd[3] as changed, * from file('*-report.tsv', TSV, 'left float, right float, diff float, rd Array(float), query text'); @@ -201,7 +260,7 @@ create table test_times_tsv engine File(TSV, 'test-times.tsv') as floor(real / queries, 3) avg_real_per_query, floor(query_min, 3) from test_time join wall_clock using test - order by query_max / query_min desc; + order by avg_real_per_query desc; create table all_queries_tsv engine File(TSV, 'all-queries.tsv') as select left, right, diff, rd, test, query diff --git a/docker/test/performance-comparison/eqmed.sql b/docker/test/performance-comparison/eqmed.sql index d7265533208..07f72963067 100644 --- a/docker/test/performance-comparison/eqmed.sql +++ b/docker/test/performance-comparison/eqmed.sql @@ -8,7 +8,7 @@ select query from ( - select query, quantiles(0.05, 0.5, 0.95)(abs(time_by_label[1] - time_by_label[2])) rd_quantiles -- quantiles of randomization distribution + select query, quantiles(0.05, 0.5, 0.95, 0.99)(abs(time_by_label[1] - time_by_label[2])) rd_quantiles -- quantiles of randomization distribution from ( select query, virtual_run, groupArrayInsertAt(median_time, random_label) time_by_label -- make array 'random label' -> 'median time' diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index c45b9f93827..1205fc97ffd 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -10,6 +10,15 @@ import pprint import time import traceback +stage_start_seconds = time.perf_counter() + +def report_stage_end(stage_name): + global stage_start_seconds + print('{}\t{}'.format(stage_name, time.perf_counter() - stage_start_seconds)) + stage_start_seconds = time.perf_counter() + +report_stage_end('start') + parser = argparse.ArgumentParser(description='Run performance test.') # Explicitly decode files as UTF-8 because sometimes we have Russian characters in queries, and LANG=C is set. parser.add_argument('file', metavar='FILE', type=argparse.FileType('r', encoding='utf-8'), nargs=1, help='test description file') @@ -35,6 +44,8 @@ if infinite_sign is not None: servers = [{'host': host, 'port': port} for (host, port) in zip(args.host, args.port)] connections = [clickhouse_driver.Client(**server) for server in servers] +report_stage_end('connect') + # Check tables that should exist tables = [e.text for e in root.findall('preconditions/table_exists')] for t in tables: @@ -47,6 +58,8 @@ for c in connections: for s in settings: c.execute("set {} = '{}'".format(s.tag, s.text)) +report_stage_end('preconditions') + # Process substitutions subst_elems = root.findall('substitutions/substitution') @@ -61,6 +74,8 @@ parameter_combinations = [dict(zip(parameter_keys, parameter_combination)) for p def substitute_parameters(query_templates, parameter_combinations): return list(set([template.format(**parameters) for template, parameters in itertools.product(query_templates, parameter_combinations)])) +report_stage_end('substitute') + # Run drop queries, ignoring errors drop_query_templates = [q.text for q in root.findall('drop_query')] drop_queries = substitute_parameters(drop_query_templates, parameter_combinations) @@ -86,6 +101,8 @@ for c in connections: for q in fill_queries: c.execute(q) +report_stage_end('fill') + # Run test queries def tsv_escape(s): return s.replace('\\', '\\\\').replace('\t', '\\t').replace('\n', '\\n').replace('\r','') @@ -93,6 +110,8 @@ def tsv_escape(s): test_query_templates = [q.text for q in root.findall('query')] test_queries = substitute_parameters(test_query_templates, parameter_combinations) +report_stage_end('substitute2') + for q in test_queries: # Prewarm: run once on both servers. Helps to bring the data into memory, # precompile the queries, etc. @@ -115,9 +134,13 @@ for q in test_queries: client_seconds = time.perf_counter() - start_seconds print('client-time\t{}\t{}\t{}'.format(tsv_escape(q), client_seconds, server_seconds)) +report_stage_end('benchmark') + # Run drop queries drop_query_templates = [q.text for q in root.findall('drop_query')] drop_queries = substitute_parameters(drop_query_templates, parameter_combinations) for c in connections: for q in drop_queries: c.execute(q) + +report_stage_end('drop') From e43fe35a69429f6c0435c50cc004182268b2fc2c Mon Sep 17 00:00:00 2001 From: Ivan <5627721+abyss7@users.noreply.github.com> Date: Fri, 14 Feb 2020 16:15:50 +0300 Subject: [PATCH 12/12] noop (#9122) --- utils/github/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/utils/github/__init__.py b/utils/github/__init__.py index 40a96afc6ff..e0fb6c8aee5 100644 --- a/utils/github/__init__.py +++ b/utils/github/__init__.py @@ -1 +1,3 @@ # -*- coding: utf-8 -*- + +# REMOVE ME