From f15688fe9c1580f2b652a28685320ca5d753c454 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jun 2022 09:04:18 +0200 Subject: [PATCH 01/49] Dashboard --- utils/dashboard.html | 453 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 453 insertions(+) create mode 100644 utils/dashboard.html diff --git a/utils/dashboard.html b/utils/dashboard.html new file mode 100644 index 00000000000..5543ea945e4 --- /dev/null +++ b/utils/dashboard.html @@ -0,0 +1,453 @@ + + + + + Dashboard + + + + + +
+
+ start: + word2: + +
+ 🌚🌞 +
+
+
+ + + From 24a57efbf281f7a3c3272c11a4b4075f64e28c52 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jun 2022 09:25:01 +0200 Subject: [PATCH 02/49] Dashboard --- utils/dashboard.html | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/utils/dashboard.html b/utils/dashboard.html index 5543ea945e4..0358815483e 100644 --- a/utils/dashboard.html +++ b/utils/dashboard.html @@ -18,10 +18,13 @@ padding: 1rem; overflow-x: hidden; background: linear-gradient(to bottom, #00CCFF, #00D0D0); + display: grid; + grid-template-columns: auto; + grid-template-rows: fit-content(10%) auto; } #charts { - height: calc(100% - 3em); + height: 100%; display: flex; flex-flow: row wrap; gap: 1rem; @@ -38,7 +41,7 @@ .chart div { position: absolute; } - .inputs { height: 3em; } + .inputs {} .inputs input { box-shadow: 0 0 1rem rgba(0, 0, 0, 0.25); @@ -54,9 +57,13 @@ .u-legend th { display: none; } - #toggle-dark, #toggle-light { + .themes { float: right; font-size: 20pt; + margin-bottom: 1rem; + } + + #toggle-dark, #toggle-light { padding-right: 0.5rem; cursor: pointer; } @@ -70,6 +77,7 @@ background: #FFCB80; font-weight: bold; cursor: pointer; + margin-bottom: 1rem; } #run:hover { @@ -77,7 +85,6 @@ } #add { - background: white; font-weight: bold; cursor: pointer; padding-left: 0.5rem; @@ -86,6 +93,7 @@ float: right; margin-right: 0; margin-left: 1rem; + margin-bottom: 1rem; } #add:hover { @@ -96,12 +104,13 @@ display: inline; } - form span { + form .param_name { font-size: 14pt; padding: 0.25rem; background: #EEE; display: inline-block; box-shadow: 0 0 1rem rgba(0, 0, 0, 0.25); + margin-bottom: 1rem; } input:focus { @@ -174,16 +183,21 @@ .edit-confirm:hover { filter: contrast(125%); } + + .nowrap { + white-space: pre; + }
+ + 🌚🌞
- start: - word2: + start: + word2:
- 🌚🌞
@@ -301,10 +315,8 @@ query_editor_textarea.focus(); }); chart.addEventListener('dblclick', e => { - if (e.target != insert) { - query_editor.style.display = 'grid'; - query_editor_textarea.focus(); - } + query_editor.style.display = 'grid'; + query_editor_textarea.focus(); }); let trash = document.createElement('a'); From 204f1741a19577cd4526a761a5319e2b0a67de4b Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 20 Jun 2022 09:27:24 +0200 Subject: [PATCH 03/49] Dashboard --- utils/dashboard.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/dashboard.html b/utils/dashboard.html index 0358815483e..9e9f968a2be 100644 --- a/utils/dashboard.html +++ b/utils/dashboard.html @@ -41,7 +41,7 @@ .chart div { position: absolute; } - .inputs {} + .inputs { font-size: 14pt; } .inputs input { box-shadow: 0 0 1rem rgba(0, 0, 0, 0.25); From 37554bcd800cb0cd1ff8642211cd283839d1370e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Jun 2022 00:01:44 +0200 Subject: [PATCH 04/49] Dashboard --- utils/dashboard.html | 4 ---- 1 file changed, 4 deletions(-) diff --git a/utils/dashboard.html b/utils/dashboard.html index 9e9f968a2be..5a207a457f4 100644 --- a/utils/dashboard.html +++ b/utils/dashboard.html @@ -314,10 +314,6 @@ query_editor.style.display = 'grid'; query_editor_textarea.focus(); }); - chart.addEventListener('dblclick', e => { - query_editor.style.display = 'grid'; - query_editor_textarea.focus(); - }); let trash = document.createElement('a'); let trash_text = document.createTextNode('✖️'); From 917fe5b36644dd172ba00abef3a026464289dade Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 21 Jun 2022 02:43:52 +0200 Subject: [PATCH 05/49] Dashboard --- utils/dashboard.html | 61 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 5 deletions(-) diff --git a/utils/dashboard.html b/utils/dashboard.html index 5a207a457f4..d45d10cbeac 100644 --- a/utils/dashboard.html +++ b/utils/dashboard.html @@ -3,7 +3,7 @@ Dashboard - + + + +
+
+
+ + + +
+
+ + 🌚🌞 +
+
+
+
+
+ + + diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index 526b86a5c28..cbf0a2b40a5 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -169,10 +169,17 @@ void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IS replicas_status_handler->allowGetAndHeadRequest(); factory.addHandler(replicas_status_handler); - auto web_ui_handler = std::make_shared>(server, "play.html"); - web_ui_handler->attachNonStrictPath("/play"); - web_ui_handler->allowGetAndHeadRequest(); - factory.addHandler(web_ui_handler); + auto play_handler = std::make_shared>(server); + play_handler->attachNonStrictPath("/play"); + factory.addHandler(play_handler); + + auto dashboard_handler = std::make_shared>(server); + dashboard_handler->attachNonStrictPath("/dashboard"); + factory.addHandler(dashboard_handler); + + auto js_handler = std::make_shared>(server); + js_handler->attachNonStrictPath("/js/"); + factory.addHandler(js_handler); } void addDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IServer & server, AsynchronousMetrics & async_metrics) diff --git a/src/Server/WebUIRequestHandler.cpp b/src/Server/WebUIRequestHandler.cpp index 50aa0be4778..c1d4535c22a 100644 --- a/src/Server/WebUIRequestHandler.cpp +++ b/src/Server/WebUIRequestHandler.cpp @@ -12,8 +12,8 @@ namespace DB { -WebUIRequestHandler::WebUIRequestHandler(IServer & server_, std::string resource_name_) - : server(server_), resource_name(std::move(resource_name_)) +WebUIRequestHandler::WebUIRequestHandler(IServer & server_) + : server(server_) { } @@ -28,8 +28,38 @@ void WebUIRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerR response.setChunkedTransferEncoding(true); setResponseDefaultHeaders(response, keep_alive_timeout); - response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_OK); - *response.send() << getResource(resource_name); + + if (request.getURI().starts_with("/play")) + { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_OK); + *response.send() << getResource("play.html"); + } + else if (request.getURI().starts_with("/dashboard")) + { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_OK); + + std::string html(getResource("dashboard.html")); + + /// Replace a link to external JavaScript file to embedded file. + /// This allows to open the HTML without running a server and to host it on server. + /// Note: we can embed the JavaScript file inline to the HTML, + /// but we don't do it to keep the "view-source" perfectly readable. + + static re2::RE2 uplot_url = R"(https://[^\s"'`]+u[Pp]lot[^\s"'`]*\.js)"; + RE2::Replace(&html, uplot_url, "/js/uplot.js"); + + *response.send() << html; + } + else if (request.getURI() == "/js/uplot.js") + { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_OK); + *response.send() << getResource("js/uplot.js"); + } + else + { + response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_NOT_FOUND); + *response.send() << "Not found.\n"; + } } } diff --git a/src/Server/WebUIRequestHandler.h b/src/Server/WebUIRequestHandler.h index 1c52b626091..e05c84c68c2 100644 --- a/src/Server/WebUIRequestHandler.h +++ b/src/Server/WebUIRequestHandler.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -13,11 +14,10 @@ class WebUIRequestHandler : public HTTPRequestHandler { private: IServer & server; - std::string resource_name; + public: - WebUIRequestHandler(IServer & server_, std::string resource_name_); + WebUIRequestHandler(IServer & server_); void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response) override; }; } - From fa9df87198d312379bae66ca61f462a0e25d4fd6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Aug 2022 12:56:23 +0200 Subject: [PATCH 27/49] Fix margin --- programs/server/dashboard.html | 1 + 1 file changed, 1 insertion(+) diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index d209f50827d..6a7b1a2265c 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -245,6 +245,7 @@ background: var(--chart-background); color: var(--text-color); resize: none; + margin: 0; } .query-editor input { From ab079b270ad47f8863544a6d1eeefa365da5ceba Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Aug 2022 13:07:46 +0200 Subject: [PATCH 28/49] Add comments --- programs/server/dashboard.html | 116 ++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 18 deletions(-) diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index 6a7b1a2265c..ec70d7d6bc3 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -299,79 +299,155 @@ let host = 'https://play.clickhouse.com/'; let user = 'explorer'; let password = ''; +/// If it is hosted on server, assume that it is the address of ClickHouse. if (location.protocol != 'file:') { host = location.origin; user = 'default'; } +/// This is just a demo configuration of the dashboard. + let queries = [ { "title": "Queries/second", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_Query)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_Query) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "CPU Usage (cores)", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUVirtualTimeMicroseconds) / 1000000\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUVirtualTimeMicroseconds) / 1000000 +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "Queries Running", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_Query)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_Query) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "Merges Running", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_Merge)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_Merge) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { - "title": "Selected Bytes/second", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_SelectedBytes)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "title": "SELECTed Bytes/second", + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_SelectedBytes) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "IO Wait", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSIOWaitMicroseconds) / 1000000\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSIOWaitMicroseconds) / 1000000 +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "CPU Wait", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUWaitMicroseconds) / 1000000\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSCPUWaitMicroseconds) / 1000000 +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "OS CPU Usage (Userspace)", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)\nFROM system.asynchronous_metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'OSUserTimeNormalized'\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) +FROM system.asynchronous_metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +AND metric = 'OSUserTimeNormalized' +GROUP BY t +ORDER BY t` }, { "title": "OS CPU Usage (Kernel)", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)\nFROM system.asynchronous_metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'OSSystemTimeNormalized'\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) +FROM system.asynchronous_metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +AND metric = 'OSSystemTimeNormalized' +GROUP BY t +ORDER BY t` }, { "title": "Read From Disk", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSReadBytes)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSReadBytes) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "Read From Filesystem", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSReadChars)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_OSReadChars) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "Memory (tracked)", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_MemoryTracking)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(CurrentMetric_MemoryTracking) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "Load Average (15 minutes)", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)\nFROM system.asynchronous_metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'LoadAverage15'\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) +FROM system.asynchronous_metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +AND metric = 'LoadAverage15' +GROUP BY t +ORDER BY t` }, { "title": "Selected Rows/second", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_SelectedRows)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_SelectedRows) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "Inserted Rows/second", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_InsertedRows)\nFROM system.metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_InsertedRows) +FROM system.metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +GROUP BY t +ORDER BY t` }, { "title": "Total MergeTree Parts", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value)\nFROM system.asynchronous_metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'TotalPartsOfMergeTreeTables'\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) +FROM system.asynchronous_metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +AND metric = 'TotalPartsOfMergeTreeTables' +GROUP BY t +ORDER BY t` }, { "title": "Max Parts For Partition", - "query": "SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, max(value)\nFROM system.asynchronous_metric_log\nWHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}\nAND metric = 'MaxPartCountForPartition'\nGROUP BY t\nORDER BY t" + "query": `SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, max(value) +FROM system.asynchronous_metric_log +WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} +AND metric = 'MaxPartCountForPartition' +GROUP BY t +ORDER BY t` } ]; @@ -382,6 +458,10 @@ let params = { "seconds": "86400" }; +/// Implementation note: it might be more natural to use some reactive framework. +/// But for now it is small enough to avoid it. As a bonus we have less number of dependencies, +/// which is better for maintainability. + let theme = 'light'; function setTheme(new_theme) { From 861e5dbb761c690df81afd4310ae25071416faae Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 21 Aug 2022 13:26:32 +0200 Subject: [PATCH 29/49] Add comments --- programs/server/dashboard.html | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html index ec70d7d6bc3..e18f068d0e3 100644 --- a/programs/server/dashboard.html +++ b/programs/server/dashboard.html @@ -295,6 +295,22 @@
- - - -
-
-
- - - -
-
- - 🌚🌞 -
-
-
-
-
- - - From 4caf2c4c338312df4a5ea1aea6ba1d55409bfa08 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Sun, 21 Aug 2022 18:10:32 +0000 Subject: [PATCH 32/49] Reduce some usage of StringRef See #39535 and #39300 --- src/Columns/ColumnConst.h | 2 +- src/Columns/ColumnDecimal.h | 4 ++-- src/Columns/ColumnFixedString.h | 2 +- src/Columns/ColumnVector.h | 4 ++-- src/Columns/IColumn.h | 2 +- src/Common/ColumnsHashing.h | 6 +++--- src/Functions/reinterpretAs.cpp | 2 +- src/Interpreters/ExpressionJIT.cpp | 6 +++--- src/Interpreters/JIT/compileFunction.cpp | 4 ++-- 9 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index 21cfaf7f64c..b8b3984be20 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -269,7 +269,7 @@ public: bool isFixedAndContiguous() const override { return data->isFixedAndContiguous(); } bool valuesHaveFixedSize() const override { return data->valuesHaveFixedSize(); } size_t sizeOfValueIfFixed() const override { return data->sizeOfValueIfFixed(); } - StringRef getRawData() const override { return data->getRawData(); } + std::string_ref getRawData() const override { return data->getRawData(); } /// Not part of the common interface. diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index 03875121637..afae2cd641a 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -71,9 +71,9 @@ public: data.resize_assume_reserved(data.size() - n); } - StringRef getRawData() const override + std::string_view getRawData() const override { - return StringRef(reinterpret_cast(data.data()), byteSize()); + return {reinterpret_cast(data.data()), byteSize()}; } StringRef getDataAt(size_t n) const override diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 711db056ee6..6d0b1910725 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -209,7 +209,7 @@ public: bool isFixedAndContiguous() const override { return true; } size_t sizeOfValueIfFixed() const override { return n; } - StringRef getRawData() const override { return StringRef(chars.data(), chars.size()); } + std::string_view getRawData() const override { return {chars.data(), chars.size()}; } /// Specialized part of interface, not from IColumn. void insertString(const String & string) { insertData(string.c_str(), string.size()); } diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h index 6ba9abaca32..88e953891cc 100644 --- a/src/Columns/ColumnVector.h +++ b/src/Columns/ColumnVector.h @@ -332,9 +332,9 @@ public: bool isFixedAndContiguous() const override { return true; } size_t sizeOfValueIfFixed() const override { return sizeof(T); } - StringRef getRawData() const override + std::string_view getRawData() const override { - return StringRef(reinterpret_cast(data.data()), byteSize()); + return {reinterpret_cast(data.data()), byteSize()}; } StringRef getDataAt(size_t n) const override diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index 8b693015674..974925d247e 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -507,7 +507,7 @@ public: [[nodiscard]] virtual bool isFixedAndContiguous() const { return false; } /// If isFixedAndContiguous, returns the underlying data array, otherwise throws an exception. - [[nodiscard]] virtual StringRef getRawData() const { throw Exception("Column " + getName() + " is not a contiguous block of memory", ErrorCodes::NOT_IMPLEMENTED); } + [[nodiscard]] virtual std::string_view getRawData() const { throw Exception("Column " + getName() + " is not a contiguous block of memory", ErrorCodes::NOT_IMPLEMENTED); } /// If valuesHaveFixedSize, returns size of value, otherwise throw an exception. [[nodiscard]] virtual size_t sizeOfValueIfFixed() const { throw Exception("Values of column " + getName() + " are not fixed size.", ErrorCodes::CANNOT_GET_SIZE_OF_FIELD); } diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index 0e4617345fa..54446950567 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -41,12 +41,12 @@ struct HashMethodOneNumber /// If the keys of a fixed length then key_sizes contains their lengths, empty otherwise. HashMethodOneNumber(const ColumnRawPtrs & key_columns, const Sizes & /*key_sizes*/, const HashMethodContextPtr &) { - vec = key_columns[0]->getRawData().data; + vec = key_columns[0]->getRawData().data(); } explicit HashMethodOneNumber(const IColumn * column) { - vec = column->getRawData().data; + vec = column->getRawData().data(); } /// Creates context. Method is called once and result context is used in all threads. @@ -577,7 +577,7 @@ struct HashMethodKeysFixed columns_data.reset(new const char*[keys_size]); for (size_t i = 0; i < keys_size; ++i) - columns_data[i] = Base::getActualColumns()[i]->getRawData().data; + columns_data[i] = Base::getActualColumns()[i]->getRawData().data(); } #endif } diff --git a/src/Functions/reinterpretAs.cpp b/src/Functions/reinterpretAs.cpp index 8e656863cdb..76afedb4f06 100644 --- a/src/Functions/reinterpretAs.cpp +++ b/src/Functions/reinterpretAs.cpp @@ -301,7 +301,7 @@ private: ColumnFixedString::Chars & data_to = dst.getChars(); data_to.resize(n * rows); - memcpy(data_to.data(), src.getRawData().data, data_to.size()); + memcpy(data_to.data(), src.getRawData().data(), data_to.size()); } static void NO_INLINE executeToString(const IColumn & src, ColumnString & dst) diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index a179c4d8bf6..c37d4d5b6a2 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -113,14 +113,14 @@ public: const auto & null_map_column = nullable_column->getNullMapColumn(); auto nested_column_raw_data = nested_column.getRawData(); - __msan_unpoison(nested_column_raw_data.data, nested_column_raw_data.size); + __msan_unpoison(nested_column_raw_data.data(), nested_column_raw_data.size()); auto null_map_column_raw_data = null_map_column.getRawData(); - __msan_unpoison(null_map_column_raw_data.data, null_map_column_raw_data.size); + __msan_unpoison(null_map_column_raw_data.data(), null_map_column_raw_data.size()); } else { - __msan_unpoison(result_column->getRawData().data, result_column->getRawData().size); + __msan_unpoison(result_column->getRawData().data(), result_column->getRawData().size()); } #endif diff --git a/src/Interpreters/JIT/compileFunction.cpp b/src/Interpreters/JIT/compileFunction.cpp index 9d2ab40bf76..353ab84674c 100644 --- a/src/Interpreters/JIT/compileFunction.cpp +++ b/src/Interpreters/JIT/compileFunction.cpp @@ -47,11 +47,11 @@ ColumnData getColumnData(const IColumn * column) if (const auto * nullable = typeid_cast(column)) { - result.null_data = nullable->getNullMapColumn().getRawData().data; + result.null_data = nullable->getNullMapColumn().getRawData().data(); column = & nullable->getNestedColumn(); } - result.data = column->getRawData().data; + result.data = column->getRawData().data(); return result; } From 26384b2543e1e02c7157aa60a5ef1cff888b7dcd Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 19 Aug 2022 20:13:46 +0200 Subject: [PATCH 33/49] Fix --- src/Common/FileSegment.cpp | 222 ++++++++---------- src/Common/FileSegment.h | 10 +- src/Core/Settings.h | 1 - .../IO/CachedOnDiskReadBufferFromFile.cpp | 14 +- src/IO/ReadSettings.h | 1 - src/Interpreters/Context.cpp | 1 - 6 files changed, 98 insertions(+), 151 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index c2a12b38320..43f0c9dc831 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -430,19 +430,20 @@ void FileSegment::completeBatchAndResetDownloader() cv.notify_all(); } -void FileSegment::completeWithState(State state, bool auto_resize) +void FileSegment::completeWithState(State state) { std::lock_guard cache_lock(cache->mutex); std::lock_guard segment_lock(mutex); assertNotDetached(segment_lock); - bool is_downloader = isDownloaderImpl(segment_lock); - if (!is_downloader) + auto caller_id = getCallerId(); + if (caller_id != downloader_id) { - cv.notify_all(); - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "File segment can be completed only by downloader or downloader's FileSegmentsHodler"); + throw Exception( + ErrorCodes::LOGICAL_ERROR, + "File segment completion can be done only by downloader. (CallerId: {}, downloader id: {}", + caller_id, downloader_id); } if (state != State::DOWNLOADED @@ -450,140 +451,46 @@ void FileSegment::completeWithState(State state, bool auto_resize) && state != State::PARTIALLY_DOWNLOADED_NO_CONTINUATION) { cv.notify_all(); - throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Cannot complete file segment with state: {}", stateToString(state)); - } - - if (state == State::DOWNLOADED) - { - if (auto_resize && downloaded_size != range().size()) - { - LOG_TEST(log, "Resize cell {} to downloaded: {}", range().toString(), downloaded_size); - assert(downloaded_size <= range().size()); - segment_range = Range(segment_range.left, segment_range.left + downloaded_size - 1); - } - - /// Update states and finalize cache write buffer. - setDownloaded(segment_lock); - - if (downloaded_size != range().size()) - throw Exception( - ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, - "Cannot complete file segment as DOWNLOADED, because downloaded size ({}) does not match expected size ({})", - downloaded_size, range().size()); + throw Exception( + ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, + "Cannot complete file segment with state: {}", stateToString(state)); } download_state = state; - - try - { - completeImpl(cache_lock, segment_lock); - } - catch (...) - { - if (!downloader_id.empty() && is_downloader) - downloader_id.clear(); - - cv.notify_all(); - throw; - } - + completeBasedOnCurrentState(cache_lock, segment_lock); cv.notify_all(); } -void FileSegment::completeBasedOnCurrentState(std::lock_guard & cache_lock) +void FileSegment::completeWithoutState(std::lock_guard & cache_lock) { std::lock_guard segment_lock(mutex); + completeBasedOnCurrentState(cache_lock, segment_lock); +} +void FileSegment::completeBasedOnCurrentState(std::lock_guard & cache_lock, std::lock_guard & segment_lock) +{ if (is_detached) return; - assertNotDetached(segment_lock); + SCOPE_EXIT({ + cv.notify_one(); + }); - completeBasedOnCurrentStateUnlocked(cache_lock, segment_lock); -} - -void FileSegment::completeBasedOnCurrentStateUnlocked( - std::lock_guard & cache_lock, std::lock_guard & segment_lock) -{ + bool is_downloader = isDownloaderImpl(segment_lock); bool is_last_holder = cache->isLastFileSegmentHolder(key(), offset(), cache_lock, segment_lock); + bool can_update_segment_state = is_downloader || is_last_holder; + size_t current_downloaded_size = getDownloadedSize(segment_lock); - if (is_last_holder && download_state == State::SKIP_CACHE) + LOG_TEST(log, "Complete without state (is_last_holder: {}). File segment info: {}", is_last_holder, getInfoForLogImpl(segment_lock)); + + if (can_update_segment_state) { - cache->remove(key(), offset(), cache_lock, segment_lock); - return; - } - - if (download_state == State::SKIP_CACHE || is_detached) - return; - - if (isDownloaderImpl(segment_lock) - && download_state != State::DOWNLOADED - && getDownloadedSize(segment_lock) == range().size()) - { - setDownloaded(segment_lock); - } - - assertNotDetached(segment_lock); - - if (download_state == State::DOWNLOADING || download_state == State::EMPTY) - { - /// Segment state can be changed from DOWNLOADING or EMPTY only if the caller is the - /// downloader or the only owner of the segment. - - bool can_update_segment_state = isDownloaderImpl(segment_lock) || is_last_holder; - - if (can_update_segment_state) - download_state = State::PARTIALLY_DOWNLOADED; - } - - try - { - completeImpl(cache_lock, segment_lock); - } - catch (...) - { - if (!downloader_id.empty() && isDownloaderImpl(segment_lock)) - downloader_id.clear(); - - cv.notify_all(); - throw; - } - - cv.notify_all(); -} - -void FileSegment::completeImpl(std::lock_guard & cache_lock, std::lock_guard & segment_lock) -{ - bool is_last_holder = cache->isLastFileSegmentHolder(key(), offset(), cache_lock, segment_lock); - - if (is_last_holder - && (download_state == State::PARTIALLY_DOWNLOADED || download_state == State::PARTIALLY_DOWNLOADED_NO_CONTINUATION)) - { - size_t current_downloaded_size = getDownloadedSize(segment_lock); - if (current_downloaded_size == 0) - { - download_state = State::SKIP_CACHE; - LOG_TEST(log, "Remove cell {} (nothing downloaded)", range().toString()); - cache->remove(key(), offset(), cache_lock, segment_lock); - } + if (current_downloaded_size == range().size()) + setDownloaded(segment_lock); else - { - /** - * Only last holder of current file segment can resize the cell, - * because there is an invariant that file segments returned to users - * in FileSegmentsHolder represent a contiguous range, so we can resize - * it only when nobody needs it. - */ - download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; - /// Resize this file segment by creating a copy file segment with DOWNLOADED state, - /// but current file segment should remain PARRTIALLY_DOWNLOADED_NO_CONTINUATION and with detached state, - /// because otherwise an invariant that getOrSet() returns a contiguous range of file segments will be broken - /// (this will be crucial for other file segment holder, not for current one). - cache->reduceSizeToDownloaded(key(), offset(), cache_lock, segment_lock); - } + download_state = State::PARTIALLY_DOWNLOADED; - markAsDetached(segment_lock); + resetDownloaderImpl(segment_lock); if (cache_writer) { @@ -593,10 +500,62 @@ void FileSegment::completeImpl(std::lock_guard & cache_lock, std::lo } } - if (!downloader_id.empty() && (isDownloaderImpl(segment_lock) || is_last_holder)) + switch (download_state) { - LOG_TEST(log, "Clearing downloader id: {}, current state: {}", downloader_id, stateToString(download_state)); - downloader_id.clear(); + case State::SKIP_CACHE: + { + if (is_last_holder) + cache->remove(key(), offset(), cache_lock, segment_lock); + + return; + } + case State::DOWNLOADED: + { + assert(downloaded_size == range().size()); + assert(is_downloaded); + break; + } + case State::DOWNLOADING: + case State::EMPTY: + { + assert(!is_last_holder); + break; + } + case State::PARTIALLY_DOWNLOADED: + case State::PARTIALLY_DOWNLOADED_NO_CONTINUATION: + { + if (is_last_holder) + { + if (current_downloaded_size == 0) + { + LOG_TEST(log, "Remove cell {} (nothing downloaded)", range().toString()); + + download_state = State::SKIP_CACHE; + cache->remove(key(), offset(), cache_lock, segment_lock); + } + else + { + LOG_TEST(log, "Resize cell {} to downloaded: {}", range().toString(), current_downloaded_size); + + /** + * Only last holder of current file segment can resize the cell, + * because there is an invariant that file segments returned to users + * in FileSegmentsHolder represent a contiguous range, so we can resize + * it only when nobody needs it. + */ + download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION; + + /// Resize this file segment by creating a copy file segment with DOWNLOADED state, + /// but current file segment should remain PARRTIALLY_DOWNLOADED_NO_CONTINUATION and with detached state, + /// because otherwise an invariant that getOrSet() returns a contiguous range of file segments will be broken + /// (this will be crucial for other file segment holder, not for current one). + cache->reduceSizeToDownloaded(key(), offset(), cache_lock, segment_lock); + } + + markAsDetached(segment_lock); + } + break; + } } LOG_TEST(log, "Completed file segment: {}", getInfoForLogImpl(segment_lock)); @@ -793,7 +752,7 @@ FileSegmentsHolder::~FileSegmentsHolder() /// under the same mutex, because complete() checks for segment pointers. std::lock_guard cache_lock(cache->mutex); - file_segment->completeBasedOnCurrentState(cache_lock); + file_segment->completeWithoutState(cache_lock); file_segment_it = file_segments.erase(current_file_segment_it); } @@ -859,13 +818,20 @@ void FileSegmentRangeWriter::completeFileSegment(FileSegment & file_segment) /// was initially set with a margin as `max_file_segment_size`. => We need to always /// resize to actual size after download finished. file_segment.getOrSetDownloader(); - file_segment.completeWithState(FileSegment::State::DOWNLOADED, /* auto_resize */true); + + assert(file_segment.downloaded_size <= file_segment.range().size()); + file_segment.segment_range = FileSegment::Range( + file_segment.segment_range.left, file_segment.segment_range.left + file_segment.downloaded_size - 1); + file_segment.reserved_size = file_segment.downloaded_size; + + file_segment.completeWithState(FileSegment::State::DOWNLOADED); + on_complete_file_segment_func(file_segment); } else { std::lock_guard cache_lock(cache->mutex); - file_segment.completeBasedOnCurrentState(cache_lock); + file_segment.completeWithoutState(cache_lock); } } diff --git a/src/Common/FileSegment.h b/src/Common/FileSegment.h index 4168cf8bd03..2b25cfd172e 100644 --- a/src/Common/FileSegment.h +++ b/src/Common/FileSegment.h @@ -142,7 +142,7 @@ public: void completeBatchAndResetDownloader(); - void completeWithState(State state, bool auto_resize = false); + void completeWithState(State state); String getInfoForLog() const; @@ -195,12 +195,8 @@ private: /// FileSegmentsHolder. complete() might check if the caller of the method /// is the last alive holder of the segment. Therefore, complete() and destruction /// of the file segment pointer must be done under the same cache mutex. - void completeBasedOnCurrentState(std::lock_guard & cache_lock); - void completeBasedOnCurrentStateUnlocked(std::lock_guard & cache_lock, std::lock_guard & segment_lock); - - void completeImpl( - std::lock_guard & cache_lock, - std::lock_guard & segment_lock); + void completeBasedOnCurrentState(std::lock_guard & cache_lock, std::lock_guard & segment_lock); + void completeWithoutState(std::lock_guard & cache_lock); void resetDownloaderImpl(std::lock_guard & segment_lock); diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 75062ce9c94..b5b67807e2f 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -588,7 +588,6 @@ static constexpr UInt64 operator""_GiB(unsigned long long value) M(UInt64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \ M(UInt64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \ M(Bool, enable_filesystem_cache, true, "Use cache for remote filesystem. This setting does not turn on/off cache for disks (must be done via disk config), but allows to bypass cache for some queries if intended", 0) \ - M(UInt64, filesystem_cache_max_wait_sec, 5, "Allow to wait at most this number of seconds for download of current remote_fs_buffer_size bytes, and skip cache if exceeded", 0) \ M(Bool, enable_filesystem_cache_on_write_operations, false, "Write into cache on write operations. To actually work this setting requires be added to disk config too", 0) \ M(Bool, enable_filesystem_cache_log, false, "Allows to record the filesystem caching log for each query", 0) \ M(Bool, read_from_filesystem_cache_if_exists_otherwise_bypass_cache, false, "", 0) \ diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp index d3777fbcbd4..dac59c596f5 100644 --- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp +++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp @@ -222,9 +222,6 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil { auto range = file_segment->range(); - size_t wait_download_max_tries = settings.filesystem_cache_max_wait_sec; - size_t wait_download_tries = 0; - auto download_state = file_segment->state(); LOG_TEST(log, "getReadBufferForFileSegment: {}", file_segment->getInfoForLog()); @@ -274,16 +271,7 @@ CachedOnDiskReadBufferFromFile::getReadBufferForFileSegment(FileSegmentPtr & fil return getCacheReadBuffer(range.left); } - if (wait_download_tries++ < wait_download_max_tries) - { - download_state = file_segment->wait(); - } - else - { - LOG_DEBUG(log, "Retries to wait for file segment download exceeded ({})", wait_download_tries); - download_state = FileSegment::State::SKIP_CACHE; - } - + download_state = file_segment->wait(); continue; } case FileSegment::State::DOWNLOADED: diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index 199ae4dcf7f..e639ecbedc2 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -80,7 +80,6 @@ struct ReadSettings size_t remote_fs_read_backoff_max_tries = 4; bool enable_filesystem_cache = true; - size_t filesystem_cache_max_wait_sec = 1; bool read_from_filesystem_cache_if_exists_otherwise_bypass_cache = false; bool enable_filesystem_cache_log = false; bool is_file_cache_persistent = false; /// Some files can be made non-evictable. diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 55ac93f7e7b..6335748cdcf 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -3451,7 +3451,6 @@ ReadSettings Context::getReadSettings() const res.remote_fs_read_max_backoff_ms = settings.remote_fs_read_max_backoff_ms; res.remote_fs_read_backoff_max_tries = settings.remote_fs_read_backoff_max_tries; res.enable_filesystem_cache = settings.enable_filesystem_cache; - res.filesystem_cache_max_wait_sec = settings.filesystem_cache_max_wait_sec; res.read_from_filesystem_cache_if_exists_otherwise_bypass_cache = settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache; res.enable_filesystem_cache_log = settings.enable_filesystem_cache_log; res.enable_filesystem_cache_on_lower_level = settings.enable_filesystem_cache_on_lower_level; From 6ae9e0f64f56e58077875e3c28330e8630b8cf87 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Aug 2022 01:46:46 +0200 Subject: [PATCH 34/49] Fix error --- src/Server/HTTPHandlerFactory.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Server/HTTPHandlerFactory.cpp b/src/Server/HTTPHandlerFactory.cpp index cbf0a2b40a5..6659c9b8390 100644 --- a/src/Server/HTTPHandlerFactory.cpp +++ b/src/Server/HTTPHandlerFactory.cpp @@ -171,14 +171,17 @@ void addCommonDefaultHandlersFactory(HTTPRequestHandlerFactoryMain & factory, IS auto play_handler = std::make_shared>(server); play_handler->attachNonStrictPath("/play"); + play_handler->allowGetAndHeadRequest(); factory.addHandler(play_handler); auto dashboard_handler = std::make_shared>(server); dashboard_handler->attachNonStrictPath("/dashboard"); + dashboard_handler->allowGetAndHeadRequest(); factory.addHandler(dashboard_handler); auto js_handler = std::make_shared>(server); js_handler->attachNonStrictPath("/js/"); + js_handler->allowGetAndHeadRequest(); factory.addHandler(js_handler); } From 1aa289f80a963a3ab407edbf636550e22d1acc5d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Aug 2022 01:49:23 +0200 Subject: [PATCH 35/49] Add a smoke test --- tests/queries/0_stateless/02389_dashboard.reference | 2 ++ tests/queries/0_stateless/02389_dashboard.sh | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/queries/0_stateless/02389_dashboard.reference create mode 100755 tests/queries/0_stateless/02389_dashboard.sh diff --git a/tests/queries/0_stateless/02389_dashboard.reference b/tests/queries/0_stateless/02389_dashboard.reference new file mode 100644 index 00000000000..bcde69ce24a --- /dev/null +++ b/tests/queries/0_stateless/02389_dashboard.reference @@ -0,0 +1,2 @@ +🌚 +leeoniya diff --git a/tests/queries/0_stateless/02389_dashboard.sh b/tests/queries/0_stateless/02389_dashboard.sh new file mode 100755 index 00000000000..9250663e3e8 --- /dev/null +++ b/tests/queries/0_stateless/02389_dashboard.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/dashboard" | grep -oF '🌚' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/js/uplot.js" | grep -oF 'leeoniya' From 756b704f81a2a4b437490b75fb07267d0e463c42 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 22 Aug 2022 08:55:53 +0000 Subject: [PATCH 36/49] build: fix build --- src/Columns/ColumnConst.h | 2 +- src/Columns/ColumnFixedString.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h index b8b3984be20..99a230720a4 100644 --- a/src/Columns/ColumnConst.h +++ b/src/Columns/ColumnConst.h @@ -269,7 +269,7 @@ public: bool isFixedAndContiguous() const override { return data->isFixedAndContiguous(); } bool valuesHaveFixedSize() const override { return data->valuesHaveFixedSize(); } size_t sizeOfValueIfFixed() const override { return data->sizeOfValueIfFixed(); } - std::string_ref getRawData() const override { return data->getRawData(); } + std::string_view getRawData() const override { return data->getRawData(); } /// Not part of the common interface. diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h index 6d0b1910725..7c2d9b1a155 100644 --- a/src/Columns/ColumnFixedString.h +++ b/src/Columns/ColumnFixedString.h @@ -209,7 +209,7 @@ public: bool isFixedAndContiguous() const override { return true; } size_t sizeOfValueIfFixed() const override { return n; } - std::string_view getRawData() const override { return {chars.data(), chars.size()}; } + std::string_view getRawData() const override { return {reinterpret_cast(chars.data()), chars.size()}; } /// Specialized part of interface, not from IColumn. void insertString(const String & string) { insertData(string.c_str(), string.size()); } From 74087d57c45e8fca905d2085abd26aa4a11e0f7b Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 22 Aug 2022 12:00:14 +0200 Subject: [PATCH 37/49] Less notify --- src/Common/FileSegment.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Common/FileSegment.cpp b/src/Common/FileSegment.cpp index 43f0c9dc831..0b2e874e9ab 100644 --- a/src/Common/FileSegment.cpp +++ b/src/Common/FileSegment.cpp @@ -458,7 +458,6 @@ void FileSegment::completeWithState(State state) download_state = state; completeBasedOnCurrentState(cache_lock, segment_lock); - cv.notify_all(); } void FileSegment::completeWithoutState(std::lock_guard & cache_lock) @@ -472,15 +471,18 @@ void FileSegment::completeBasedOnCurrentState(std::lock_guard & cach if (is_detached) return; - SCOPE_EXIT({ - cv.notify_one(); - }); - bool is_downloader = isDownloaderImpl(segment_lock); bool is_last_holder = cache->isLastFileSegmentHolder(key(), offset(), cache_lock, segment_lock); bool can_update_segment_state = is_downloader || is_last_holder; size_t current_downloaded_size = getDownloadedSize(segment_lock); + SCOPE_EXIT({ + if (is_downloader) + { + cv.notify_all(); + } + }); + LOG_TEST(log, "Complete without state (is_last_holder: {}). File segment info: {}", is_last_holder, getInfoForLogImpl(segment_lock)); if (can_update_segment_state) From 019686f6e1b5500ef56e214c95ef25e07491c395 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 22 Aug 2022 12:40:58 +0200 Subject: [PATCH 38/49] Fix bug in HadoopSnappyReadBuffer --- src/IO/HadoopSnappyReadBuffer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/IO/HadoopSnappyReadBuffer.cpp b/src/IO/HadoopSnappyReadBuffer.cpp index 2a65ca9826b..2a9f6141faa 100644 --- a/src/IO/HadoopSnappyReadBuffer.cpp +++ b/src/IO/HadoopSnappyReadBuffer.cpp @@ -212,6 +212,10 @@ bool HadoopSnappyReadBuffer::nextImpl() } return true; } + else if (decoder->result == Status::NEEDS_MORE_INPUT) + { + return nextImpl(); + } else if (decoder->result == Status::INVALID_INPUT || decoder->result == Status::BUFFER_TOO_SMALL) { throw Exception(String("hadoop snappy decode error:") + statusToString(decoder->result), ErrorCodes::SNAPPY_UNCOMPRESS_FAILED); From ae81341ab58dc6c00c66cc17dbe0252e0e4d17d1 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 22 Aug 2022 12:42:18 +0200 Subject: [PATCH 39/49] Fix hadoop unit test --- src/IO/tests/gtest_hadoop_snappy_decoder.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/IO/tests/gtest_hadoop_snappy_decoder.cpp b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp index f681e8e61e1..4db0deac08e 100644 --- a/src/IO/tests/gtest_hadoop_snappy_decoder.cpp +++ b/src/IO/tests/gtest_hadoop_snappy_decoder.cpp @@ -60,7 +60,8 @@ TEST(HadoopSnappyDecoder, repeatNeedMoreInput) String output; WriteBufferFromString out(output); copyData(read_buffer, out); + out.finalize(); UInt128 hashcode = sipHash128(output.c_str(), output.size()); String hashcode_str = getHexUIntLowercase(hashcode); - ASSERT_EQ(hashcode_str, "593afe14f61866915cc00b8c7bd86046"); + ASSERT_EQ(hashcode_str, "673e5b065186cec146789451c2a8f703"); } From f8149083642f748cdf0255ab6a4e8d5fe676d54d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Mon, 22 Aug 2022 17:39:33 +0200 Subject: [PATCH 40/49] Improvements based on comments from the PR --- src/Parsers/ASTQueryWithOutput.cpp | 2 +- src/Parsers/ASTSelectQuery.cpp | 2 +- src/Parsers/ASTSetQuery.h | 7 ++++++- src/Parsers/ParserQueryWithOutput.cpp | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Parsers/ASTQueryWithOutput.cpp b/src/Parsers/ASTQueryWithOutput.cpp index 6e2fefb35c7..6db011417a6 100644 --- a/src/Parsers/ASTQueryWithOutput.cpp +++ b/src/Parsers/ASTQueryWithOutput.cpp @@ -41,7 +41,7 @@ void ASTQueryWithOutput::formatImpl(const FormatSettings & s, FormatState & stat format->formatImpl(s, state, frame); } - if (settings_ast && !assert_cast(settings_ast.get())->is_clone) + if (settings_ast && assert_cast(settings_ast.get())->print_in_format) { s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "SETTINGS " << (s.hilite ? hilite_none : ""); settings_ast->formatImpl(s, state, frame); diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index f16377797f0..76849653b4e 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -192,7 +192,7 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F limitOffset()->formatImpl(s, state, frame); } - if (settings() && !assert_cast(settings().get())->is_clone) + if (settings() && assert_cast(settings().get())->print_in_format) { s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "SETTINGS " << (s.hilite ? hilite_none : ""); settings()->formatImpl(s, state, frame); diff --git a/src/Parsers/ASTSetQuery.h b/src/Parsers/ASTSetQuery.h index 931b1e9a01a..2c79ea18359 100644 --- a/src/Parsers/ASTSetQuery.h +++ b/src/Parsers/ASTSetQuery.h @@ -13,7 +13,12 @@ class ASTSetQuery : public IAST { public: bool is_standalone = true; /// If false, this AST is a part of another query, such as SELECT. - bool is_clone = false; /// If true, this AST is a clone from other part of the query and should not be printed in format() + + /// To support overriding certain settings in a **subquery**, we add a ASTSetQuery with Settings to all subqueries, containing + /// the list of all settings that affect them (specifically or globally to the whole query). + /// We use `print_in_format` to avoid printing these nodes when they were left unchanged from the parent copy + /// See more: https://github.com/ClickHouse/ClickHouse/issues/38895 + bool print_in_format = true; SettingsChanges changes; NameToNameMap query_parameters; diff --git a/src/Parsers/ParserQueryWithOutput.cpp b/src/Parsers/ParserQueryWithOutput.cpp index 693f4eb1551..163e71e3201 100644 --- a/src/Parsers/ParserQueryWithOutput.cpp +++ b/src/Parsers/ParserQueryWithOutput.cpp @@ -143,7 +143,7 @@ bool ParserQueryWithOutput::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (query->as()) { auto settings = query_with_output.settings_ast->clone(); - assert_cast(settings.get())->is_clone = true; + assert_cast(settings.get())->print_in_format = false; QueryWithOutputSettingsPushDownVisitor::Data data{settings}; QueryWithOutputSettingsPushDownVisitor(data).visit(query); } From 2636c032c40f49759dd041fa886cc35753c38a11 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 22 Aug 2022 18:48:02 +0200 Subject: [PATCH 41/49] Use loop instead of recursion --- src/IO/HadoopSnappyReadBuffer.cpp | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/IO/HadoopSnappyReadBuffer.cpp b/src/IO/HadoopSnappyReadBuffer.cpp index 2a9f6141faa..b6c59b18381 100644 --- a/src/IO/HadoopSnappyReadBuffer.cpp +++ b/src/IO/HadoopSnappyReadBuffer.cpp @@ -183,23 +183,28 @@ bool HadoopSnappyReadBuffer::nextImpl() if (eof) return false; - if (!in_available) + do { - in->nextIfAtEnd(); - in_available = in->buffer().end() - in->position(); - in_data = in->position(); + if (!in_available) + { + in->nextIfAtEnd(); + in_available = in->buffer().end() - in->position(); + in_data = in->position(); + } + + if (decoder->result == Status::NEEDS_MORE_INPUT && (!in_available || in->eof())) + { + throw Exception(String("hadoop snappy decode error:") + statusToString(decoder->result), ErrorCodes::SNAPPY_UNCOMPRESS_FAILED); + } + + out_capacity = internal_buffer.size(); + out_data = internal_buffer.begin(); + decoder->result = decoder->readBlock(&in_available, &in_data, &out_capacity, &out_data); + + in->position() = in->buffer().end() - in_available; } + while (decoder->result == Status::NEEDS_MORE_INPUT); - if (decoder->result == Status::NEEDS_MORE_INPUT && (!in_available || in->eof())) - { - throw Exception(String("hadoop snappy decode error:") + statusToString(decoder->result), ErrorCodes::SNAPPY_UNCOMPRESS_FAILED); - } - - out_capacity = internal_buffer.size(); - out_data = internal_buffer.begin(); - decoder->result = decoder->readBlock(&in_available, &in_data, &out_capacity, &out_data); - - in->position() = in->buffer().end() - in_available; working_buffer.resize(internal_buffer.size() - out_capacity); if (decoder->result == Status::OK) From 3666b25e6bc7680c570a203f2c30626c9b84efb2 Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Mon, 22 Aug 2022 18:48:32 +0200 Subject: [PATCH 42/49] Fix --- src/IO/HadoopSnappyReadBuffer.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/IO/HadoopSnappyReadBuffer.cpp b/src/IO/HadoopSnappyReadBuffer.cpp index b6c59b18381..408e76e19be 100644 --- a/src/IO/HadoopSnappyReadBuffer.cpp +++ b/src/IO/HadoopSnappyReadBuffer.cpp @@ -217,10 +217,6 @@ bool HadoopSnappyReadBuffer::nextImpl() } return true; } - else if (decoder->result == Status::NEEDS_MORE_INPUT) - { - return nextImpl(); - } else if (decoder->result == Status::INVALID_INPUT || decoder->result == Status::BUFFER_TOO_SMALL) { throw Exception(String("hadoop snappy decode error:") + statusToString(decoder->result), ErrorCodes::SNAPPY_UNCOMPRESS_FAILED); From 191f39ae8dc64d35a7cf4cba713f3006baf3de86 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Mon, 22 Aug 2022 21:03:25 +0200 Subject: [PATCH 43/49] DNSResolver remove AI_V4MAPPED, AI_ALL hints --- src/Common/DNSResolver.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index 10797b7a809..cf61d2795f0 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -86,13 +86,10 @@ static void splitHostAndPort(const std::string & host_and_port, std::string & ou static DNSResolver::IPAddresses hostByName(const std::string & host) { - /// Family: AF_UNSPEC - /// AI_ALL is required for checking if client is allowed to connect from an address - auto flags = Poco::Net::DNS::DNS_HINT_AI_V4MAPPED | Poco::Net::DNS::DNS_HINT_AI_ALL; /// Do not resolve IPv6 (or IPv4) if no local IPv6 (or IPv4) addresses are configured. /// It should not affect client address checking, since client cannot connect from IPv6 address /// if server has no IPv6 addresses. - flags |= Poco::Net::DNS::DNS_HINT_AI_ADDRCONFIG; + auto flags = Poco::Net::DNS::DNS_HINT_AI_ADDRCONFIG; DNSResolver::IPAddresses addresses; From 7c8e540dea0d1c357d34fb2394ca9a988c3ef8a1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 22 Aug 2022 21:55:41 +0200 Subject: [PATCH 44/49] Implement code review suggestions --- src/Server/WebUIRequestHandler.cpp | 2 ++ src/Server/WebUIRequestHandler.h | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Server/WebUIRequestHandler.cpp b/src/Server/WebUIRequestHandler.cpp index c1d4535c22a..3997e0f19b6 100644 --- a/src/Server/WebUIRequestHandler.cpp +++ b/src/Server/WebUIRequestHandler.cpp @@ -8,6 +8,8 @@ #include #include +#include + namespace DB { diff --git a/src/Server/WebUIRequestHandler.h b/src/Server/WebUIRequestHandler.h index e05c84c68c2..09fe62d41c3 100644 --- a/src/Server/WebUIRequestHandler.h +++ b/src/Server/WebUIRequestHandler.h @@ -1,7 +1,6 @@ #pragma once #include -#include namespace DB From 7428193a82778fdd0db08c6152605deaa0573dda Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 22 Aug 2022 19:17:39 -0300 Subject: [PATCH 45/49] Update named-collections.md --- docs/en/operations/named-collections.md | 87 ++++++++++++++++++++----- 1 file changed, 71 insertions(+), 16 deletions(-) diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md index 7623f7b7203..beda05f10bc 100644 --- a/docs/en/operations/named-collections.md +++ b/docs/en/operations/named-collections.md @@ -1,6 +1,6 @@ --- sidebar_position: 69 -sidebar_label: "Named connections" +sidebar_label: "Named collections" --- # Storing details for connecting to external sources in configuration files @@ -12,7 +12,7 @@ from users with only SQL access. Parameters can be set in XML `CSV` and overridden in SQL `, format = 'TSV'`. The parameters in SQL can be overridden using format `key` = `value`: `compression_method = 'gzip'`. -Named connections are stored in the `config.xml` file of the ClickHouse server in the `` section and are applied when ClickHouse starts. +Named collections are stored in the `config.xml` file of the ClickHouse server in the `` section and are applied when ClickHouse starts. Example of configuration: ```xml @@ -24,7 +24,7 @@ $ cat /etc/clickhouse-server/config.d/named_collections.xml ``` -## Named connections for accessing S3. +## Named collections for accessing S3. The description of parameters see [s3 Table Function](../sql-reference/table-functions/s3.md). @@ -42,7 +42,7 @@ Example of configuration: ``` -### Example of using named connections with the s3 function +### Example of using named collections with the s3 function ```sql INSERT INTO FUNCTION s3(s3_mydata, filename = 'test_file.tsv.gz', @@ -58,7 +58,7 @@ FROM s3(s3_mydata, filename = 'test_file.tsv.gz') 1 rows in set. Elapsed: 0.279 sec. Processed 10.00 thousand rows, 90.00 KB (35.78 thousand rows/s., 322.02 KB/s.) ``` -### Example of using named connections with an S3 table +### Example of using named collections with an S3 table ```sql CREATE TABLE s3_engine_table (number Int64) @@ -73,7 +73,7 @@ SELECT * FROM s3_engine_table LIMIT 3; └────────┘ ``` -## Named connections for accessing MySQL database +## Named collections for accessing MySQL database The description of parameters see [mysql](../sql-reference/table-functions/mysql.md). @@ -95,7 +95,7 @@ Example of configuration: ``` -### Example of using named connections with the mysql function +### Example of using named collections with the mysql function ```sql SELECT count() FROM mysql(mymysql, table = 'test'); @@ -105,7 +105,7 @@ SELECT count() FROM mysql(mymysql, table = 'test'); └─────────┘ ``` -### Example of using named connections with an MySQL table +### Example of using named collections with an MySQL table ```sql CREATE TABLE mytable(A Int64) ENGINE = MySQL(mymysql, table = 'test', connection_pool_size=3, replace_query=0); @@ -116,7 +116,7 @@ SELECT count() FROM mytable; └─────────┘ ``` -### Example of using named connections with database with engine MySQL +### Example of using named collections with database with engine MySQL ```sql CREATE DATABASE mydatabase ENGINE = MySQL(mymysql); @@ -129,7 +129,7 @@ SHOW TABLES FROM mydatabase; └────────┘ ``` -### Example of using named connections with an external dictionary with source MySQL +### Example of using named collections with an external dictionary with source MySQL ```sql CREATE DICTIONARY dict (A Int64, B String) @@ -145,7 +145,7 @@ SELECT dictGet('dict', 'B', 2); └─────────────────────────┘ ``` -## Named connections for accessing PostgreSQL database +## Named collections for accessing PostgreSQL database The description of parameters see [postgresql](../sql-reference/table-functions/postgresql.md). @@ -166,7 +166,7 @@ Example of configuration: ``` -### Example of using named connections with the postgresql function +### Example of using named collections with the postgresql function ```sql SELECT * FROM postgresql(mypg, table = 'test'); @@ -186,8 +186,7 @@ SELECT * FROM postgresql(mypg, table = 'test', schema = 'public'); └───┘ ``` - -### Example of using named connections with database with engine PostgreSQL +### Example of using named collections with database with engine PostgreSQL ```sql CREATE TABLE mypgtable (a Int64) ENGINE = PostgreSQL(mypg, table = 'test', schema = 'public'); @@ -201,7 +200,7 @@ SELECT * FROM mypgtable; └───┘ ``` -### Example of using named connections with database with engine PostgreSQL +### Example of using named collections with database with engine PostgreSQL ```sql CREATE DATABASE mydatabase ENGINE = PostgreSQL(mypg); @@ -213,7 +212,7 @@ SHOW TABLES FROM mydatabase └──────┘ ``` -### Example of using named connections with an external dictionary with source POSTGRESQL +### Example of using named collections with an external dictionary with source POSTGRESQL ```sql CREATE DICTIONARY dict (a Int64, b String) @@ -228,3 +227,59 @@ SELECT dictGet('dict', 'b', 2); │ two │ └─────────────────────────┘ ``` + +## Named collections for accessing remote ClickHouse database + +The description of parameters see [remote](../sql-reference/table-functions/remote.md/#parameters). + +Example of configuration: + +```xml + + + + localhost + 9000 + system + foo + secret + + + +``` + +### Example of using named collections with the remote/remoteSecure function + +```sql +SELECT * FROM remote(remote1, table = one); +┌─dummy─┐ +│ 0 │ +└───────┘ + +SELECT * FROM remote(remote1, database = merge(system, '^one')); +┌─dummy─┐ +│ 0 │ +└───────┘ + +INSERT INTO FUNCTION remote(remote1, database = default, table = test) VALUES (1,'a'); + +SELECT * FROM remote(remote1, database = default, table = test); +┌─a─┬─b─┐ +│ 1 │ a │ +└───┴───┘ +``` + +### Example of using named collections with an external dictionary with source ClickHouse + +```sql +CREATE DICTIONARY dict(a Int64, b String) +PRIMARY KEY a +SOURCE(CLICKHOUSE(NAME remote1 TABLE test DB default)) +LIFETIME(MIN 1 MAX 2) +LAYOUT(HASHED()); + +SELECT dictGet('dict', 'b', 1); +┌─dictGet('dict', 'b', 1)─┐ +│ a │ +└─────────────────────────┘ +``` From e6912c86b91f313153d9b9e3b10246a817da63f8 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 22 Aug 2022 19:36:01 -0300 Subject: [PATCH 46/49] Update named-collections.md --- docs/en/operations/named-collections.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md index beda05f10bc..3dace93fa4b 100644 --- a/docs/en/operations/named-collections.md +++ b/docs/en/operations/named-collections.md @@ -248,7 +248,7 @@ Example of configuration: ``` -### Example of using named collections with the remote/remoteSecure function +### Example of using named collections with the `remote`/`remoteSecure` functions ```sql SELECT * FROM remote(remote1, table = one); From 8b6e513a2b468b1a41178264cbf4bda2c056e1ff Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Aug 2022 07:51:31 +0000 Subject: [PATCH 47/49] Fix Jepsen with compressed binary --- .../src/jepsen/clickhouse_keeper/utils.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj index 596458a8f3e..11b090af140 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj @@ -196,13 +196,14 @@ [url] (let [encoded-url (md5 url) expected-file-name (.getName (io/file url)) - dest-file (str binaries-cache-dir "/" encoded-url) + dest-folder (str binaries-cache-dir "/" encoded-url) + dest-file (str dest-folder "/" "clickhouse") dest-symlink (str common-prefix "/" expected-file-name) wget-opts (concat cu/std-wget-opts [:-O dest-file])] (when-not (cu/exists? dest-file) (info "Downloading" url) - (do (c/exec :mkdir :-p binaries-cache-dir) - (c/cd binaries-cache-dir + (do (c/exec :mkdir :-p dest-folder) + (c/cd dest-folder (cu/wget-helper! wget-opts url)))) (c/exec :rm :-rf dest-symlink) (c/exec :ln :-s dest-file dest-symlink) From 032df2d7a1299a113ac6f9708ff07dfd15220e10 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Tue, 23 Aug 2022 09:56:34 +0200 Subject: [PATCH 48/49] Update tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj --- .../src/jepsen/clickhouse_keeper/utils.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj index 11b090af140..3625b24b4f9 100644 --- a/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj +++ b/tests/jepsen.clickhouse-keeper/src/jepsen/clickhouse_keeper/utils.clj @@ -197,7 +197,7 @@ (let [encoded-url (md5 url) expected-file-name (.getName (io/file url)) dest-folder (str binaries-cache-dir "/" encoded-url) - dest-file (str dest-folder "/" "clickhouse") + dest-file (str dest-folder "/clickhouse") dest-symlink (str common-prefix "/" expected-file-name) wget-opts (concat cu/std-wget-opts [:-O dest-file])] (when-not (cu/exists? dest-file) From 7e1313d9e2faa7e4738cd3f0e3111d94edf34bd2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 23 Aug 2022 15:36:13 +0300 Subject: [PATCH 49/49] Update CHANGELOG.md --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a8abd0e534..ef2b59abdcc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,6 @@ * Make the remote filesystem cache composable, allow not to evict certain files (regarding idx, mrk, ..), delete old cache version. Now it is possible to configure cache over Azure blob storage disk, over Local disk, over StaticWeb disk, etc. This PR is marked backward incompatible because cache configuration changes and in order for cache to work need to update the config file. Old cache will still be used with new configuration. The server will startup fine with the old cache configuration. Closes https://github.com/ClickHouse/ClickHouse/issues/36140. Closes https://github.com/ClickHouse/ClickHouse/issues/37889. ([Kseniia Sumarokova](https://github.com/kssenii)). [#36171](https://github.com/ClickHouse/ClickHouse/pull/36171)) #### New Feature -* Support SQL standard DELETE FROM syntax on merge tree tables and lightweight delete implementation for merge tree families. [#37893](https://github.com/ClickHouse/ClickHouse/pull/37893) ([Jianmei Zhang](https://github.com/zhangjmruc)) ([Alexander Gololobov](https://github.com/davenger)). Note: this new feature does not make ClickHouse an HTAP DBMS. * Query parameters can be set in interactive mode as `SET param_abc = 'def'` and transferred via the native protocol as settings. [#39906](https://github.com/ClickHouse/ClickHouse/pull/39906) ([Nikita Taranov](https://github.com/nickitat)). * Quota key can be set in the native protocol ([Yakov Olkhovsky](https://github.com/ClickHouse/ClickHouse/pull/39874)). * Added a setting `exact_rows_before_limit` (0/1). When enabled, ClickHouse will provide exact value for `rows_before_limit_at_least` statistic, but with the cost that the data before limit will have to be read completely. This closes [#6613](https://github.com/ClickHouse/ClickHouse/issues/6613). [#25333](https://github.com/ClickHouse/ClickHouse/pull/25333) ([kevin wan](https://github.com/MaxWk)). @@ -33,6 +32,8 @@ * Add formats `PrettyMonoBlock`, `PrettyNoEscapesMonoBlock`, `PrettyCompactNoEscapes`, `PrettyCompactNoEscapesMonoBlock`, `PrettySpaceNoEscapes`, `PrettySpaceMonoBlock`, `PrettySpaceNoEscapesMonoBlock`. [#39646](https://github.com/ClickHouse/ClickHouse/pull/39646) ([Kruglov Pavel](https://github.com/Avogar)). * Add new setting schema_inference_hints that allows to specify structure hints in schema inference for specific columns. Closes [#39569](https://github.com/ClickHouse/ClickHouse/issues/39569). [#40068](https://github.com/ClickHouse/ClickHouse/pull/40068) ([Kruglov Pavel](https://github.com/Avogar)). +#### Experimental Feature +* Support SQL standard DELETE FROM syntax on merge tree tables and lightweight delete implementation for merge tree families. [#37893](https://github.com/ClickHouse/ClickHouse/pull/37893) ([Jianmei Zhang](https://github.com/zhangjmruc)) ([Alexander Gololobov](https://github.com/davenger)). Note: this new feature does not make ClickHouse an HTAP DBMS. #### Performance Improvement * Improved memory usage during memory efficient merging of aggregation results. [#39429](https://github.com/ClickHouse/ClickHouse/pull/39429) ([Nikita Taranov](https://github.com/nickitat)).