From 9c0ae5a824f194ffe7ae2c83e7545df7838e55df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 8 Jul 2022 07:20:45 +0200 Subject: [PATCH] Correct Pinot result --- benchmark/generate-results.sh | 3 + benchmark/index.html | 89 +++++++++++++----------- benchmark/pinot/results/c6a.4xlarge.json | 2 +- 3 files changed, 51 insertions(+), 43 deletions(-) diff --git a/benchmark/generate-results.sh b/benchmark/generate-results.sh index 87d67bcc81c..833f2a88935 100755 --- a/benchmark/generate-results.sh +++ b/benchmark/generate-results.sh @@ -1,5 +1,8 @@ #!/bin/bash -e +# This script will substitute the benchmark results into the HTML page. +# Note: editing HTML with sed may look strange, but at least we avoid using node.js and npm, and that's good. + ( sed '/^const data = \[$/q' index.html diff --git a/benchmark/index.html b/benchmark/index.html index 895fe581d76..90fbf47ec40 100644 --- a/benchmark/index.html +++ b/benchmark/index.html @@ -206,7 +206,7 @@ const data = [ "cluster_size": "serverless", "comment": "", - "tags": ["stateless", "managed", "SQL", "Java", "column-oriented"], + "tags": ["stateless", "managed", "Java", "column-oriented"], "load_time": 0, "data_size": 13800000000, @@ -265,7 +265,7 @@ const data = [ "cluster_size": "serverless", "comment": "", - "tags": ["stateless", "managed", "SQL", "Java", "column-oriented"], + "tags": ["stateless", "managed", "Java", "column-oriented"], "load_time": 0, "data_size": 13800000000, @@ -324,7 +324,7 @@ const data = [ "cluster_size": 1, "comment": "Some queries cannot run due to ERROR 1114 (HY000) at line 1: The table '/rdsdbdata/tmp/#sqlaff_e5_0' is full", - "tags": ["managed", "SQL", "C++", "MySQL compatible", "row-oriented"], + "tags": ["managed", "C++", "MySQL compatible", "row-oriented"], "load_time": 7687.318, "data_size": 89614492631, @@ -383,7 +383,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["managed", "SQL", "C", "PostgreSQL compatible", "row-oriented"], + "tags": ["managed", "C", "PostgreSQL compatible", "row-oriented"], "load_time": 2127, "data_size": 52183852646, @@ -442,7 +442,7 @@ const data = [ "cluster_size": "serverless", "comment": "Looks like it shows uncompressed (billable) data size, and it is higher than CSV size", - "tags": ["managed", "SQL", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 519.485, "data_size": 94410000000, @@ -501,7 +501,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C", "PostgreSQL compatible", "column-oriented"], + "tags": ["C", "PostgreSQL compatible", "column-oriented"], "load_time": 1579, "data_size": 18980918899, @@ -560,7 +560,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C++", "column-oriented", "embedded", "stateless", "ClickHouse derivative"], + "tags": ["C++", "column-oriented", "embedded", "stateless", "ClickHouse derivative"], "load_time": 0, "data_size": 14737666736, @@ -619,7 +619,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C++", "column-oriented", "embedded", "stateless", "ClickHouse derivative"], + "tags": ["C++", "column-oriented", "embedded", "stateless", "ClickHouse derivative"], "load_time": 0, "data_size": 14779976446, @@ -678,7 +678,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C++", "column-oriented", "ClickHouse derivative"], + "tags": ["C++", "column-oriented", "ClickHouse derivative"], "load_time": 475.529, "data_size": 14345515782, @@ -737,7 +737,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C++", "column-oriented", "ClickHouse derivative"], + "tags": ["C++", "column-oriented", "ClickHouse derivative"], "load_time": 136.869, "data_size": 14571706777, @@ -796,7 +796,7 @@ const data = [ "cluster_size": 1, "comment": "For some queries it gives \"Data too large\".", - "tags": ["SQL", "Java", "column-oriented"], + "tags": ["Java", "column-oriented"], "load_time": 10687, "data_size": 109636633416, @@ -855,7 +855,7 @@ const data = [ "cluster_size": 1, "comment": "Only 90% of data successfully loaded. For some queries it gives \"Data too large\".", - "tags": ["SQL", "Rust", "column-oriented", "ClickHouse derivative"], + "tags": ["Rust", "column-oriented", "ClickHouse derivative"], "load_time": 484, "data_size": 43016643271, @@ -914,7 +914,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "Java", "column-oriented"], + "tags": ["Java", "column-oriented"], "load_time": 19620, "data_size": 45188608472, @@ -973,7 +973,7 @@ const data = [ "cluster_size": 1, "comment": "Many queries triggered OOM", - "tags": ["SQL", "C", "column-oriented", "embedded"], + "tags": ["C", "column-oriented", "embedded"], "load_time": 4217, "data_size": 27241492480, @@ -1032,7 +1032,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C", "column-oriented", "PostgreSQL compatible"], + "tags": ["C", "column-oriented", "PostgreSQL compatible"], "load_time": 1080, "data_size": 42000000000, @@ -1091,7 +1091,7 @@ const data = [ "cluster_size": 1, "comment": "Previous names: OmniSci, mapD. Many queries cannot run due to errors and limitations.", - "tags": ["SQL", "C++", "column-oriented"], + "tags": ["C++", "column-oriented"], "load_time": 572.633, "data_size": 50887437386, @@ -1150,7 +1150,7 @@ const data = [ "cluster_size": 1, "comment": "Only 90% of data successfully loaded. Some queries run for days.", - "tags": ["SQL", "C++", "column-oriented", "MySQL compatible"], + "tags": ["C++", "column-oriented", "MySQL compatible"], "load_time": 2317, "data_size": 13760341294, @@ -1209,7 +1209,7 @@ const data = [ "cluster_size": 1, "comment": "Previous name: InfiniDB.", - "tags": ["SQL", "C++", "column-oriented", "MySQL compatible"], + "tags": ["C++", "column-oriented", "MySQL compatible"], "load_time": 2507.8, "data_size": 19712857022, @@ -1268,7 +1268,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C", "column-oriented"], + "tags": ["C", "column-oriented"], "load_time": 939, "data_size": 49696606499, @@ -1327,7 +1327,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C++", "row-oriented", "MySQL compatible"], + "tags": ["C++", "row-oriented", "MySQL compatible"], "load_time": 2512, "data_size": 121588958061, @@ -1386,7 +1386,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C++", "row-oriented", "MySQL compatible"], + "tags": ["C++", "row-oriented", "MySQL compatible"], "load_time": 9472, "data_size": 171953585825, @@ -1445,10 +1445,10 @@ const data = [ "cluster_size": 1, "comment": "It successfully loaded only 94465149 out of 99997497 records. Some queries returned NullPointerException. The loading process is painful - splitting to 100 pieces required. It does not correctly report errors on data loading, the results may be incorrect.", - "tags": ["SQL", "Java", "column-oriented"], + "tags": ["Java", "column-oriented"], "load_time": 2032, - "data_size": 9245538172, + "data_size": null, "result": [ [0.002, 0.001, 0.001], @@ -1504,7 +1504,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C", "row-oriented", "PostgreSQL compatible"], + "tags": ["C", "row-oriented", "PostgreSQL compatible"], "load_time": 2342, "data_size": 77797067741, @@ -1563,7 +1563,7 @@ const data = [ "cluster_size": 1, "comment": "Many queries cannot run. It also crashes and hangs.", - "tags": ["SQL", "Java", "time-series"], + "tags": ["Java", "time-series"], "load_time": 1667.5, "data_size": 126680518680, @@ -1622,7 +1622,7 @@ const data = [ "cluster_size": 4, "comment": "One query did not run due to overflow check and another due to missing regexp function.", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2136, "data_size": 30794579968, @@ -1681,7 +1681,7 @@ const data = [ "cluster_size": "serverless", "comment": "One query did not run due to overflow check and another due to missing regexp function.", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 1933, "data_size": 30300000000, @@ -1740,7 +1740,7 @@ const data = [ "cluster_size": 1, "comment": "Previous name: MemSQL. Some queries did not run due to memory limits", - "tags": ["SQL", "MySQL compatible", "column-oriented"], + "tags": ["MySQL compatible", "column-oriented"], "load_time": 690, "data_size": 29836263469, @@ -1799,7 +1799,7 @@ const data = [ "cluster_size": 32, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -1858,7 +1858,7 @@ const data = [ "cluster_size": 64, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -1917,7 +1917,7 @@ const data = [ "cluster_size": 128, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -1976,7 +1976,7 @@ const data = [ "cluster_size": 8, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -2035,7 +2035,7 @@ const data = [ "cluster_size": 4, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -2094,7 +2094,7 @@ const data = [ "cluster_size": 2, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -2153,7 +2153,7 @@ const data = [ "cluster_size": 16, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -2212,7 +2212,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "managed", "column-oriented"], + "tags": ["managed", "column-oriented"], "load_time": 2524, "data_size": 12300000000, @@ -2271,7 +2271,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C", "embedded", "row-oriented"], + "tags": ["C", "embedded", "row-oriented"], "load_time": 2608, "data_size": 75776589824, @@ -2330,7 +2330,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C", "PostgreSQL compatible", "column-oriented", "time-series"], + "tags": ["C", "PostgreSQL compatible", "column-oriented", "time-series"], "load_time": 4605, "data_size": 20333747165, @@ -2389,7 +2389,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "C", "PostgreSQL compatible", "row-oriented", "time-series"], + "tags": ["C", "PostgreSQL compatible", "row-oriented", "time-series"], "load_time": 1620, "data_size": 72882392030, @@ -2448,7 +2448,7 @@ const data = [ "cluster_size": 1, "comment": "", - "tags": ["SQL", "column-oriented"], + "tags": ["column-oriented"], "load_time": 12239, "data_size": 22597367016, @@ -2752,6 +2752,9 @@ function renderSummary(filtered_data) { sorted_indices.map(idx => { const elem = filtered_data[idx]; + + if (selectors.metric == 'size' && elem.data_size === null) { return; } + let tr = document.createElement('tr'); tr.className = 'summary-row'; @@ -2881,6 +2884,7 @@ function render() { details_head.appendChild(th); }); + /// Load times { let tr = document.createElement('tr'); tr.className = 'shadow'; @@ -2905,6 +2909,7 @@ function render() { details_body.appendChild(tr); } + /// Data sizes { let tr = document.createElement('tr'); tr.className = 'shadow'; @@ -2917,10 +2922,10 @@ function render() { sorted_indices.map(idx => { const curr_size = filtered_data[idx].data_size; const baseline_size = Math.min(...filtered_data.map(elem => elem.data_size).filter(x => x)); - const ratio = curr_size / baseline_size; + const ratio = curr_size !== null ? curr_size / baseline_size : null; let td = document.createElement('td'); - td.appendChild(document.createTextNode(curr_size ? `${(curr_size / 1024 / 1024 / 1024).toFixed(2)} GiB (×${ratio.toFixed(2)})` : '0')); + td.appendChild(document.createTextNode(curr_size !== null ? `${(curr_size / 1024 / 1024 / 1024).toFixed(2)} GiB (×${ratio.toFixed(2)})` : '☠')); colorize(td, ratio); tr.appendChild(td); diff --git a/benchmark/pinot/results/c6a.4xlarge.json b/benchmark/pinot/results/c6a.4xlarge.json index 04855bd604a..8be94900f42 100644 --- a/benchmark/pinot/results/c6a.4xlarge.json +++ b/benchmark/pinot/results/c6a.4xlarge.json @@ -8,7 +8,7 @@ "tags": ["Java", "column-oriented"], "load_time": 2032, - "data_size": 9245538172, + "data_size": null, "result": [ [0.002, 0.001, 0.001],