mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-12-13 09:52:38 +00:00
refactor
This commit is contained in:
serxa
parent
f2b4f9184e
commit
df2640eece
@ -3,6 +3,20 @@ import { MergeTreeUtilityVisualizer, MergeTreeTimeVisualizer } from './MergeTree
|
||||
// Component that renders a scroll bar for MergeTree that rewinds time and controls
|
||||
// MergeTreeVisualizer(s)
|
||||
export class MergeTreeRewinder {
|
||||
constructor(mt, visualizers, container) {
|
||||
this.mt = mt;
|
||||
this.visualizers = visualizers;
|
||||
this.time = mt.time;
|
||||
this.minTime = 0;
|
||||
this.maxTime = mt.time;
|
||||
this.container = container;
|
||||
|
||||
// Render the initial slider
|
||||
this.speedMultipliers = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000];
|
||||
this.currentSpeedIndex = 0;
|
||||
this.renderSlider();
|
||||
}
|
||||
|
||||
stepBackward() {
|
||||
if (this.isPlaying) {
|
||||
this.togglePlay(); // Stop playback if playing
|
||||
@ -20,11 +34,13 @@ export class MergeTreeRewinder {
|
||||
this.onTimeSet(this.time);
|
||||
this.container.select("input").property("value", this.time);
|
||||
}
|
||||
|
||||
toggleSpeed() {
|
||||
this.currentSpeedIndex = (this.currentSpeedIndex + 1) % this.speedMultipliers.length;
|
||||
this.speedButton.text(`x${this.speedMultipliers[this.currentSpeedIndex]}`);
|
||||
this.speed = 0.1 * this.speedMultipliers[this.currentSpeedIndex];
|
||||
}
|
||||
|
||||
togglePlay() {
|
||||
if (this.isPlaying) {
|
||||
clearInterval(this.playInterval);
|
||||
@ -45,24 +61,27 @@ export class MergeTreeRewinder {
|
||||
}, 100);
|
||||
}
|
||||
}
|
||||
|
||||
updateLabel() {
|
||||
const minutes = Math.floor(this.time / 60);
|
||||
const days = Math.floor(this.time / 60 / 60 / 24);
|
||||
const hours = Math.floor((this.time / 60 / 60) % 24).toString().padStart(2, '0');
|
||||
const minutes = Math.floor((this.time / 60) % 60).toString().padStart(2, '0');
|
||||
const seconds = Math.floor(this.time % 60).toString().padStart(2, '0');
|
||||
const fraction = (this.time % 1).toFixed(2).substring(2);
|
||||
this.label.text(`${minutes}:${seconds}.${fraction}`);
|
||||
let text = `${minutes}:${seconds}.${fraction}`;
|
||||
if (hours != 0 || days != 0)
|
||||
text = `${hours}:${text}`;
|
||||
if (days != 0)
|
||||
text = `${days}d${text}`;
|
||||
this.label.text(text);
|
||||
}
|
||||
constructor(mt, visualizers, container) {
|
||||
this.mt = mt;
|
||||
this.visualizers = visualizers;
|
||||
this.time = mt.time;
|
||||
this.minTime = 0;
|
||||
this.maxTime = mt.time;
|
||||
this.container = container;
|
||||
|
||||
// Render the initial slider
|
||||
this.speedMultipliers = [1, 2, 5, 10, 20, 50, 100, 200, 500, 1000];
|
||||
this.currentSpeedIndex = 0;
|
||||
this.renderSlider();
|
||||
setMinTime(value) {
|
||||
this.minTime = value;
|
||||
if (this.time < this.minTime) {
|
||||
this.time = this.minTime;
|
||||
}
|
||||
this.update();
|
||||
}
|
||||
|
||||
// Render a slider [this.minTime, this.maxTime] using HTML in `container` element with callback to this.onTimeSet
|
||||
@ -112,7 +131,7 @@ export class MergeTreeRewinder {
|
||||
.on("click", () => this.stepForward());
|
||||
|
||||
// Create slider input
|
||||
const slider = this.container.append("input")
|
||||
this.slider = this.container.append("input")
|
||||
.attr("type", "range")
|
||||
.attr("min", this.minTime)
|
||||
.attr("max", this.maxTime)
|
||||
@ -146,6 +165,7 @@ export class MergeTreeRewinder {
|
||||
// Update the slider attributes and label
|
||||
this.container.select("input")
|
||||
.attr("max", this.maxTime)
|
||||
.attr("min", this.minTime)
|
||||
.property("value", this.time);
|
||||
|
||||
this.updateLabel();
|
||||
|
||||
58
utils/merge-selector-lab/SimulationContainer.js
Normal file
58
utils/merge-selector-lab/SimulationContainer.js
Normal file
@ -0,0 +1,58 @@
|
||||
import { MergeTreeUtilityVisualizer, MergeTreeTimeVisualizer } from './MergeTreeVisualizer.js';
|
||||
import { MergeTreeRewinder } from './MergeTreeRewinder.js';
|
||||
|
||||
export class SimulationContainer {
|
||||
constructor() {
|
||||
this.visualizers = null;
|
||||
this.rewinder = null;
|
||||
}
|
||||
|
||||
#showMetrics(mt)
|
||||
{
|
||||
// Append text with metrics
|
||||
d3.select("#metrics-container")
|
||||
.text(`
|
||||
${mt.title === undefined ? "" : mt.title}
|
||||
Total: ${mt.written_bytes / 1024 / 1024} MB,
|
||||
Inserted: ${mt.inserted_bytes / 1024 / 1024} MB,
|
||||
WA: ${mt.writeAmplification().toFixed(2)},
|
||||
AvgPartCount: ${mt.avgActivePartCount().toFixed(2)}
|
||||
Time: ${(mt.time).toFixed(2)}s
|
||||
`);
|
||||
}
|
||||
|
||||
#init(data, automatic = false)
|
||||
{
|
||||
if (automatic && window.__pause)
|
||||
return;
|
||||
if (!automatic)
|
||||
console.log("SHOW", data);
|
||||
const {mt} = data;
|
||||
if (mt !== undefined)
|
||||
{
|
||||
this.visualizers = {
|
||||
util: new MergeTreeUtilityVisualizer(mt, d3.select("#util-container")),
|
||||
time: new MergeTreeTimeVisualizer(mt, d3.select("#time-container")),
|
||||
}
|
||||
this.rewinder = new MergeTreeRewinder(mt, this.visualizers, d3.select("#rewind-container"));
|
||||
this.#showMetrics(mt);
|
||||
}
|
||||
}
|
||||
|
||||
update(data, automatic = false)
|
||||
{
|
||||
if (automatic && window.__pause)
|
||||
return;
|
||||
if (!this.visualizers) {
|
||||
this.#init(data, automatic);
|
||||
} else {
|
||||
const {mt} = data;
|
||||
if (!automatic)
|
||||
console.log("SHOW UPDATE", data);
|
||||
for (const v of Object.values(this.visualizers))
|
||||
v.update();
|
||||
this.rewinder.update();
|
||||
this.#showMetrics(mt);
|
||||
}
|
||||
}
|
||||
}
|
||||
76
utils/merge-selector-lab/float_layer_merges.html
Normal file
76
utils/merge-selector-lab/float_layer_merges.html
Normal file
@ -0,0 +1,76 @@
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Merge Selector Lab</title>
|
||||
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/4.5.2/css/bootstrap.min.css">
|
||||
<script src="https://d3js.org/d3.v7.min.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container-fluid">
|
||||
<div class="p-3" id="opt-container"></div>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid bg-primary text-white">
|
||||
<div class="p-3" id="metrics-container"></div>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div class="col-md-12" id="util-container"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid bg-primary text-white">
|
||||
<div class="p-3" id="rewind-container"></div>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div class="col-md-12" id="time-container"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="p-3" id="var-container"></div>
|
||||
</div>
|
||||
|
||||
<script type="module">
|
||||
import { SimulationContainer } from './SimulationContainer.js';
|
||||
import { sequenceInserter } from './sequenceInserter.js';
|
||||
import { customScenario } from './customScenario.js';
|
||||
import { floatLayerMerges } from './floatLayerMerges.js';
|
||||
import * as util from './util.js';
|
||||
|
||||
const simContainer = new SimulationContainer();
|
||||
|
||||
const insertPartSize = 10 << 20;
|
||||
const layerBases = [Math.E, Math.E, Math.E, Math.E, Math.E, Math.E, Math.E, 3];
|
||||
|
||||
const scenario = {
|
||||
inserts: [
|
||||
sequenceInserter({
|
||||
start_time: 0,
|
||||
interval: 0.05,
|
||||
parts: 10000,
|
||||
bytes: insertPartSize,
|
||||
}),
|
||||
],
|
||||
selector: floatLayerMerges({insertPartSize, layerBases}),
|
||||
pool_size: 100,
|
||||
}
|
||||
|
||||
async function updateMergeTree({mt}) {
|
||||
simContainer.update({mt}, true);
|
||||
await util.delayMs(1);
|
||||
}
|
||||
|
||||
const signals = {
|
||||
on_every_frame: updateMergeTree,
|
||||
on_inserter_end: ({sim}) => sim.stop(),
|
||||
};
|
||||
|
||||
const mt = await customScenario(scenario, signals);
|
||||
simContainer.update({mt}, true);
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@ -1,16 +1,20 @@
|
||||
import { Chart } from './Chart.js';
|
||||
import { MergeTree } from './MergeTree.js';
|
||||
import { visualizeUtility } from './visualizeUtility.js';
|
||||
import { visualizeExecutionTime } from './visualizeExecutionTime.js';
|
||||
import { SimulationContainer } from './SimulationContainer.js';
|
||||
import { runScenario, noArrivalsScenario } from './Scenarios.js';
|
||||
import { sequenceInserter } from './sequenceInserter.js';
|
||||
import { customScenario } from './customScenario.js';
|
||||
import { fixedBaseMerges } from './fixedBaseMerges.js';
|
||||
import { floatBaseMerges } from './floatBaseMerges.js';
|
||||
import { factorsAsBaseMerges } from './factorsAsBaseMerges.js';
|
||||
import { simpleMerges } from './simpleMerges.js';
|
||||
import { integerLayerMerges } from './integerLayerMerges.js';
|
||||
import { floatLayerMerges } from './floatLayerMerges.js';
|
||||
import { factorizeNumber, allFactorPermutations } from './factorization.js';
|
||||
|
||||
const delayMs = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
import { clickHousePartsInserter } from './clickHousePartsInserter.js';
|
||||
import { getOptimalBases } from './getOptimalBases.js';
|
||||
import { gradientDescent } from './gradientDescent.js';
|
||||
import * as util from './util.js';
|
||||
|
||||
async function iterateAnalyticalSolution(series, parts, total_time = 1.0)
|
||||
{
|
||||
@ -33,7 +37,7 @@ async function iterateAnalyticalSolution(series, parts, total_time = 1.0)
|
||||
best_base = base;
|
||||
}
|
||||
|
||||
await delayMs(1);
|
||||
await util.delayMs(1);
|
||||
}
|
||||
return {y: min_y, base: best_base};
|
||||
}
|
||||
@ -60,7 +64,7 @@ async function iterateBaseLinear(selector, series, parts, total_time = 1.0)
|
||||
best = mt;
|
||||
}
|
||||
|
||||
await delayMs(1);
|
||||
await util.delayMs(1);
|
||||
}
|
||||
return {y: min_y, mt: best};
|
||||
}
|
||||
@ -91,36 +95,11 @@ async function iteratePartsFactors(selector, series, parts, total_time = 1.0)
|
||||
best = mt;
|
||||
}
|
||||
|
||||
await delayMs(1);
|
||||
await util.delayMs(1);
|
||||
}
|
||||
return {y: min_y, mt: best};
|
||||
}
|
||||
|
||||
function showSimulation(data, automatic = false)
|
||||
{
|
||||
if (automatic && window.__pause)
|
||||
return;
|
||||
if (!automatic)
|
||||
console.log("SHOW", data);
|
||||
const {mt} = data;
|
||||
if (mt !== undefined)
|
||||
{
|
||||
visualizeUtility(mt, d3.select("#util-container"), true);
|
||||
visualizeExecutionTime(mt, d3.select("#exec-container"));
|
||||
|
||||
// Append text with metrics
|
||||
d3.select("#metrics-container")
|
||||
.text(`
|
||||
${mt.title === undefined ? "" : mt.title}
|
||||
Total: ${mt.written_bytes / 1024 / 1024} MB,
|
||||
Inserted: ${mt.inserted_bytes / 1024 / 1024} MB,
|
||||
WA: ${mt.writeAmplification().toFixed(2)},
|
||||
AvgPartCount: ${mt.avgActivePartCount().toFixed(2)}
|
||||
Time: ${(mt.time).toFixed(2)}s
|
||||
`);
|
||||
}
|
||||
}
|
||||
|
||||
function argMin(array, func)
|
||||
{
|
||||
let min_value = Infinity;
|
||||
@ -181,7 +160,11 @@ async function minimizeAvgPartCount(parts, chart)
|
||||
|
||||
export async function demo()
|
||||
{
|
||||
showSimulation({mt: runScenario()}, true);
|
||||
const simContainer = new SimulationContainer();
|
||||
const mt = runScenario();
|
||||
simContainer.update({mt}, true);
|
||||
if (mt.time > 30 * 86400)
|
||||
simContainer.rewinder.setMinTime(30 * 86400);
|
||||
}
|
||||
|
||||
export async function compareAvgPartCount()
|
||||
@ -222,11 +205,11 @@ export async function compareAvgPartCount()
|
||||
series[name] = optimal_chart.addSeries(name, showSimulation);
|
||||
series[name].addPoint({x: parts, ...results[name]});
|
||||
}
|
||||
await delayMs(10);
|
||||
await util.delayMs(10);
|
||||
|
||||
// Show what simple selector is doing
|
||||
// showSimulation(simple, true);
|
||||
//await delayMs(100);
|
||||
//await util.delayMs(100);
|
||||
|
||||
// analytical_series.addPoint({x: parts, y: analytical.y});
|
||||
// analytical_series.addPoint({x: parts, y: analytical.base});
|
||||
@ -238,55 +221,149 @@ export async function compareAvgPartCount()
|
||||
}
|
||||
}
|
||||
|
||||
function* sequenceInserter({start_time, interval, parts, bytes})
|
||||
{
|
||||
yield {type: 'sleep', delay: start_time};
|
||||
for (let i = 0; i < parts; i++)
|
||||
{
|
||||
yield {type: 'insert', bytes};
|
||||
yield {type: 'sleep', delay: interval};
|
||||
}
|
||||
}
|
||||
|
||||
async function custom()
|
||||
{
|
||||
const simContainer = new SimulationContainer();
|
||||
|
||||
const scenario = {
|
||||
inserts: [
|
||||
sequenceInserter({
|
||||
start_time: 0,
|
||||
interval: 0,
|
||||
parts: 5000,
|
||||
bytes: 10 << 20,
|
||||
// sequenceInserter({
|
||||
// start_time: 0,
|
||||
// interval: 0,
|
||||
// parts: 260,
|
||||
// bytes: 10 << 20,
|
||||
// }),
|
||||
clickHousePartsInserter({
|
||||
host: "http://localhost:8123",
|
||||
user: "default",
|
||||
password: "",
|
||||
query: "SELECT * FROM parts WHERE active = 1 AND database='colossus_metrics_loadtest' AND table='local_samples' AND partition='202411' ORDER BY database, table, partition, min_block_number"
|
||||
// database: "colossus_metrics_loadtest",
|
||||
// table: "local_samples",
|
||||
// partition: "202411",
|
||||
}),
|
||||
],
|
||||
selector: simpleMerges(),
|
||||
pool_size: 1000,
|
||||
pool_size: 1,
|
||||
}
|
||||
|
||||
let show_postponed = false;
|
||||
function showMergeTree({sim, mt}) {
|
||||
showSimulation({mt}, true);
|
||||
simContainer.update({mt}, true);
|
||||
if (!show_postponed)
|
||||
{
|
||||
show_postponed = true;
|
||||
sim.postpone("show", async () => {
|
||||
show_postponed = false;
|
||||
await delayMs(10);
|
||||
//show_postponed = false;
|
||||
await util.delayMs(10);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let shown = false;
|
||||
function showMergeTreeOnce({mt}) {
|
||||
if (!shown)
|
||||
simContainer.update({mt}, true);
|
||||
shown = true;
|
||||
}
|
||||
|
||||
async function updateMergeTree({mt}) {
|
||||
simContainer.update({mt}, true);
|
||||
await util.delayMs(100);
|
||||
}
|
||||
|
||||
async function everySecond({mt}) {
|
||||
updateMergeTree({mt}, true);
|
||||
await util.delayMs(1);
|
||||
}
|
||||
|
||||
const signals = {
|
||||
//on_merge_begin: showMergeTree,
|
||||
on_merge_end: showMergeTree,
|
||||
//on_insert: showMergeTree,
|
||||
on_merge_begin: updateMergeTree,
|
||||
on_merge_end: updateMergeTree,
|
||||
// on_insert: updateMergeTree,
|
||||
on_every_real_second: everySecond,
|
||||
};
|
||||
|
||||
const mt = await customScenario(scenario, signals);
|
||||
showSimulation({mt}, true);
|
||||
simContainer.update({mt}, true);
|
||||
}
|
||||
|
||||
function solverTest()
|
||||
{
|
||||
// let n = 16; // Given constant n
|
||||
// const f = (x) => x.reduce((a, xi) => a + Math.exp(xi), 0);
|
||||
// const gradF = (x) => x.map(xi => Math.exp(xi));
|
||||
|
||||
const n = 101; // B / b
|
||||
const Ni = 1;
|
||||
const Ai = (xi) => Math.exp(xi) * (Ni + 0.5) - 0.5;
|
||||
const f = (x) => x.reduce((a, xi) => a + Ai(xi), 0);
|
||||
const diffAi = (xi) => Math.exp(xi) * (Ni + 0.5);
|
||||
const gradF = (x) => x.map(xi => diffAi(xi));
|
||||
|
||||
let eta = 1e-3; // Step size
|
||||
let fopt = Infinity;
|
||||
let Lopt = 1;
|
||||
let xopt = [];
|
||||
for (let L = 1; L <= 2 * Math.log(n); L++) {
|
||||
let x = gradientDescent(gradF, L, n, eta);
|
||||
let fx = f(x);
|
||||
if (fx < fopt) {
|
||||
fopt = fx;
|
||||
xopt = x;
|
||||
Lopt = L;
|
||||
}
|
||||
console.log("OPTION", {L, x, fx});
|
||||
}
|
||||
|
||||
console.log("OPTIMAL", {Lopt, xopt, fopt});
|
||||
}
|
||||
|
||||
async function periodicArrivals()
|
||||
{
|
||||
const simContainer = new SimulationContainer();
|
||||
|
||||
const insertPartSize = 10 << 20;
|
||||
const layerBases = [Math.E, Math.E, Math.E, Math.E, Math.E, Math.E, Math.E, 3];
|
||||
|
||||
const scenario = {
|
||||
inserts: [
|
||||
sequenceInserter({
|
||||
start_time: 0,
|
||||
interval: 0.05,
|
||||
parts: 10000,
|
||||
bytes: insertPartSize,
|
||||
}),
|
||||
],
|
||||
selector: floatLayerMerges({insertPartSize, layerBases}),
|
||||
pool_size: 100,
|
||||
}
|
||||
|
||||
async function updateMergeTree({mt}) {
|
||||
simContainer.update({mt}, true);
|
||||
await util.delayMs(1);
|
||||
}
|
||||
|
||||
const signals = {
|
||||
// on_merge_begin: updateMergeTree,
|
||||
// on_merge_end: updateMergeTree,
|
||||
// on_insert: updateMergeTree,
|
||||
on_every_frame: updateMergeTree,
|
||||
on_inserter_end: ({sim}) => sim.stop(),
|
||||
};
|
||||
|
||||
const mt = await customScenario(scenario, signals);
|
||||
simContainer.update({mt}, true);
|
||||
}
|
||||
|
||||
export async function main()
|
||||
{
|
||||
// demo();
|
||||
// compareAvgPartCount();
|
||||
custom();
|
||||
// custom();
|
||||
// solverTest();
|
||||
periodicArrivals();
|
||||
}
|
||||
|
||||
// For experiments in console
|
||||
window.getOptimalBases = getOptimalBases;
|
||||
|
||||
9
utils/merge-selector-lab/sequenceInserter.js
Normal file
9
utils/merge-selector-lab/sequenceInserter.js
Normal file
@ -0,0 +1,9 @@
|
||||
export function* sequenceInserter({start_time, interval, parts, bytes})
|
||||
{
|
||||
yield {type: 'sleep', delay: start_time};
|
||||
for (let i = 0; i < parts; i++)
|
||||
{
|
||||
yield {type: 'insert', bytes};
|
||||
yield {type: 'sleep', delay: interval};
|
||||
}
|
||||
}
|
||||
1
utils/merge-selector-lab/util.js
Normal file
1
utils/merge-selector-lab/util.js
Normal file
@ -0,0 +1 @@
|
||||
export const delayMs = (ms) => new Promise(resolve => setTimeout(resolve, ms));
|
||||
Loading…
Reference in New Issue
Block a user