mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 07:01:59 +00:00
dynamic updates of visualizations to avoid OOM
This commit is contained in:
serxa
parent
7623b5becd
commit
97fda7f23b
@ -7,7 +7,8 @@ export class MergeTree {
|
||||
|
||||
// Metrics
|
||||
this.time = 0;
|
||||
this.inserted_parts_count = 0;
|
||||
this.total_part_count = 0;
|
||||
this.inserted_part_count = 0;
|
||||
this.inserted_bytes = 0;
|
||||
this.written_bytes = 0; // inserts + merges
|
||||
this.inserted_utility = 0; // utility = size * log(size)
|
||||
@ -49,14 +50,15 @@ export class MergeTree {
|
||||
let log_bytes = Math.log2(bytes);
|
||||
let utility = bytes * log_bytes;
|
||||
let result = {
|
||||
id: this.total_part_count,
|
||||
bytes,
|
||||
log_bytes,
|
||||
utility,
|
||||
entropy: 0,
|
||||
created: now,
|
||||
level: 0,
|
||||
begin: this.inserted_parts_count,
|
||||
end: this.inserted_parts_count + 1,
|
||||
begin: this.inserted_part_count,
|
||||
end: this.inserted_part_count + 1,
|
||||
left_bytes: this.inserted_bytes,
|
||||
right_bytes: this.inserted_bytes + bytes,
|
||||
is_leftmost: false,
|
||||
@ -68,7 +70,8 @@ export class MergeTree {
|
||||
};
|
||||
this.parts.push(result);
|
||||
this.active_part_count++;
|
||||
this.inserted_parts_count++;
|
||||
this.inserted_part_count++;
|
||||
this.total_part_count++;
|
||||
this.inserted_bytes += bytes;
|
||||
this.written_bytes += bytes;
|
||||
this.inserted_utility += utility;
|
||||
@ -123,6 +126,7 @@ export class MergeTree {
|
||||
const entropy = (utility - utility0) / bytes;
|
||||
|
||||
let result = {
|
||||
id: this.total_part_count,
|
||||
bytes,
|
||||
log_bytes,
|
||||
utility,
|
||||
@ -140,6 +144,7 @@ export class MergeTree {
|
||||
};
|
||||
this.parts.push(result);
|
||||
this.active_part_count++;
|
||||
this.total_part_count++;
|
||||
for (let p of parts_to_merge)
|
||||
{
|
||||
if (p.active == false)
|
||||
|
||||
@ -26,7 +26,7 @@ export class MergeTreeInserter
|
||||
case 'insert':
|
||||
const part = this.mt.insertPart(value.bytes);
|
||||
if (this.signals.on_insert)
|
||||
this.signals.on_insert({sim: this.sim, mt: this.mt, part});
|
||||
await this.signals.on_insert({sim: this.sim, mt: this.mt, part});
|
||||
break;
|
||||
case 'sleep':
|
||||
if (value.delay > 0)
|
||||
|
||||
@ -38,7 +38,7 @@ export class MergeTreeMerger
|
||||
const {parts_to_merge} = value;
|
||||
this.#beginMerge(parts_to_merge);
|
||||
if (this.signals.on_merge_begin)
|
||||
this.signals.on_merge_begin({sim: this.sim, mt: this.mt, parts_to_merge});
|
||||
await this.signals.on_merge_begin({sim: this.sim, mt: this.mt, parts_to_merge});
|
||||
break;
|
||||
case 'wait':
|
||||
if (this.merges_running == 0)
|
||||
@ -64,16 +64,16 @@ export class MergeTreeMerger
|
||||
const bytes = parts_to_merge.reduce((sum, d) => sum + d.bytes, 0);
|
||||
const mergeDuration = this.mt.mergeDuration(bytes, parts_to_merge.length);
|
||||
|
||||
// Create a Promise that will be resolved on merge finish
|
||||
return this.pool.schedule(mergeDuration, "MergeEnd", (sim, event) => this.#onMergeEnd(parts_to_merge));
|
||||
// Schedule merge finish event
|
||||
return this.pool.schedule(mergeDuration, "MergeEnd", async (sim, event) => await this.#onMergeEnd(parts_to_merge));
|
||||
}
|
||||
|
||||
#onMergeEnd(parts_to_merge)
|
||||
async #onMergeEnd(parts_to_merge)
|
||||
{
|
||||
this.mt.advanceTime(this.sim.time);
|
||||
let part = this.mt.finishMergeParts(parts_to_merge);
|
||||
if (this.signals.on_merge_end)
|
||||
this.signals.on_merge_end({sim: this.sim, mt: this.mt, part, parts_to_merge});
|
||||
await this.signals.on_merge_end({sim: this.sim, mt: this.mt, part, parts_to_merge});
|
||||
this.merges_running--;
|
||||
this.#iterateSelector();
|
||||
}
|
||||
|
||||
@ -1,6 +1,35 @@
|
||||
import { valueToColor, formatBytesWithUnit, determineTickStep } from './visualizeHelpers.js';
|
||||
import { infoButton } from './infoButton.js';
|
||||
|
||||
function formatNumber(number)
|
||||
{
|
||||
if (number >= Math.pow(1000, 4))
|
||||
return (number / Math.pow(1000, 4)).toFixed(1) + 'T';
|
||||
else if (number >= Math.pow(1000, 3))
|
||||
return (number / Math.pow(1000, 3)).toFixed(1) + 'G';
|
||||
else if (number >= Math.pow(1000, 2))
|
||||
return (number / Math.pow(1000, 2)).toFixed(1) + 'M';
|
||||
else if (number >= 1000)
|
||||
return (number / 1000).toFixed(1) + 'K';
|
||||
else
|
||||
return number;
|
||||
}
|
||||
|
||||
function formatBytes(bytes)
|
||||
{
|
||||
const digits = 0;
|
||||
if (bytes >= Math.pow(1024, 4))
|
||||
return (bytes / Math.pow(1024, 4)).toFixed(digits) + 'TB';
|
||||
else if (bytes >= Math.pow(1024, 3))
|
||||
return (bytes / Math.pow(1024, 3)).toFixed(digits) + 'GB';
|
||||
else if (bytes >= Math.pow(1024, 2))
|
||||
return (bytes / Math.pow(1024, 2)).toFixed(digits) + 'MB';
|
||||
else if (bytes >= 1024)
|
||||
return (bytes / 1024).toFixed(digits) + 'KB';
|
||||
else
|
||||
return bytes;
|
||||
}
|
||||
|
||||
class MergeTreeVisualizer {
|
||||
getMargin() { return { left: 50, right: 40, top: 60, bottom: 60 }; }
|
||||
|
||||
@ -50,15 +79,8 @@ class MergeTreeVisualizer {
|
||||
this.max_source_part_count = d3.max(mt.parts, d => d.source_part_count);
|
||||
|
||||
// Compute scale ranges
|
||||
this.minXValue = d3.min(mt.parts, d => this.getLeft(d));
|
||||
this.maxXValue = d3.max(mt.parts, d => this.getRight(d));
|
||||
if (this.isYAxisReversed()) {
|
||||
this.minYValue = d3.min(mt.parts, d => this.getTop(d));
|
||||
this.maxYValue = d3.max(mt.parts, d => this.getBottom(d));
|
||||
} else {
|
||||
this.minYValue = d3.min(mt.parts, d => this.getBottom(d));
|
||||
this.maxYValue = d3.max(mt.parts, d => this.getTop(d));
|
||||
}
|
||||
this.computeXAggregates(mt);
|
||||
this.computeYAggregates(mt);
|
||||
|
||||
// Create the SVG container
|
||||
this.svgContainer = container
|
||||
@ -67,11 +89,28 @@ class MergeTreeVisualizer {
|
||||
.attr("height", this.svgHeight);
|
||||
}
|
||||
|
||||
computeXAggregates(mt) {
|
||||
this.minXValue = d3.min(mt.parts, d => this.getLeft(d));
|
||||
this.maxXValue = d3.max(mt.parts, d => this.getRight(d));
|
||||
}
|
||||
|
||||
computeYAggregates(mt) {
|
||||
if (this.isYAxisReversed()) {
|
||||
this.minYValue = d3.min(mt.parts, d => this.getTop(d));
|
||||
this.maxYValue = d3.max(mt.parts, d => this.getBottom(d));
|
||||
} else {
|
||||
this.minYValue = d3.min(mt.parts, d => this.getBottom(d));
|
||||
this.maxYValue = d3.max(mt.parts, d => this.getTop(d));
|
||||
}
|
||||
}
|
||||
|
||||
initXScaleLinear() {
|
||||
// Set up the horizontal scale (x-axis) — linear scale
|
||||
this.xScale = d3.scaleLinear()
|
||||
.domain([this.minXValue, this.maxXValue])
|
||||
.range([this.margin.left, this.svgWidth - this.margin.right]);
|
||||
|
||||
this.updateXDomain = () => this.xScale.domain([this.minXValue, this.maxXValue]);
|
||||
this.updateXDomain();
|
||||
}
|
||||
|
||||
getYRange() {
|
||||
@ -83,25 +122,33 @@ class MergeTreeVisualizer {
|
||||
// Set up the vertical scale (y-axis) — logarithmic scale
|
||||
this.yScale = d3.scaleLog()
|
||||
.base(2)
|
||||
.domain([Math.max(1, this.minYValue), Math.pow(2, Math.ceil(Math.log2(this.maxYValue)))])
|
||||
.range(this.getYRange());
|
||||
|
||||
this.updateYDomain = () => this.yScale.domain([Math.max(1, this.minYValue), Math.pow(2, Math.ceil(Math.log2(this.maxYValue)))])
|
||||
this.updateYDomain();
|
||||
}
|
||||
|
||||
initYScaleLinear() {
|
||||
// Set up the vertical scale (y-axis) — linear scale
|
||||
this.yScale = d3.scaleLinear()
|
||||
.domain([this.minYValue, this.maxYValue])
|
||||
.range(this.getYRange());
|
||||
|
||||
this.updateYDomain = () => this.yScale.domain([this.minYValue, this.maxYValue])
|
||||
this.updateYDomain();
|
||||
}
|
||||
|
||||
createXAxisLinear() {
|
||||
this.xAxis = this.isYAxisReversed() ? d3.axisTop(this.xScale) : d3.axisBottom(this.xScale);
|
||||
this.xAxisFormatter = formatBytes; // formatBytesWithUnit;
|
||||
this.xAxisType = this.isYAxisReversed() ? d3.axisTop : d3.axisBottom;
|
||||
|
||||
const tickStep = determineTickStep(this.maxXValue);
|
||||
const tickStep = determineTickStep(this.maxXValue - this.minXValue);
|
||||
const translateY = this.isYAxisReversed() ? this.margin.top : this.svgHeight - this.margin.bottom;
|
||||
this.svgContainer.append("g")
|
||||
this.xAxis = this.xAxisType(this.xScale)
|
||||
.tickValues(d3.range(this.minXValue, this.maxXValue, tickStep))
|
||||
.tickFormat(this.xAxisFormatter);
|
||||
this.xAxisGroup = this.svgContainer.append("g")
|
||||
.attr("transform", `translate(0, ${translateY})`)
|
||||
.call(this.xAxis.tickValues(d3.range(0, this.maxXValue, tickStep)).tickFormat(formatBytesWithUnit));
|
||||
.call(this.xAxis);
|
||||
|
||||
// Add axis title
|
||||
this.svgContainer.append("text")
|
||||
@ -113,11 +160,14 @@ class MergeTreeVisualizer {
|
||||
}
|
||||
|
||||
createYAxisPowersOfTwo() {
|
||||
this.yAxisFormatter = formatBytes; // d => `2^${Math.log2(d)}`;
|
||||
this.yAxisType = d3.axisLeft;
|
||||
|
||||
const powersOfTwo = Array.from({ length: 50 }, (v, i) => Math.pow(2, i + 1));
|
||||
this.yAxis = d3.axisLeft(this.yScale)
|
||||
this.yAxis = this.yAxisType(this.yScale)
|
||||
.tickValues(powersOfTwo.filter(d => d >= this.minYValue && d <= this.maxYValue))
|
||||
.tickFormat(d => `2^${Math.log2(d)}`);
|
||||
const yAxisGroup = this.svgContainer.append("g")
|
||||
.tickFormat(this.yAxisFormatter);
|
||||
this.yAxisGroup = this.svgContainer.append("g")
|
||||
.attr("transform", `translate(${this.margin.left}, 0)`)
|
||||
.call(this.yAxis);
|
||||
|
||||
@ -130,23 +180,26 @@ class MergeTreeVisualizer {
|
||||
.attr("font-size", "14px")
|
||||
.text("Log(PartSize)");
|
||||
|
||||
yAxisGroup.selectAll(".tick text")
|
||||
.each(function(d) {
|
||||
const exponent = Math.log2(d);
|
||||
const self = d3.select(this);
|
||||
self.text("");
|
||||
self.append("tspan").text("2");
|
||||
self.append("tspan")
|
||||
.attr("dy", "-0.7em")
|
||||
.attr("font-size", "70%").text(exponent);
|
||||
});
|
||||
// This does not work with updating, switched to formatBytes
|
||||
// this.yAxisGroup.selectAll(".tick text")
|
||||
// .each(function(d) {
|
||||
// const exponent = Math.log2(d);
|
||||
// const self = d3.select(this);
|
||||
// self.text("");
|
||||
// self.append("tspan").text("2");
|
||||
// self.append("tspan")
|
||||
// .attr("dy", "-0.7em")
|
||||
// .attr("font-size", "70%").text(exponent);
|
||||
// });
|
||||
}
|
||||
|
||||
createYAxisLinear() {
|
||||
this.yAxis = d3.axisLeft(this.yScale)
|
||||
this.yAxisFormatter = formatNumber;
|
||||
this.yAxisType = d3.axisLeft;
|
||||
this.yAxis = this.yAxisType(this.yScale)
|
||||
.tickArguments([5])
|
||||
.tickFormat(d => Number.isInteger(d) ? d : "");
|
||||
this.svgContainer.append("g")
|
||||
.tickFormat(this.yAxisFormatter);
|
||||
this.yAxisGroup = this.svgContainer.append("g")
|
||||
.attr("transform", `translate(${this.margin.left}, 0)`)
|
||||
.call(this.yAxis);
|
||||
|
||||
@ -160,6 +213,26 @@ class MergeTreeVisualizer {
|
||||
.text("Source parts count");
|
||||
}
|
||||
|
||||
updateX(mt) {
|
||||
// Rescale axis
|
||||
this.computeXAggregates(mt);
|
||||
this.updateXDomain();
|
||||
|
||||
// Update axes with transitions
|
||||
this.xAxisGroup.transition() // .duration(1000)
|
||||
.call(this.xAxisType(this.xScale).tickFormat(this.xAxisFormatter));
|
||||
};
|
||||
|
||||
updateY(mt) {
|
||||
// Rescale axis
|
||||
this.computeYAggregates(mt);
|
||||
this.updateYDomain();
|
||||
|
||||
// Update axes with transitions
|
||||
this.yAxisGroup.transition() // .duration(1000)
|
||||
.call(this.yAxisType(this.yScale).tickFormat(this.yAxisFormatter));
|
||||
};
|
||||
|
||||
createDescription(text, x = 10, y = 60) {
|
||||
this.svgContainer.node().__tippy = infoButton(this.svgContainer, x, y, text);
|
||||
}
|
||||
@ -169,44 +242,90 @@ class MergeTreeVisualizer {
|
||||
pxt(value) { return value; }
|
||||
pxb(value) { return Math.max(1, value); }
|
||||
|
||||
createMerges(mt) {
|
||||
// Append rectangles for merges
|
||||
this.svgContainer.append("g").attr("class", "viz-merge").selectAll("rect")
|
||||
.data(mt.parts)
|
||||
.enter()
|
||||
.filter(d => !d.active)
|
||||
.append("rect")
|
||||
processMerges(mt) {
|
||||
// Check if the merges group already exists, create it if not, and save the reference
|
||||
if (!this.mergesGroup) {
|
||||
this.mergesGroup = this.svgContainer.append("g").attr("class", "viz-merge");
|
||||
}
|
||||
|
||||
// Filter inactive parts and join the data to the rectangles
|
||||
const inactiveParts = mt.parts.filter(d => !d.active);
|
||||
const merges = this.mergesGroup.selectAll("rect")
|
||||
.data(inactiveParts, d => d.id); // Assuming each part has a unique 'id'
|
||||
|
||||
// Handle the enter phase for new elements
|
||||
const mergesEnter = merges.enter().append("rect");
|
||||
|
||||
// Merge the enter and update selections, and set attributes for both
|
||||
mergesEnter.merge(merges)
|
||||
.attr("x", d => this.pxl(this.getMergeLeft(d)))
|
||||
.attr("y", d => this.pxt(this.getMergeTop(d)))
|
||||
.attr("width", d => this.pxr(this.getMergeRight(d) - this.getMergeLeft(d)))
|
||||
.attr("height", d => this.pxb(this.getMergeBottom(d) - this.getMergeTop(d)))
|
||||
.attr("fill", d => this.getMergeColor(d));
|
||||
|
||||
// Handle the exit phase for removed elements
|
||||
merges.exit().remove();
|
||||
}
|
||||
|
||||
createParts(mt) {
|
||||
// Append rectangles for parts
|
||||
this.svgContainer.append("g").attr("class", "viz-part").selectAll("rect")
|
||||
.data(mt.parts)
|
||||
.enter()
|
||||
.append("rect")
|
||||
processParts(mt) {
|
||||
// Check if the parts group already exists, create it if not, and save the reference
|
||||
if (!this.partsGroup) {
|
||||
this.partsGroup = this.svgContainer.append("g").attr("class", "viz-part");
|
||||
}
|
||||
|
||||
// Join the data to the rectangles
|
||||
const parts = this.partsGroup.selectAll("rect")
|
||||
.data(mt.parts, d => d.id); // Assuming each part has a unique 'id'
|
||||
|
||||
// Handle the enter phase for new elements
|
||||
const partsEnter = parts.enter()
|
||||
.append("rect");
|
||||
|
||||
// Merge the enter and update selections, and set attributes for both
|
||||
partsEnter.merge(parts)
|
||||
.attr("x", d => this.pxl(this.getPartLeft(d)))
|
||||
.attr("y", d => this.pxt(this.getPartTop(d)))
|
||||
.attr("width", d => this.pxr(this.getPartRight(d) - this.getPartLeft(d)))
|
||||
.attr("height", d => this.pxb(this.getPartBottom(d) - this.getPartTop(d)))
|
||||
.attr("fill", d => this.getPartColor(d));
|
||||
|
||||
// Handle the exit phase for removed elements
|
||||
parts.exit().remove();
|
||||
}
|
||||
|
||||
createPartMarks(mt) {
|
||||
// Append marks for parts begin
|
||||
this.svgContainer.append("g").attr("class", "viz-part-mark").selectAll("rect")
|
||||
.data(mt.parts)
|
||||
.enter()
|
||||
.append("rect")
|
||||
processPartMarks(mt) {
|
||||
// Check if the part marks group already exists, create it if not, and save the reference
|
||||
if (!this.partMarksGroup) {
|
||||
this.partMarksGroup = this.svgContainer.append("g").attr("class", "viz-part-mark");
|
||||
}
|
||||
|
||||
// Join the data to the rectangles
|
||||
const partMarks = this.partMarksGroup.selectAll("rect")
|
||||
.data(mt.parts, d => d.id); // Assuming each part has a unique 'id'
|
||||
|
||||
// Handle the enter phase for new elements
|
||||
const partMarksEnter = partMarks.enter().append("rect");
|
||||
|
||||
// Merge the enter and update selections, and set attributes for both
|
||||
partMarksEnter.merge(partMarks)
|
||||
.attr("x", d => this.pxl(this.getPartLeft(d)))
|
||||
.attr("y", d => this.pxt(this.getPartTop(d)))
|
||||
.attr("width", d => this.pxr(Math.min(this.part_mark_width, this.getPartRight(d) - this.getPartLeft(d))))
|
||||
.attr("height", d => this.pxb(this.getPartBottom(d) - this.getPartTop(d)))
|
||||
.attr("fill", d => this.getPartMarkColor(d));
|
||||
|
||||
// Handle the exit phase for removed elements
|
||||
partMarks.exit().remove();
|
||||
}
|
||||
|
||||
update(mt) {
|
||||
this.updateX(mt);
|
||||
this.updateY(mt);
|
||||
|
||||
this.processMerges(mt);
|
||||
this.processParts(mt);
|
||||
this.processPartMarks(mt);
|
||||
}
|
||||
}
|
||||
|
||||
@ -236,9 +355,9 @@ class MergeTreeUtilityVisualizer extends MergeTreeVisualizer {
|
||||
this.initXScaleLinear();
|
||||
this.initYScalePowersOfTwo();
|
||||
|
||||
this.createMerges(mt);
|
||||
this.createParts(mt);
|
||||
this.createPartMarks(mt);
|
||||
this.processMerges(mt);
|
||||
this.processParts(mt);
|
||||
this.processPartMarks(mt);
|
||||
|
||||
this.createXAxisLinear();
|
||||
this.createYAxisPowersOfTwo();
|
||||
@ -322,9 +441,9 @@ class MergeTreeTimeVisualizer extends MergeTreeVisualizer {
|
||||
this.initXScaleLinear();
|
||||
this.initYScaleLinear();
|
||||
|
||||
this.createMerges(mt);
|
||||
this.createParts(mt);
|
||||
this.createPartMarks(mt);
|
||||
this.processMerges(mt);
|
||||
this.processParts(mt);
|
||||
this.processPartMarks(mt);
|
||||
|
||||
this.createXAxisLinear();
|
||||
this.createYAxisLinear();
|
||||
|
||||
@ -18,7 +18,7 @@ export class WorkerPool
|
||||
// Method to schedule a task
|
||||
schedule(duration, name, callback)
|
||||
{
|
||||
const task = new Event(name, (sim, event) => this.#finishTask(sim, event, callback));
|
||||
const task = new Event(name, async (sim, event) => await this.#finishTask(sim, event, callback));
|
||||
task.duration = duration;
|
||||
|
||||
if (this.available_workers > 0) // If a worker is available, start the task immediately
|
||||
@ -36,11 +36,11 @@ export class WorkerPool
|
||||
this.sim.scheduleEventAt(this.sim.time + task.duration, task);
|
||||
}
|
||||
|
||||
#finishTask(sim, event, callback)
|
||||
async #finishTask(sim, event, callback)
|
||||
{
|
||||
//console.log(`Ending task with duration ${task.duration} at time ${this.sim.time}`);
|
||||
this.available_workers++;
|
||||
callback(sim, event);
|
||||
await callback(sim, event);
|
||||
while (this.available_workers > 0 && this.queue.length > 0)
|
||||
this.#beginTask(this.queue.shift());
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
<div class="col-md-6" id="exec-container"></div>
|
||||
<div class="col-md-6" id="time-container"></div>
|
||||
<div class="col-md-6" id="util-container"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -63,12 +63,12 @@ export async function testMergeTreeMerger()
|
||||
.call(function(row) {
|
||||
row.append("div")
|
||||
.attr("class", "col-md-6")
|
||||
.attr("id", "merge-tree-merge-exec-container");
|
||||
.attr("id", "merge-tree-merge-time-container");
|
||||
row.append("div")
|
||||
.attr("class", "col-md-6")
|
||||
.attr("id", "merge-tree-merge-util-container");
|
||||
});
|
||||
|
||||
new MergeTreeUtilityVisualizer(mt, d3.select("#merge-tree-merge-util-container"));
|
||||
new MergeTreeTimeVisualizer(mt, d3.select("#merge-tree-merge-exec-container"));
|
||||
new MergeTreeTimeVisualizer(mt, d3.select("#merge-tree-merge-time-container"));
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user