diff --git a/utils/merge-selector-lab/MergeTree.js b/utils/merge-selector-lab/MergeTree.js index 9ecea45a55e..c2b8f63c01b 100644 --- a/utils/merge-selector-lab/MergeTree.js +++ b/utils/merge-selector-lab/MergeTree.js @@ -7,7 +7,8 @@ export class MergeTree { // Metrics this.time = 0; - this.inserted_parts_count = 0; + this.total_part_count = 0; + this.inserted_part_count = 0; this.inserted_bytes = 0; this.written_bytes = 0; // inserts + merges this.inserted_utility = 0; // utility = size * log(size) @@ -49,14 +50,15 @@ export class MergeTree { let log_bytes = Math.log2(bytes); let utility = bytes * log_bytes; let result = { + id: this.total_part_count, bytes, log_bytes, utility, entropy: 0, created: now, level: 0, - begin: this.inserted_parts_count, - end: this.inserted_parts_count + 1, + begin: this.inserted_part_count, + end: this.inserted_part_count + 1, left_bytes: this.inserted_bytes, right_bytes: this.inserted_bytes + bytes, is_leftmost: false, @@ -68,7 +70,8 @@ export class MergeTree { }; this.parts.push(result); this.active_part_count++; - this.inserted_parts_count++; + this.inserted_part_count++; + this.total_part_count++; this.inserted_bytes += bytes; this.written_bytes += bytes; this.inserted_utility += utility; @@ -123,6 +126,7 @@ export class MergeTree { const entropy = (utility - utility0) / bytes; let result = { + id: this.total_part_count, bytes, log_bytes, utility, @@ -140,6 +144,7 @@ export class MergeTree { }; this.parts.push(result); this.active_part_count++; + this.total_part_count++; for (let p of parts_to_merge) { if (p.active == false) diff --git a/utils/merge-selector-lab/MergeTreeInserter.js b/utils/merge-selector-lab/MergeTreeInserter.js index 9e5babdb8f6..f3de01ac7e2 100644 --- a/utils/merge-selector-lab/MergeTreeInserter.js +++ b/utils/merge-selector-lab/MergeTreeInserter.js @@ -26,7 +26,7 @@ export class MergeTreeInserter case 'insert': const part = this.mt.insertPart(value.bytes); if (this.signals.on_insert) - this.signals.on_insert({sim: this.sim, mt: this.mt, part}); + await this.signals.on_insert({sim: this.sim, mt: this.mt, part}); break; case 'sleep': if (value.delay > 0) diff --git a/utils/merge-selector-lab/MergeTreeMerger.js b/utils/merge-selector-lab/MergeTreeMerger.js index ef88ce8fb3c..61de0e4cb8c 100644 --- a/utils/merge-selector-lab/MergeTreeMerger.js +++ b/utils/merge-selector-lab/MergeTreeMerger.js @@ -38,7 +38,7 @@ export class MergeTreeMerger const {parts_to_merge} = value; this.#beginMerge(parts_to_merge); if (this.signals.on_merge_begin) - this.signals.on_merge_begin({sim: this.sim, mt: this.mt, parts_to_merge}); + await this.signals.on_merge_begin({sim: this.sim, mt: this.mt, parts_to_merge}); break; case 'wait': if (this.merges_running == 0) @@ -64,16 +64,16 @@ export class MergeTreeMerger const bytes = parts_to_merge.reduce((sum, d) => sum + d.bytes, 0); const mergeDuration = this.mt.mergeDuration(bytes, parts_to_merge.length); - // Create a Promise that will be resolved on merge finish - return this.pool.schedule(mergeDuration, "MergeEnd", (sim, event) => this.#onMergeEnd(parts_to_merge)); + // Schedule merge finish event + return this.pool.schedule(mergeDuration, "MergeEnd", async (sim, event) => await this.#onMergeEnd(parts_to_merge)); } - #onMergeEnd(parts_to_merge) + async #onMergeEnd(parts_to_merge) { this.mt.advanceTime(this.sim.time); let part = this.mt.finishMergeParts(parts_to_merge); if (this.signals.on_merge_end) - this.signals.on_merge_end({sim: this.sim, mt: this.mt, part, parts_to_merge}); + await this.signals.on_merge_end({sim: this.sim, mt: this.mt, part, parts_to_merge}); this.merges_running--; this.#iterateSelector(); } diff --git a/utils/merge-selector-lab/MergeTreeVisualizer.js b/utils/merge-selector-lab/MergeTreeVisualizer.js index 1ac839630fd..aa83686c5b4 100644 --- a/utils/merge-selector-lab/MergeTreeVisualizer.js +++ b/utils/merge-selector-lab/MergeTreeVisualizer.js @@ -1,6 +1,35 @@ import { valueToColor, formatBytesWithUnit, determineTickStep } from './visualizeHelpers.js'; import { infoButton } from './infoButton.js'; +function formatNumber(number) +{ + if (number >= Math.pow(1000, 4)) + return (number / Math.pow(1000, 4)).toFixed(1) + 'T'; + else if (number >= Math.pow(1000, 3)) + return (number / Math.pow(1000, 3)).toFixed(1) + 'G'; + else if (number >= Math.pow(1000, 2)) + return (number / Math.pow(1000, 2)).toFixed(1) + 'M'; + else if (number >= 1000) + return (number / 1000).toFixed(1) + 'K'; + else + return number; +} + +function formatBytes(bytes) +{ + const digits = 0; + if (bytes >= Math.pow(1024, 4)) + return (bytes / Math.pow(1024, 4)).toFixed(digits) + 'TB'; + else if (bytes >= Math.pow(1024, 3)) + return (bytes / Math.pow(1024, 3)).toFixed(digits) + 'GB'; + else if (bytes >= Math.pow(1024, 2)) + return (bytes / Math.pow(1024, 2)).toFixed(digits) + 'MB'; + else if (bytes >= 1024) + return (bytes / 1024).toFixed(digits) + 'KB'; + else + return bytes; +} + class MergeTreeVisualizer { getMargin() { return { left: 50, right: 40, top: 60, bottom: 60 }; } @@ -50,15 +79,8 @@ class MergeTreeVisualizer { this.max_source_part_count = d3.max(mt.parts, d => d.source_part_count); // Compute scale ranges - this.minXValue = d3.min(mt.parts, d => this.getLeft(d)); - this.maxXValue = d3.max(mt.parts, d => this.getRight(d)); - if (this.isYAxisReversed()) { - this.minYValue = d3.min(mt.parts, d => this.getTop(d)); - this.maxYValue = d3.max(mt.parts, d => this.getBottom(d)); - } else { - this.minYValue = d3.min(mt.parts, d => this.getBottom(d)); - this.maxYValue = d3.max(mt.parts, d => this.getTop(d)); - } + this.computeXAggregates(mt); + this.computeYAggregates(mt); // Create the SVG container this.svgContainer = container @@ -67,11 +89,28 @@ class MergeTreeVisualizer { .attr("height", this.svgHeight); } + computeXAggregates(mt) { + this.minXValue = d3.min(mt.parts, d => this.getLeft(d)); + this.maxXValue = d3.max(mt.parts, d => this.getRight(d)); + } + + computeYAggregates(mt) { + if (this.isYAxisReversed()) { + this.minYValue = d3.min(mt.parts, d => this.getTop(d)); + this.maxYValue = d3.max(mt.parts, d => this.getBottom(d)); + } else { + this.minYValue = d3.min(mt.parts, d => this.getBottom(d)); + this.maxYValue = d3.max(mt.parts, d => this.getTop(d)); + } + } + initXScaleLinear() { // Set up the horizontal scale (x-axis) — linear scale this.xScale = d3.scaleLinear() - .domain([this.minXValue, this.maxXValue]) .range([this.margin.left, this.svgWidth - this.margin.right]); + + this.updateXDomain = () => this.xScale.domain([this.minXValue, this.maxXValue]); + this.updateXDomain(); } getYRange() { @@ -83,25 +122,33 @@ class MergeTreeVisualizer { // Set up the vertical scale (y-axis) — logarithmic scale this.yScale = d3.scaleLog() .base(2) - .domain([Math.max(1, this.minYValue), Math.pow(2, Math.ceil(Math.log2(this.maxYValue)))]) .range(this.getYRange()); + + this.updateYDomain = () => this.yScale.domain([Math.max(1, this.minYValue), Math.pow(2, Math.ceil(Math.log2(this.maxYValue)))]) + this.updateYDomain(); } initYScaleLinear() { // Set up the vertical scale (y-axis) — linear scale this.yScale = d3.scaleLinear() - .domain([this.minYValue, this.maxYValue]) .range(this.getYRange()); + + this.updateYDomain = () => this.yScale.domain([this.minYValue, this.maxYValue]) + this.updateYDomain(); } createXAxisLinear() { - this.xAxis = this.isYAxisReversed() ? d3.axisTop(this.xScale) : d3.axisBottom(this.xScale); + this.xAxisFormatter = formatBytes; // formatBytesWithUnit; + this.xAxisType = this.isYAxisReversed() ? d3.axisTop : d3.axisBottom; - const tickStep = determineTickStep(this.maxXValue); + const tickStep = determineTickStep(this.maxXValue - this.minXValue); const translateY = this.isYAxisReversed() ? this.margin.top : this.svgHeight - this.margin.bottom; - this.svgContainer.append("g") + this.xAxis = this.xAxisType(this.xScale) + .tickValues(d3.range(this.minXValue, this.maxXValue, tickStep)) + .tickFormat(this.xAxisFormatter); + this.xAxisGroup = this.svgContainer.append("g") .attr("transform", `translate(0, ${translateY})`) - .call(this.xAxis.tickValues(d3.range(0, this.maxXValue, tickStep)).tickFormat(formatBytesWithUnit)); + .call(this.xAxis); // Add axis title this.svgContainer.append("text") @@ -113,11 +160,14 @@ class MergeTreeVisualizer { } createYAxisPowersOfTwo() { + this.yAxisFormatter = formatBytes; // d => `2^${Math.log2(d)}`; + this.yAxisType = d3.axisLeft; + const powersOfTwo = Array.from({ length: 50 }, (v, i) => Math.pow(2, i + 1)); - this.yAxis = d3.axisLeft(this.yScale) + this.yAxis = this.yAxisType(this.yScale) .tickValues(powersOfTwo.filter(d => d >= this.minYValue && d <= this.maxYValue)) - .tickFormat(d => `2^${Math.log2(d)}`); - const yAxisGroup = this.svgContainer.append("g") + .tickFormat(this.yAxisFormatter); + this.yAxisGroup = this.svgContainer.append("g") .attr("transform", `translate(${this.margin.left}, 0)`) .call(this.yAxis); @@ -130,23 +180,26 @@ class MergeTreeVisualizer { .attr("font-size", "14px") .text("Log(PartSize)"); - yAxisGroup.selectAll(".tick text") - .each(function(d) { - const exponent = Math.log2(d); - const self = d3.select(this); - self.text(""); - self.append("tspan").text("2"); - self.append("tspan") - .attr("dy", "-0.7em") - .attr("font-size", "70%").text(exponent); - }); + // This does not work with updating, switched to formatBytes + // this.yAxisGroup.selectAll(".tick text") + // .each(function(d) { + // const exponent = Math.log2(d); + // const self = d3.select(this); + // self.text(""); + // self.append("tspan").text("2"); + // self.append("tspan") + // .attr("dy", "-0.7em") + // .attr("font-size", "70%").text(exponent); + // }); } createYAxisLinear() { - this.yAxis = d3.axisLeft(this.yScale) + this.yAxisFormatter = formatNumber; + this.yAxisType = d3.axisLeft; + this.yAxis = this.yAxisType(this.yScale) .tickArguments([5]) - .tickFormat(d => Number.isInteger(d) ? d : ""); - this.svgContainer.append("g") + .tickFormat(this.yAxisFormatter); + this.yAxisGroup = this.svgContainer.append("g") .attr("transform", `translate(${this.margin.left}, 0)`) .call(this.yAxis); @@ -160,6 +213,26 @@ class MergeTreeVisualizer { .text("Source parts count"); } + updateX(mt) { + // Rescale axis + this.computeXAggregates(mt); + this.updateXDomain(); + + // Update axes with transitions + this.xAxisGroup.transition() // .duration(1000) + .call(this.xAxisType(this.xScale).tickFormat(this.xAxisFormatter)); + }; + + updateY(mt) { + // Rescale axis + this.computeYAggregates(mt); + this.updateYDomain(); + + // Update axes with transitions + this.yAxisGroup.transition() // .duration(1000) + .call(this.yAxisType(this.yScale).tickFormat(this.yAxisFormatter)); + }; + createDescription(text, x = 10, y = 60) { this.svgContainer.node().__tippy = infoButton(this.svgContainer, x, y, text); } @@ -169,44 +242,90 @@ class MergeTreeVisualizer { pxt(value) { return value; } pxb(value) { return Math.max(1, value); } - createMerges(mt) { - // Append rectangles for merges - this.svgContainer.append("g").attr("class", "viz-merge").selectAll("rect") - .data(mt.parts) - .enter() - .filter(d => !d.active) - .append("rect") + processMerges(mt) { + // Check if the merges group already exists, create it if not, and save the reference + if (!this.mergesGroup) { + this.mergesGroup = this.svgContainer.append("g").attr("class", "viz-merge"); + } + + // Filter inactive parts and join the data to the rectangles + const inactiveParts = mt.parts.filter(d => !d.active); + const merges = this.mergesGroup.selectAll("rect") + .data(inactiveParts, d => d.id); // Assuming each part has a unique 'id' + + // Handle the enter phase for new elements + const mergesEnter = merges.enter().append("rect"); + + // Merge the enter and update selections, and set attributes for both + mergesEnter.merge(merges) .attr("x", d => this.pxl(this.getMergeLeft(d))) .attr("y", d => this.pxt(this.getMergeTop(d))) .attr("width", d => this.pxr(this.getMergeRight(d) - this.getMergeLeft(d))) .attr("height", d => this.pxb(this.getMergeBottom(d) - this.getMergeTop(d))) .attr("fill", d => this.getMergeColor(d)); + + // Handle the exit phase for removed elements + merges.exit().remove(); } - createParts(mt) { - // Append rectangles for parts - this.svgContainer.append("g").attr("class", "viz-part").selectAll("rect") - .data(mt.parts) - .enter() - .append("rect") + processParts(mt) { + // Check if the parts group already exists, create it if not, and save the reference + if (!this.partsGroup) { + this.partsGroup = this.svgContainer.append("g").attr("class", "viz-part"); + } + + // Join the data to the rectangles + const parts = this.partsGroup.selectAll("rect") + .data(mt.parts, d => d.id); // Assuming each part has a unique 'id' + + // Handle the enter phase for new elements + const partsEnter = parts.enter() + .append("rect"); + + // Merge the enter and update selections, and set attributes for both + partsEnter.merge(parts) .attr("x", d => this.pxl(this.getPartLeft(d))) .attr("y", d => this.pxt(this.getPartTop(d))) .attr("width", d => this.pxr(this.getPartRight(d) - this.getPartLeft(d))) .attr("height", d => this.pxb(this.getPartBottom(d) - this.getPartTop(d))) .attr("fill", d => this.getPartColor(d)); + + // Handle the exit phase for removed elements + parts.exit().remove(); } - createPartMarks(mt) { - // Append marks for parts begin - this.svgContainer.append("g").attr("class", "viz-part-mark").selectAll("rect") - .data(mt.parts) - .enter() - .append("rect") + processPartMarks(mt) { + // Check if the part marks group already exists, create it if not, and save the reference + if (!this.partMarksGroup) { + this.partMarksGroup = this.svgContainer.append("g").attr("class", "viz-part-mark"); + } + + // Join the data to the rectangles + const partMarks = this.partMarksGroup.selectAll("rect") + .data(mt.parts, d => d.id); // Assuming each part has a unique 'id' + + // Handle the enter phase for new elements + const partMarksEnter = partMarks.enter().append("rect"); + + // Merge the enter and update selections, and set attributes for both + partMarksEnter.merge(partMarks) .attr("x", d => this.pxl(this.getPartLeft(d))) .attr("y", d => this.pxt(this.getPartTop(d))) .attr("width", d => this.pxr(Math.min(this.part_mark_width, this.getPartRight(d) - this.getPartLeft(d)))) .attr("height", d => this.pxb(this.getPartBottom(d) - this.getPartTop(d))) .attr("fill", d => this.getPartMarkColor(d)); + + // Handle the exit phase for removed elements + partMarks.exit().remove(); + } + + update(mt) { + this.updateX(mt); + this.updateY(mt); + + this.processMerges(mt); + this.processParts(mt); + this.processPartMarks(mt); } } @@ -236,9 +355,9 @@ class MergeTreeUtilityVisualizer extends MergeTreeVisualizer { this.initXScaleLinear(); this.initYScalePowersOfTwo(); - this.createMerges(mt); - this.createParts(mt); - this.createPartMarks(mt); + this.processMerges(mt); + this.processParts(mt); + this.processPartMarks(mt); this.createXAxisLinear(); this.createYAxisPowersOfTwo(); @@ -322,9 +441,9 @@ class MergeTreeTimeVisualizer extends MergeTreeVisualizer { this.initXScaleLinear(); this.initYScaleLinear(); - this.createMerges(mt); - this.createParts(mt); - this.createPartMarks(mt); + this.processMerges(mt); + this.processParts(mt); + this.processPartMarks(mt); this.createXAxisLinear(); this.createYAxisLinear(); diff --git a/utils/merge-selector-lab/WorkerPool.js b/utils/merge-selector-lab/WorkerPool.js index f80e5a36458..1267a3c7b32 100644 --- a/utils/merge-selector-lab/WorkerPool.js +++ b/utils/merge-selector-lab/WorkerPool.js @@ -18,7 +18,7 @@ export class WorkerPool // Method to schedule a task schedule(duration, name, callback) { - const task = new Event(name, (sim, event) => this.#finishTask(sim, event, callback)); + const task = new Event(name, async (sim, event) => await this.#finishTask(sim, event, callback)); task.duration = duration; if (this.available_workers > 0) // If a worker is available, start the task immediately @@ -36,11 +36,11 @@ export class WorkerPool this.sim.scheduleEventAt(this.sim.time + task.duration, task); } - #finishTask(sim, event, callback) + async #finishTask(sim, event, callback) { //console.log(`Ending task with duration ${task.duration} at time ${this.sim.time}`); this.available_workers++; - callback(sim, event); + await callback(sim, event); while (this.available_workers > 0 && this.queue.length > 0) this.#beginTask(this.queue.shift()); } diff --git a/utils/merge-selector-lab/index.html b/utils/merge-selector-lab/index.html index 1a597be934b..20fc32c796a 100644 --- a/utils/merge-selector-lab/index.html +++ b/utils/merge-selector-lab/index.html @@ -16,7 +16,7 @@
-
+
diff --git a/utils/merge-selector-lab/tests/testMergeTreeMerger.js b/utils/merge-selector-lab/tests/testMergeTreeMerger.js index 17b5afbf3d7..c33debf7c3c 100644 --- a/utils/merge-selector-lab/tests/testMergeTreeMerger.js +++ b/utils/merge-selector-lab/tests/testMergeTreeMerger.js @@ -63,12 +63,12 @@ export async function testMergeTreeMerger() .call(function(row) { row.append("div") .attr("class", "col-md-6") - .attr("id", "merge-tree-merge-exec-container"); + .attr("id", "merge-tree-merge-time-container"); row.append("div") .attr("class", "col-md-6") .attr("id", "merge-tree-merge-util-container"); }); new MergeTreeUtilityVisualizer(mt, d3.select("#merge-tree-merge-util-container")); - new MergeTreeTimeVisualizer(mt, d3.select("#merge-tree-merge-exec-container")); + new MergeTreeTimeVisualizer(mt, d3.select("#merge-tree-merge-time-container")); }