dynamic updates of visualizations to avoid OOM

utils/merge-selector-lab/MergeTree.js

@@ -7,7 +7,8 @@ export class MergeTree {
 
         // Metrics
         this.time = 0;
-        this.inserted_parts_count = 0;
+        this.total_part_count = 0;
+        this.inserted_part_count = 0;
         this.inserted_bytes = 0;
         this.written_bytes = 0; // inserts + merges
         this.inserted_utility = 0; // utility = size * log(size)
@@ -49,14 +50,15 @@ export class MergeTree {
         let log_bytes = Math.log2(bytes);
         let utility = bytes * log_bytes;
         let result = {
+            id: this.total_part_count,
             bytes,
             log_bytes,
             utility,
             entropy: 0,
             created: now,
             level: 0,
-            begin: this.inserted_parts_count,
+            begin: this.inserted_part_count,
-            end: this.inserted_parts_count + 1,
+            end: this.inserted_part_count + 1,
             left_bytes: this.inserted_bytes,
             right_bytes: this.inserted_bytes + bytes,
             is_leftmost: false,
@@ -68,7 +70,8 @@ export class MergeTree {
         };
         this.parts.push(result);
         this.active_part_count++;
-        this.inserted_parts_count++;
+        this.inserted_part_count++;
+        this.total_part_count++;
         this.inserted_bytes += bytes;
         this.written_bytes += bytes;
         this.inserted_utility += utility;
@@ -123,6 +126,7 @@ export class MergeTree {
         const entropy = (utility - utility0) / bytes;
 
         let result = {
+            id: this.total_part_count,
             bytes,
             log_bytes,
             utility,
@@ -140,6 +144,7 @@ export class MergeTree {
         };
         this.parts.push(result);
         this.active_part_count++;
+        this.total_part_count++;
         for (let p of parts_to_merge)
         {
             if (p.active == false)

utils/merge-selector-lab/MergeTreeInserter.js

@@ -26,7 +26,7 @@ export class MergeTreeInserter
                 case 'insert':
                     const part = this.mt.insertPart(value.bytes);
                     if (this.signals.on_insert)
-                        this.signals.on_insert({sim: this.sim, mt: this.mt, part});
+                        await this.signals.on_insert({sim: this.sim, mt: this.mt, part});
                     break;
                 case 'sleep':
                     if (value.delay > 0)

utils/merge-selector-lab/MergeTreeMerger.js

@@ -38,7 +38,7 @@ export class MergeTreeMerger
                     const {parts_to_merge} = value;
                     this.#beginMerge(parts_to_merge);
                     if (this.signals.on_merge_begin)
-                        this.signals.on_merge_begin({sim: this.sim, mt: this.mt, parts_to_merge});
+                        await this.signals.on_merge_begin({sim: this.sim, mt: this.mt, parts_to_merge});
                     break;
                 case 'wait':
                     if (this.merges_running == 0)
@@ -64,16 +64,16 @@ export class MergeTreeMerger
         const bytes = parts_to_merge.reduce((sum, d) => sum + d.bytes, 0);
         const mergeDuration = this.mt.mergeDuration(bytes, parts_to_merge.length);
 
-        // Create a Promise that will be resolved on merge finish
+        // Schedule merge finish event
-        return this.pool.schedule(mergeDuration, "MergeEnd", (sim, event) => this.#onMergeEnd(parts_to_merge));
+        return this.pool.schedule(mergeDuration, "MergeEnd", async (sim, event) => await this.#onMergeEnd(parts_to_merge));
     }
 
-    #onMergeEnd(parts_to_merge)
+    async #onMergeEnd(parts_to_merge)
     {
         this.mt.advanceTime(this.sim.time);
         let part = this.mt.finishMergeParts(parts_to_merge);
         if (this.signals.on_merge_end)
-            this.signals.on_merge_end({sim: this.sim, mt: this.mt, part, parts_to_merge});
+            await this.signals.on_merge_end({sim: this.sim, mt: this.mt, part, parts_to_merge});
         this.merges_running--;
         this.#iterateSelector();
     }

utils/merge-selector-lab/MergeTreeVisualizer.js

@@ -1,6 +1,35 @@
 import { valueToColor, formatBytesWithUnit, determineTickStep } from './visualizeHelpers.js';
 import { infoButton } from './infoButton.js';
 
+function formatNumber(number)
+{
+    if (number >= Math.pow(1000, 4))
+        return (number / Math.pow(1000, 4)).toFixed(1) + 'T';
+    else if (number >= Math.pow(1000, 3))
+        return (number / Math.pow(1000, 3)).toFixed(1) + 'G';
+    else if (number >= Math.pow(1000, 2))
+        return (number / Math.pow(1000, 2)).toFixed(1) + 'M';
+    else if (number >= 1000)
+        return (number / 1000).toFixed(1) + 'K';
+    else
+        return number;
+}
+
+function formatBytes(bytes)
+{
+    const digits = 0;
+    if (bytes >= Math.pow(1024, 4))
+        return (bytes / Math.pow(1024, 4)).toFixed(digits) + 'TB';
+    else if (bytes >= Math.pow(1024, 3))
+        return (bytes / Math.pow(1024, 3)).toFixed(digits) + 'GB';
+    else if (bytes >= Math.pow(1024, 2))
+        return (bytes / Math.pow(1024, 2)).toFixed(digits) + 'MB';
+    else if (bytes >= 1024)
+        return (bytes / 1024).toFixed(digits) + 'KB';
+    else
+        return bytes;
+}
+
 class MergeTreeVisualizer {
     getMargin() { return { left: 50, right: 40, top: 60, bottom: 60 }; }
 
@@ -50,15 +79,8 @@ class MergeTreeVisualizer {
         this.max_source_part_count = d3.max(mt.parts, d => d.source_part_count);
 
         // Compute scale ranges
-        this.minXValue = d3.min(mt.parts, d => this.getLeft(d));
+        this.computeXAggregates(mt);
-        this.maxXValue = d3.max(mt.parts, d => this.getRight(d));
+        this.computeYAggregates(mt);
-        if (this.isYAxisReversed()) {
-            this.minYValue = d3.min(mt.parts, d => this.getTop(d));
-            this.maxYValue = d3.max(mt.parts, d => this.getBottom(d));
-        } else {
-            this.minYValue = d3.min(mt.parts, d => this.getBottom(d));
-            this.maxYValue = d3.max(mt.parts, d => this.getTop(d));
-        }
 
         // Create the SVG container
         this.svgContainer = container
@@ -67,11 +89,28 @@ class MergeTreeVisualizer {
             .attr("height", this.svgHeight);
     }
 
+    computeXAggregates(mt) {
+        this.minXValue = d3.min(mt.parts, d => this.getLeft(d));
+        this.maxXValue = d3.max(mt.parts, d => this.getRight(d));
+    }
+
+    computeYAggregates(mt) {
+        if (this.isYAxisReversed()) {
+            this.minYValue = d3.min(mt.parts, d => this.getTop(d));
+            this.maxYValue = d3.max(mt.parts, d => this.getBottom(d));
+        } else {
+            this.minYValue = d3.min(mt.parts, d => this.getBottom(d));
+            this.maxYValue = d3.max(mt.parts, d => this.getTop(d));
+        }
+    }
+
     initXScaleLinear() {
         // Set up the horizontal scale (x-axis) — linear scale
         this.xScale = d3.scaleLinear()
-            .domain([this.minXValue, this.maxXValue])
             .range([this.margin.left, this.svgWidth - this.margin.right]);
+
+        this.updateXDomain = () => this.xScale.domain([this.minXValue, this.maxXValue]);
+        this.updateXDomain();
     }
 
     getYRange() {
@@ -83,25 +122,33 @@ class MergeTreeVisualizer {
         // Set up the vertical scale (y-axis) — logarithmic scale
         this.yScale = d3.scaleLog()
             .base(2)
-            .domain([Math.max(1, this.minYValue), Math.pow(2, Math.ceil(Math.log2(this.maxYValue)))])
             .range(this.getYRange());
+
+        this.updateYDomain = () => this.yScale.domain([Math.max(1, this.minYValue), Math.pow(2, Math.ceil(Math.log2(this.maxYValue)))])
+        this.updateYDomain();
     }
 
     initYScaleLinear() {
         // Set up the vertical scale (y-axis) — linear scale
         this.yScale = d3.scaleLinear()
-            .domain([this.minYValue, this.maxYValue])
             .range(this.getYRange());
+
+        this.updateYDomain = () => this.yScale.domain([this.minYValue, this.maxYValue])
+        this.updateYDomain();
     }
 
     createXAxisLinear() {
-        this.xAxis = this.isYAxisReversed() ? d3.axisTop(this.xScale) : d3.axisBottom(this.xScale);
+        this.xAxisFormatter = formatBytes; // formatBytesWithUnit;
+        this.xAxisType = this.isYAxisReversed() ? d3.axisTop : d3.axisBottom;
 
-        const tickStep = determineTickStep(this.maxXValue);
+        const tickStep = determineTickStep(this.maxXValue - this.minXValue);
         const translateY = this.isYAxisReversed() ? this.margin.top : this.svgHeight - this.margin.bottom;
-        this.svgContainer.append("g")
+        this.xAxis = this.xAxisType(this.xScale)
+            .tickValues(d3.range(this.minXValue, this.maxXValue, tickStep))
+            .tickFormat(this.xAxisFormatter);
+        this.xAxisGroup = this.svgContainer.append("g")
             .attr("transform", `translate(0, ${translateY})`)
-            .call(this.xAxis.tickValues(d3.range(0, this.maxXValue, tickStep)).tickFormat(formatBytesWithUnit));
+            .call(this.xAxis);
 
         // Add axis title
         this.svgContainer.append("text")
@@ -113,11 +160,14 @@ class MergeTreeVisualizer {
     }
 
     createYAxisPowersOfTwo() {
+        this.yAxisFormatter = formatBytes; // d => `2^${Math.log2(d)}`;
+        this.yAxisType = d3.axisLeft;
+
         const powersOfTwo = Array.from({ length: 50 }, (v, i) => Math.pow(2, i + 1));
-        this.yAxis = d3.axisLeft(this.yScale)
+        this.yAxis = this.yAxisType(this.yScale)
             .tickValues(powersOfTwo.filter(d => d >= this.minYValue && d <= this.maxYValue))
-            .tickFormat(d => `2^${Math.log2(d)}`);
+            .tickFormat(this.yAxisFormatter);
-        const yAxisGroup = this.svgContainer.append("g")
+        this.yAxisGroup = this.svgContainer.append("g")
             .attr("transform", `translate(${this.margin.left}, 0)`)
             .call(this.yAxis);
 
@@ -130,23 +180,26 @@ class MergeTreeVisualizer {
             .attr("font-size", "14px")
             .text("Log(PartSize)");
 
-        yAxisGroup.selectAll(".tick text")
+        // This does not work with updating, switched to formatBytes
-            .each(function(d) {
+        // this.yAxisGroup.selectAll(".tick text")
-                const exponent = Math.log2(d);
+        //     .each(function(d) {
-                const self = d3.select(this);
+        //         const exponent = Math.log2(d);
-                self.text("");
+        //         const self = d3.select(this);
-                self.append("tspan").text("2");
+        //         self.text("");
-                self.append("tspan")
+        //         self.append("tspan").text("2");
-                    .attr("dy", "-0.7em")
+        //         self.append("tspan")
-                    .attr("font-size", "70%").text(exponent);
+        //             .attr("dy", "-0.7em")
-            });
+        //             .attr("font-size", "70%").text(exponent);
+        //     });
     }
 
     createYAxisLinear() {
-        this.yAxis = d3.axisLeft(this.yScale)
+        this.yAxisFormatter = formatNumber;
+        this.yAxisType = d3.axisLeft;
+        this.yAxis = this.yAxisType(this.yScale)
             .tickArguments([5])
-            .tickFormat(d => Number.isInteger(d) ? d : "");
+            .tickFormat(this.yAxisFormatter);
-        this.svgContainer.append("g")
+        this.yAxisGroup = this.svgContainer.append("g")
             .attr("transform", `translate(${this.margin.left}, 0)`)
             .call(this.yAxis);
 
@@ -160,6 +213,26 @@ class MergeTreeVisualizer {
             .text("Source parts count");
     }
 
+    updateX(mt) {
+        // Rescale axis
+        this.computeXAggregates(mt);
+        this.updateXDomain();
+
+        // Update axes with transitions
+        this.xAxisGroup.transition() // .duration(1000)
+            .call(this.xAxisType(this.xScale).tickFormat(this.xAxisFormatter));
+    };
+
+    updateY(mt) {
+        // Rescale axis
+        this.computeYAggregates(mt);
+        this.updateYDomain();
+
+        // Update axes with transitions
+        this.yAxisGroup.transition() // .duration(1000)
+            .call(this.yAxisType(this.yScale).tickFormat(this.yAxisFormatter));
+    };
+
     createDescription(text, x = 10, y = 60) {
         this.svgContainer.node().__tippy = infoButton(this.svgContainer, x, y, text);
     }
@@ -169,44 +242,90 @@ class MergeTreeVisualizer {
     pxt(value) { return value; }
     pxb(value) { return Math.max(1, value); }
 
-    createMerges(mt) {
+    processMerges(mt) {
-        // Append rectangles for merges
+        // Check if the merges group already exists, create it if not, and save the reference
-        this.svgContainer.append("g").attr("class", "viz-merge").selectAll("rect")
+        if (!this.mergesGroup) {
-            .data(mt.parts)
+            this.mergesGroup = this.svgContainer.append("g").attr("class", "viz-merge");
-            .enter()
+        }
-            .filter(d => !d.active)
+
-            .append("rect")
+        // Filter inactive parts and join the data to the rectangles
+        const inactiveParts = mt.parts.filter(d => !d.active);
+        const merges = this.mergesGroup.selectAll("rect")
+            .data(inactiveParts, d => d.id); // Assuming each part has a unique 'id'
+
+        // Handle the enter phase for new elements
+        const mergesEnter = merges.enter().append("rect");
+
+        // Merge the enter and update selections, and set attributes for both
+        mergesEnter.merge(merges)
             .attr("x", d => this.pxl(this.getMergeLeft(d)))
             .attr("y", d => this.pxt(this.getMergeTop(d)))
             .attr("width", d => this.pxr(this.getMergeRight(d) - this.getMergeLeft(d)))
             .attr("height", d => this.pxb(this.getMergeBottom(d) - this.getMergeTop(d)))
             .attr("fill", d => this.getMergeColor(d));
+
+        // Handle the exit phase for removed elements
+        merges.exit().remove();
     }
 
-    createParts(mt) {
+    processParts(mt) {
-        // Append rectangles for parts
+        // Check if the parts group already exists, create it if not, and save the reference
-        this.svgContainer.append("g").attr("class", "viz-part").selectAll("rect")
+        if (!this.partsGroup) {
-            .data(mt.parts)
+            this.partsGroup = this.svgContainer.append("g").attr("class", "viz-part");
-            .enter()
+        }
-            .append("rect")
+
+        // Join the data to the rectangles
+        const parts = this.partsGroup.selectAll("rect")
+            .data(mt.parts, d => d.id); // Assuming each part has a unique 'id'
+
+        // Handle the enter phase for new elements
+        const partsEnter = parts.enter()
+            .append("rect");
+
+        // Merge the enter and update selections, and set attributes for both
+        partsEnter.merge(parts)
             .attr("x", d => this.pxl(this.getPartLeft(d)))
             .attr("y", d => this.pxt(this.getPartTop(d)))
             .attr("width", d => this.pxr(this.getPartRight(d) - this.getPartLeft(d)))
             .attr("height", d => this.pxb(this.getPartBottom(d) - this.getPartTop(d)))
             .attr("fill", d => this.getPartColor(d));
+
+        // Handle the exit phase for removed elements
+        parts.exit().remove();
     }
 
-    createPartMarks(mt) {
+    processPartMarks(mt) {
-        // Append marks for parts begin
+        // Check if the part marks group already exists, create it if not, and save the reference
-        this.svgContainer.append("g").attr("class", "viz-part-mark").selectAll("rect")
+        if (!this.partMarksGroup) {
-            .data(mt.parts)
+            this.partMarksGroup = this.svgContainer.append("g").attr("class", "viz-part-mark");
-            .enter()
+        }
-            .append("rect")
+
+        // Join the data to the rectangles
+        const partMarks = this.partMarksGroup.selectAll("rect")
+            .data(mt.parts, d => d.id); // Assuming each part has a unique 'id'
+
+        // Handle the enter phase for new elements
+        const partMarksEnter = partMarks.enter().append("rect");
+
+        // Merge the enter and update selections, and set attributes for both
+        partMarksEnter.merge(partMarks)
             .attr("x", d => this.pxl(this.getPartLeft(d)))
             .attr("y", d => this.pxt(this.getPartTop(d)))
             .attr("width", d => this.pxr(Math.min(this.part_mark_width, this.getPartRight(d) - this.getPartLeft(d))))
             .attr("height", d => this.pxb(this.getPartBottom(d) - this.getPartTop(d)))
             .attr("fill", d => this.getPartMarkColor(d));
+
+        // Handle the exit phase for removed elements
+        partMarks.exit().remove();
+    }
+
+    update(mt) {
+        this.updateX(mt);
+        this.updateY(mt);
+
+        this.processMerges(mt);
+        this.processParts(mt);
+        this.processPartMarks(mt);
     }
 }
 
@@ -236,9 +355,9 @@ class MergeTreeUtilityVisualizer extends MergeTreeVisualizer {
         this.initXScaleLinear();
         this.initYScalePowersOfTwo();
 
-        this.createMerges(mt);
+        this.processMerges(mt);
-        this.createParts(mt);
+        this.processParts(mt);
-        this.createPartMarks(mt);
+        this.processPartMarks(mt);
 
         this.createXAxisLinear();
         this.createYAxisPowersOfTwo();
@@ -322,9 +441,9 @@ class MergeTreeTimeVisualizer extends MergeTreeVisualizer {
         this.initXScaleLinear();
         this.initYScaleLinear();
 
-        this.createMerges(mt);
+        this.processMerges(mt);
-        this.createParts(mt);
+        this.processParts(mt);
-        this.createPartMarks(mt);
+        this.processPartMarks(mt);
 
         this.createXAxisLinear();
         this.createYAxisLinear();

utils/merge-selector-lab/WorkerPool.js

@@ -18,7 +18,7 @@ export class WorkerPool
     // Method to schedule a task
     schedule(duration, name, callback)
     {
-        const task = new Event(name, (sim, event) => this.#finishTask(sim, event, callback));
+        const task = new Event(name, async (sim, event) => await this.#finishTask(sim, event, callback));
         task.duration = duration;
 
         if (this.available_workers > 0) // If a worker is available, start the task immediately
@@ -36,11 +36,11 @@ export class WorkerPool
         this.sim.scheduleEventAt(this.sim.time + task.duration, task);
     }
 
-    #finishTask(sim, event, callback)
+    async #finishTask(sim, event, callback)
     {
         //console.log(`Ending task with duration ${task.duration} at time ${this.sim.time}`);
         this.available_workers++;
-        callback(sim, event);
+        await callback(sim, event);
         while (this.available_workers > 0 && this.queue.length > 0)
             this.#beginTask(this.queue.shift());
     }

utils/merge-selector-lab/index.html

@@ -16,7 +16,7 @@
 
         <div class="container-fluid">
             <div class="row">
-                <div class="col-md-6" id="exec-container"></div>
+                <div class="col-md-6" id="time-container"></div>
                 <div class="col-md-6" id="util-container"></div>
             </div>
         </div>

utils/merge-selector-lab/tests/testMergeTreeMerger.js

@@ -63,12 +63,12 @@ export async function testMergeTreeMerger()
         .call(function(row) {
             row.append("div")
                 .attr("class", "col-md-6")
-                .attr("id", "merge-tree-merge-exec-container");
+                .attr("id", "merge-tree-merge-time-container");
             row.append("div")
                 .attr("class", "col-md-6")
                 .attr("id", "merge-tree-merge-util-container");
         });
 
     new MergeTreeUtilityVisualizer(mt, d3.select("#merge-tree-merge-util-container"));
-    new MergeTreeTimeVisualizer(mt, d3.select("#merge-tree-merge-exec-container"));
+    new MergeTreeTimeVisualizer(mt, d3.select("#merge-tree-merge-time-container"));
 }