Refactor Plot benchmarks.
* Faster rendering.
* Flexible mappers for custom series generation.
* Make it easier to visually inspect and find benchmark data sources.
Change-Id: I37634549e0204fff3c6059717d12566d0abdf630
Test: `npm run-script dev`.
diff --git a/development/plot-benchmarks/src/chart-transforms.ts b/development/plot-benchmarks/src/chart-transforms.ts
deleted file mode 100644
index f76049f..0000000
--- a/development/plot-benchmarks/src/chart-transforms.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-import type { ChartDataset } from "chart.js";
-import type { Data } from "./transforms.js";
-
-export function chartData(container: Array<Data>) {
- let xmax = 0;
- let datasets = [];
- for (let i = 0; i < container.length; i += 1) {
- // update xmax
- xmax = Math.max(xmax, container[i].data.length);
- datasets.push(chartDataset(container[i]));
- }
- return {
- labels: xrange(xmax),
- datasets: datasets
- };
-}
-
-function xrange(xmax: number): Array<number> {
- let labels = [];
- for (let i = 1; i <= xmax; i += 1) {
- labels.push(i);
- }
- return labels;
-}
-
-function chartDataset(data: Data): ChartDataset<'line'> {
- return {
- label: data.name,
- data: data.data,
- tension: 0.3
- };
-}
diff --git a/development/plot-benchmarks/src/files.ts b/development/plot-benchmarks/src/files.ts
index a5e26ef..6d45cd9 100644
--- a/development/plot-benchmarks/src/files.ts
+++ b/development/plot-benchmarks/src/files.ts
@@ -1,14 +1,4 @@
-import type { Benchmarks } from "./schema.js";
-
-export interface FileMetadata {
- file: File;
- enabled: boolean;
- benchmarks: Benchmarks;
-}
-
-export interface FileMetadataEvent {
- entries: Array<FileMetadata>;
-}
+import type { Benchmarks } from "./types/benchmark.js";
export async function readBenchmarks(file: File): Promise<Benchmarks> {
const contents = await readFile(file);
diff --git a/development/plot-benchmarks/src/lib/App.svelte b/development/plot-benchmarks/src/lib/App.svelte
index e50908c..ffa3e57 100644
--- a/development/plot-benchmarks/src/lib/App.svelte
+++ b/development/plot-benchmarks/src/lib/App.svelte
@@ -1,23 +1,14 @@
<script lang="ts">
import type { Writable } from "svelte/store";
- import { derived, writable } from "svelte/store";
- import type { FileMetadata } from "../files.js";
- import type { Benchmarks } from "../schema.js";
- import Chart from "./Chart.svelte";
- import Files from "./Files.svelte";
+ import { writable } from "svelte/store";
+ import type { FileMetadata } from "../types/files.js";
+ import Session from "./Session.svelte";
- let entries: Writable<Array<FileMetadata>> = writable([]);
- let containers = derived(entries, ($entries) => {
- const containers: Array<Benchmarks> = [];
- for (let i = 0; i < $entries.length; i += 1) {
- let entry = $entries[i];
- containers.push(entry.benchmarks);
- }
- return containers;
- });
+ // Stores
+ let entries: Writable<FileMetadata[]> = writable([]);
- function onFileMetadataEvent(event) {
- const detail: Array<FileMetadata> = event.detail;
+ function onFilesChanged(event) {
+ const detail: FileMetadata[] = event.detail;
if (detail) {
const enabled = detail.filter((metadata) => metadata.enabled === true);
$entries = [...enabled];
@@ -31,8 +22,5 @@
</details>
<div class="container">
- <Files on:entries={onFileMetadataEvent} />
- {#if $containers.length > 0}
- <Chart {containers} />
- {/if}
+ <Session fileEntries={$entries} on:entries={onFilesChanged} />
</div>
diff --git a/development/plot-benchmarks/src/lib/Chart.svelte b/development/plot-benchmarks/src/lib/Chart.svelte
index 257f228..7a063a8 100644
--- a/development/plot-benchmarks/src/lib/Chart.svelte
+++ b/development/plot-benchmarks/src/lib/Chart.svelte
@@ -1,42 +1,33 @@
<script lang="ts">
- import type { LegendItem } from "chart.js";
+ import type { ChartType, LegendItem } from "chart.js";
import { Chart } from "chart.js/auto";
import { onMount } from "svelte";
- import type { Readable, Writable } from "svelte/store";
- import { derived, writable } from "svelte/store";
- import { chartData } from "../chart-transforms.js";
- import { saveToClipboard } from "../clipboard.js";
+ import { writable, type Writable } from "svelte/store";
+ import type { Data } from "../types/chart.js";
import { LegendPlugin } from "../plugins.js";
- import { expressionFilter } from "../regexp.js";
- import type { Benchmarks } from "../schema.js";
- import { benchmarksDataset } from "../transforms.js";
- type FilterFn = (label: string) => boolean;
+ import Legend from "./Legend.svelte";
+ import { saveToClipboard as save } from "../clipboard.js";
- export let containers: Readable<Array<Benchmarks>>;
+ export let data: Data;
+ export let chartType: ChartType = "line";
- let canvas: HTMLCanvasElement;
- let filter: Writable<FilterFn> = writable((_: string) => true);
-
- let data = derived([containers, filter], ([$containers, $filter]) => {
- return chartData(benchmarksDataset($containers, $filter));
- });
-
- let chart: Writable<Chart | undefined> = writable(null);
- let legendLabels: Writable<Array<LegendItem> | undefined> = writable(null);
-
- data.subscribe(($data) => {
+ $: {
if ($chart) {
- $chart.data = $data;
+ $chart.data = data;
$chart.update();
}
- });
+ }
+ // State
+ let element: HTMLCanvasElement;
+ let chart: Writable<Chart | null> = writable(null);
+ let items: Writable<LegendItem[] | null> = writable(null);
+
+ // Effects
onMount(() => {
- const onUpdate = (updated: Chart) => {
- chart.set(updated);
- legendLabels.set(
- updated.options.plugins.legend.labels.generateLabels(updated)
- );
+ const onUpdate = (chart: Chart) => {
+ $chart = chart;
+ $items = chart.options.plugins.legend.labels.generateLabels(chart);
};
const plugins = {
legend: {
@@ -46,119 +37,56 @@
onUpdate: onUpdate,
},
};
- chart.set(
- new Chart(canvas, {
- type: "line",
- data: $data,
- plugins: [LegendPlugin],
- options: {
- plugins: plugins,
- },
- })
- );
+ $chart = new Chart(element, {
+ data: data,
+ type: chartType,
+ plugins: [LegendPlugin],
+ options: {
+ plugins: plugins,
+ },
+ });
});
- function onItemClick(item: LegendItem) {
- return (_: Event) => {
- // https://www.chartjs.org/docs/latest/samples/legend/html.html
- $chart.setDatasetVisibility(
- item.datasetIndex,
- !$chart.isDatasetVisible(item.datasetIndex)
- );
- // Update chart
- $chart.update();
- };
- }
-
- function onFilterChanged(event: Event) {
- const target = event.currentTarget as HTMLInputElement;
- const value = target.value;
- $filter = expressionFilter(value);
- }
-
- async function onCopy(_: Event) {
+ // Copy to clip board
+ async function copy(event: Event) {
if ($chart) {
- await saveToClipboard($chart);
+ await save($chart);
}
}
</script>
<article>
- <button
- class="copy outline"
- data-tooltip="Copy chart to clipboard."
- on:click={onCopy}
- >
- ⎘
- </button>
- <canvas id="chart" class="chart" bind:this={canvas} />
+ <div class="toolbar">
+ <button
+ class="btn outline"
+ data-tooltip="Copy chart to clipboard."
+ on:click={copy}
+ >
+ ⎘
+ </button>
+ </div>
+ <canvas id="chart" class="chart" bind:this={element} />
</article>
-{#if $legendLabels && $legendLabels.length >= 0}
- <article>
- <div class="filter">
- <label for="metricFilter">
- <input
- type="text"
- id="metricFilter"
- name="metricFilter"
- placeholder="Filter metrics (regular expressions)"
- autocomplete="off"
- on:input={onFilterChanged}
- />
- </label>
- </div>
- <div class="legend">
- {#each $legendLabels as label, index}
- <div
- class="item"
- on:dblclick={onItemClick(label)}
- aria-label="legend"
- role="listitem"
- >
- <span
- class="box"
- style="background: {label.fillStyle}; border-color: {label.strokeStyle}; border-width: {label.lineWidth}px;"
- />
- <span
- class="label"
- style="text-decoration: {label.hidden ? 'line-through' : ''};"
- >
- {label.text}
- </span>
- </div>
- {/each}
- </div>
- </article>
+{#if $items}
+ <Legend chart={$chart} items={$items} />
{/if}
<style>
- .copy {
- width: 4%;
- position: relative;
- padding: 0;
- border: none;
- /* Looked okay on my machine. */
- right: -52rem;
- top: -3rem;
- }
.chart {
width: 100%;
}
- .legend {
- display: flex;
- flex-direction: column;
- row-gap: 3px;
- }
- .item {
+ .toolbar {
+ padding: 0;
display: flex;
flex-direction: row;
- column-gap: 10px;
- align-items: center;
+ justify-content: flex-end;
}
- .item .box {
- display: inline-block;
- width: 20px;
- height: 20px;
+
+ .toolbar .btn {
+ width: auto;
+ height: auto;
+ border: none;
+ padding: 5px;
}
</style>
diff --git a/development/plot-benchmarks/src/lib/Dataset.svelte b/development/plot-benchmarks/src/lib/Dataset.svelte
new file mode 100644
index 0000000..ce37d45
--- /dev/null
+++ b/development/plot-benchmarks/src/lib/Dataset.svelte
@@ -0,0 +1,68 @@
+<script lang="ts">
+ import { Session, type IndexedWrapper } from "../wrappers/session.js";
+
+ export let name: string;
+ export let datasetGroup: IndexedWrapper[];
+
+ let sources: Set<string>;
+ let sampledMetrics: Set<string>;
+ let metrics: Set<string>;
+
+ $: {
+ sources = Session.sources(datasetGroup);
+
+ let labels = datasetGroup
+ .map((indexed) => indexed.value.metricLabels())
+ .flat();
+
+ let sampled = datasetGroup
+ .map((indexed) => indexed.value.sampledLabels())
+ .flat();
+
+ metrics = new Set<string>(labels);
+ sampledMetrics = new Set<string>(sampled);
+ }
+</script>
+
+<div class="dataset">
+ <hgroup>
+ <div>{name}</div>
+ <div class="details">
+ <div class="sources">
+ {#each sources as source (source)}
+ <div>📁 <small>{source}</small></div>
+ {/each}
+ </div>
+ {#if metrics.size > 0}
+ <div class="metrics">
+ {#each metrics as metric}
+ <div>📏 <small>{metric}</small></div>
+ {/each}
+ </div>
+ {/if}
+ {#if sampledMetrics.size > 0}
+ <div class="sampled">
+ {#each sampledMetrics as metric}
+ <div>📏 <small>{metric}</small></div>
+ {/each}
+ </div>
+ {/if}
+ </div>
+ </hgroup>
+</div>
+
+<style>
+ .dataset {
+ margin: 0.5rem;
+ padding-top: 1rem;
+ padding-left: 1rem;
+ border: 1px solid #b3cee5;
+ }
+ .details {
+ padding-left: 1rem;
+ }
+
+ .details > div {
+ margin-top: 0.25rem;
+ }
+</style>
diff --git a/development/plot-benchmarks/src/lib/Files.svelte b/development/plot-benchmarks/src/lib/Files.svelte
deleted file mode 100644
index ffe09d6..0000000
--- a/development/plot-benchmarks/src/lib/Files.svelte
+++ /dev/null
@@ -1,124 +0,0 @@
-<script lang="ts">
- import { createEventDispatcher } from "svelte";
- import type { Writable } from "svelte/store";
- import { writable } from "svelte/store";
- import type { FileMetadata, FileMetadataEvent } from "../files.js";
- import { readBenchmarks } from "../files.js";
-
- let active: Writable<boolean> = writable(false);
- let entries: Writable<Array<FileMetadata>> = writable([]);
- let dispatcher = createEventDispatcher<FileMetadataEvent>();
-
- entries.subscribe(($entries) => {
- // Dispatch an event anytime files change.
- dispatcher("entries", [...$entries]);
- });
-
- function onDropFile(event: DragEvent) {
- handleDropFile(event); // async
- active.set(false);
- event.preventDefault();
- }
-
- async function handleDropFile(event: DragEvent) {
- const items = [...event.dataTransfer.items];
- const metadata = [];
- if (items) {
- for (let i = 0; i < items.length; i += 1) {
- if (items[i].kind === "file") {
- const file = items[i].getAsFile();
- if (file.name.endsWith(".json")) {
- const benchmarks = await readBenchmarks(file);
- const entry: FileMetadata = {
- enabled: true,
- file: file,
- benchmarks: benchmarks,
- };
- metadata.push(entry);
- }
- }
- }
- // Deep copy
- $entries = [...$entries, ...metadata];
- }
- }
-
- function onDragOver(event: DragEvent) {
- active.set(true);
- event.preventDefault();
- }
-
- function onDragLeave(event: DragEvent) {
- active.set(false);
- event.preventDefault();
- }
-
- function onChecked(entries: Array<FileMetadata>, index: number) {
- return (event: Event) => {
- // Deep copy
- const copied = [...entries];
- copied[index].enabled = !copied[index].enabled;
- $entries = copied;
- };
- }
-</script>
-
-<article
- id="drop"
- class="drop"
- class:active={$active}
- on:drop={onDropFile}
- on:dragover={onDragOver}
- on:dragleave={onDragLeave}
->
- {#if $entries.length > 0}
- <div class="files">
- {#each $entries as entry, index}
- <div class="file">
- <input
- type="checkbox"
- checked={entry.enabled}
- on:change={onChecked($entries, index)}
- />
- <div class="index">
- <span>{index + 1}</span>
- </div>
- <span class="label">{entry.file.name}</span>
- </div>
- {/each}
- </div>
- {:else}
- <h5>Drag and drop benchmark results to get started.</h5>
- {/if}
-</article>
-
-<style>
- .active {
- background-color: #00bfff;
- outline: gray;
- outline-style: groove;
- }
-
- .files {
- display: flex;
- flex-direction: column;
- row-gap: 3px;
- }
-
- .file {
- display: flex;
- flex-direction: row;
- column-gap: 10px;
- align-items: center;
- }
-
- .index {
- width: 2rem;
- height: 2rem;
- display: flex;
- flex-direction: row;
- align-items: center;
- justify-content: center;
- border: 2px solid sandybrown;
- }
-</style>
diff --git a/development/plot-benchmarks/src/lib/Group.svelte b/development/plot-benchmarks/src/lib/Group.svelte
new file mode 100644
index 0000000..328e606
--- /dev/null
+++ b/development/plot-benchmarks/src/lib/Group.svelte
@@ -0,0 +1,20 @@
+<script lang="ts">
+ import { Session, type IndexedWrapper } from "../wrappers/session.js";
+ import Dataset from "./Dataset.svelte";
+
+ export let className: string;
+ export let datasetGroup: IndexedWrapper[];
+ let datasetNames: Set<string>;
+ $: {
+ datasetNames = Session.datasetNames(datasetGroup);
+ }
+</script>
+
+<details open>
+ <summary>{className}</summary>
+ <div class="details">
+ {#each datasetNames as name (name)}
+ <Dataset {datasetGroup} {name} />
+ {/each}
+ </div>
+</details>
diff --git a/development/plot-benchmarks/src/lib/Legend.svelte b/development/plot-benchmarks/src/lib/Legend.svelte
new file mode 100644
index 0000000..ef239f5
--- /dev/null
+++ b/development/plot-benchmarks/src/lib/Legend.svelte
@@ -0,0 +1,60 @@
+<script lang="ts">
+ import type { Chart, LegendItem } from "chart.js";
+ export let chart: Chart;
+ export let items: LegendItem[];
+
+ const handlerFactory = (item: LegendItem) => {
+ return (event: Event) => {
+ // https://www.chartjs.org/docs/latest/samples/legend/html.html
+ chart.setDatasetVisibility(
+ item.datasetIndex,
+ !chart.isDatasetVisible(item.datasetIndex)
+ );
+ chart.update();
+ };
+ };
+</script>
+
+<div class="legend">
+ <h6>Legend</h6>
+ {#each items as item (item)}
+ <div
+ class="item"
+ on:dblclick={handlerFactory(item)}
+ aria-label="legend"
+ role="listitem"
+ >
+ <span
+ class="box"
+ style="background: {item.fillStyle}; border-color: {item.strokeStyle}; border-width: {item.lineWidth}px;"
+ />
+ <span
+ class="label"
+ style="text-decoration: {item.hidden ? 'line-through' : ''};"
+ >
+ {item.text}
+ </span>
+ </div>
+ {/each}
+</div>
+
+<style>
+ .legend {
+ padding: 4px;
+ display: flex;
+ flex-direction: column;
+ row-gap: 3px;
+ }
+
+ .item {
+ display: flex;
+ flex-direction: row;
+ column-gap: 10px;
+ align-items: center;
+ }
+ .item .box {
+ display: inline-block;
+ width: 20px;
+ height: 20px;
+ }
+</style>
diff --git a/development/plot-benchmarks/src/lib/Session.svelte b/development/plot-benchmarks/src/lib/Session.svelte
new file mode 100644
index 0000000..4b0cf57
--- /dev/null
+++ b/development/plot-benchmarks/src/lib/Session.svelte
@@ -0,0 +1,117 @@
+<script lang="ts">
+ import { createEventDispatcher } from "svelte";
+ import type { FileMetadata } from "../types/files.js";
+ import { Session, type IndexedWrapper } from "../wrappers/session.js";
+ import Group from "./Group.svelte";
+ import type { FileMetadataEvent } from "../types/events.js";
+ import { writable, type Writable } from "svelte/store";
+ import { readBenchmarks } from "../files.js";
+ import type { Metrics } from "../types/data.js";
+ import { Transforms } from "../transforms/metric-transforms.js";
+ import type { Data, Series } from "../types/chart.js";
+ import { ChartDataTransforms } from "../transforms/data-transforms.js";
+ import { STANDARD_MAPPER } from "../transforms/standard-mappers.js";
+ import Chart from "./Chart.svelte";
+
+ export let fileEntries: FileMetadata[];
+
+ // State
+ let eventDispatcher = createEventDispatcher<FileMetadataEvent>();
+ let session: Session;
+ let metrics: Metrics<number>;
+ let series: Series[];
+ let chartData: Data;
+ let classGroups: Record<string, IndexedWrapper[]>;
+ let size: number;
+
+ // Stores
+ let activeDragDrop: Writable<boolean> = writable(false);
+
+ $: {
+ session = new Session(fileEntries);
+ metrics = Transforms.buildMetrics(session);
+ series = ChartDataTransforms.mapToSeries(metrics, STANDARD_MAPPER);
+ chartData = ChartDataTransforms.mapToDataset(series);
+ classGroups = session.classGroups;
+ size = session.fileNames.size;
+ }
+
+ // Helpers
+
+ function onDropFile(event: DragEvent) {
+ handleFileDragDrop(event); // async
+ $activeDragDrop = false;
+ event.preventDefault();
+ }
+
+ function onDragOver(event: DragEvent) {
+ $activeDragDrop = true;
+ event.preventDefault();
+ }
+
+ function onDragLeave(event: DragEvent) {
+ $activeDragDrop = false;
+ event.preventDefault();
+ }
+
+ async function handleFileDragDrop(event: DragEvent) {
+ const items = [...event.dataTransfer.items];
+ const newFiles: FileMetadata[] = [];
+ if (items) {
+ for (let i = 0; i < items.length; i += 1) {
+ if (items[i].kind === "file") {
+ const file = items[i].getAsFile();
+ if (file.name.endsWith(".json")) {
+ const benchmarks = await readBenchmarks(file);
+ const entry: FileMetadata = {
+ enabled: true,
+ file: file,
+ container: benchmarks,
+ };
+ newFiles.push(entry);
+ }
+ }
+ }
+ // Deep copy & notify
+ eventDispatcher("entries", [...fileEntries, ...newFiles]);
+ }
+ }
+</script>
+
+{#if size <= 0}
+ <article
+ id="drop"
+ class="drop"
+ class:active={$activeDragDrop}
+ on:drop={onDropFile}
+ on:dragover={onDragOver}
+ on:dragleave={onDragLeave}
+ >
+ <h5>Drag and drop benchmark results to get started.</h5>
+ </article>
+{:else}
+ <article
+ id="drop"
+ class="drop"
+ class:active={$activeDragDrop}
+ on:drop={onDropFile}
+ on:dragover={onDragOver}
+ on:dragleave={onDragLeave}
+ >
+ <h5>Benchmarks</h5>
+ {#each Object.entries(classGroups) as [className, wrappers]}
+ <Group {className} datasetGroup={wrappers} />
+ {/each}
+ </article>
+
+ {#if series.length > 0}
+ <Chart data={chartData} />
+ {/if}
+{/if}
+
+<style>
+ .active {
+ outline: beige;
+ outline-style: dashed;
+ }
+</style>
diff --git a/development/plot-benchmarks/src/regexp.ts b/development/plot-benchmarks/src/regexp.ts
deleted file mode 100644
index b99513d..0000000
--- a/development/plot-benchmarks/src/regexp.ts
+++ /dev/null
@@ -1,17 +0,0 @@
-/**
- * Supports regular expressions, while falling back gracefully to substring matching.
- */
-export function expressionFilter(expression: string) {
- let regExp: RegExp | null = null;
- try {
- regExp = new RegExp(expression, 'g')
- } catch (error) {
- // Invalid regular expression.
- // Falling back to substring matching.
- console.warn(`Invalid regular expression ${expression}. Falling back to substring matching.`)
- }
- if (regExp) {
- return (label: string) => regExp.test(label);
- }
- return (label: string) => label.indexOf(expression) >= 0;
-}
diff --git a/development/plot-benchmarks/src/transforms.ts b/development/plot-benchmarks/src/transforms.ts
deleted file mode 100644
index be74e71..0000000
--- a/development/plot-benchmarks/src/transforms.ts
+++ /dev/null
@@ -1,123 +0,0 @@
-import type { Point } from "chart.js";
-import type { Benchmark, Benchmarks, Metrics, MetricsCollection } from "./schema.js";
-
-/**
- * A container for the chart data type.
- */
-export interface Data {
- name: string;
- data: Array<Point>;
-}
-
-export function benchmarksDataset(containers: Array<Benchmarks>, filterFn: (label: string) => boolean): Array<Data> {
- const datasets: Array<Data> = [];
- for (let i = 0; i < containers.length; i += 1) {
- let container = containers[i];
- for (let j = 0; j < container.benchmarks.length; j += 1) {
- const benchmark = container.benchmarks[j];
- // 1 based index
- const prefix = datasetName(i + 1, benchmark);
- const labels = metricLabels(benchmark);
- for (let k = 0; k < labels.length; k += 1) {
- const name = `${prefix}_${labels[k]}`;
- if (filterFn(name)) {
- const metrics = metricsData(benchmark, labels[k]);
- if (metrics) {
- const sampled = metrics.runs && Array.isArray(metrics.runs[0]);
- if (sampled) {
- // This is always the case for single metrics
- const runs = metrics.runs as number[][];
- datasets.push({
- name: name,
- // Compute histograms
- data: histogramPoints(runs)
- });
- } else {
-
- const runs = metrics.runs as number[];
- datasets.push({
- name: name,
- // Might want to make this compatible for scatter plots
- data: singlePoints(runs)
- });
- }
- }
- }
- }
- }
- }
- return datasets;
-}
-
-function datasetName(index: number, benchmark: Benchmark): string {
- const className = benchmark.className;
- const parts = className.split('.');
- const lastIndex = parts.length - 1;
- return `${index}_${parts[lastIndex]}_${benchmark.name}`;
-}
-
-function metricsData(benchmark: Benchmark, label: string): Metrics | undefined {
- let data = metricsDataFor(benchmark.metrics, label);
- if (data) {
- return data;
- }
- return metricsDataFor(benchmark.sampledMetrics, label);
-}
-
-function metricsDataFor(collection: MetricsCollection, label: string): Metrics | undefined {
- for (const key in collection) {
- if (collection.hasOwnProperty(key) && key == label) {
- return collection[key];
- }
- }
- return undefined;
-}
-
-function metricLabels(benchmark: Benchmark): Array<string> {
- const labels = labelsFor(benchmark.metrics);
- const sampled = labelsFor(benchmark.sampledMetrics);
- return [...labels, ...sampled];
-}
-
-function labelsFor(collection: MetricsCollection): Array<string> {
- const labels = [];
- if (collection) {
- for (const key in collection) {
- if (collection.hasOwnProperty(key)) {
- labels.push(key);
- }
- }
- }
- return labels;
-}
-
-function histogramPoints(runs: Array<number[]>, buckets: number = 10): Array<Point> {
- const flattened = runs.flat();
- // Default comparator coerces types to string !
- flattened.sort((a, b) => a - b); // in-place
- const min = flattened[0];
- const max = flattened[flattened.length - 1];
- const histogram = new Array(buckets).fill(0);
- const slots = buckets - 1; // The actual number of slots in the histogram
- for (let i = 0; i < flattened.length; i += 1) {
- let n = normalize(flattened[i], min, max);
- let index = Math.ceil(n * slots);
- histogram[index] = histogram[index] + 1;
- }
- return singlePoints(histogram);
-}
-
-function normalize(n: number, min: number, max: number): number {
- return (n - min) / (max - min + 1e-5);
-}
-
-function singlePoints(runs: Array<number>): Array<Point> {
- const points: Array<Point> = [];
- for (let i = 0; i < runs.length; i += 1) {
- points.push({
- x: i + 1, // 1 based index
- y: runs[i]
- });
- }
- return points;
-}
diff --git a/development/plot-benchmarks/src/transforms/data-transforms.ts b/development/plot-benchmarks/src/transforms/data-transforms.ts
new file mode 100644
index 0000000..daadebb
--- /dev/null
+++ b/development/plot-benchmarks/src/transforms/data-transforms.ts
@@ -0,0 +1,66 @@
+import type { ChartDataset, ChartType, Point } from "chart.js";
+import type { Data, Series } from "../types/chart.js";
+import type { Metric, Metrics } from "../types/data.js";
+
+export interface Mapper<T = number> {
+ standard: (value: Metric<T>) => Series[];
+ sampled: (value: Metric<T[]>) => Series[];
+}
+
+/**
+ * Converts `Metrics` to the corresponding chart data structures.
+ */
+export class ChartDataTransforms {
+
+ static mapToSeries(metrics: Metrics<number>, mapper: Mapper<number>): Series[] {
+ const series: Series[] = [];
+ const standard = metrics.standard;
+ const sampled = metrics.sampled;
+ if (standard) {
+ for (let i = 0; i < standard.length; i += 1) {
+ const metric = standard[i];
+ const mapped = mapper.standard(metric);
+ series.push(...mapped);
+ }
+ }
+ if (sampled) {
+ for (let i = 0; i < sampled.length; i += 1) {
+ const metric = sampled[i];
+ const mapped = mapper.sampled(metric);
+ series.push(...mapped);
+ }
+ }
+ return series;
+ }
+
+ static mapToDataset(series: Series[]): Data {
+ let xmax = 0;
+ let datasets: ChartDataset[] = [];
+ for (let i = 0; i < series.length; i += 1) {
+ xmax = Math.max(xmax, series[i].data.length);
+ datasets.push(ChartDataTransforms.chartDataset(series[i]));
+ }
+ const chartData: Data = {
+ labels: ChartDataTransforms.xrange(xmax),
+ datasets: datasets
+ };
+ return chartData;
+ }
+
+ private static chartDataset<T extends ChartType>(series: Series): ChartDataset {
+ return {
+ label: series.label,
+ type: series.type,
+ data: series.data,
+ ...series.options
+ };
+ }
+
+ private static xrange(xmax: number): number[] {
+ let range = [];
+ for (let i = 1; i <= xmax; i += 1) {
+ range.push(i);
+ }
+ return range;
+ }
+}
diff --git a/development/plot-benchmarks/src/transforms/metric-transforms.ts b/development/plot-benchmarks/src/transforms/metric-transforms.ts
new file mode 100644
index 0000000..56a8763
--- /dev/null
+++ b/development/plot-benchmarks/src/transforms/metric-transforms.ts
@@ -0,0 +1,115 @@
+import type { ChartData, Metric, Metrics } from "../types/data.js";
+import type { Session } from "../wrappers/session.js";
+
+/**
+ * Helps with transforming benchmark data into something that can be visualized easily.
+ */
+export class Transforms {
+ private constructor() {
+ // static helpers.
+ }
+
+ static buildMetrics(session: Session): Metrics<number> {
+ const classGroups = Object.entries(session.classGroups);
+ const standard: Metric<number>[] = [];
+ const sampled: Metric<number[]>[] = [];
+ for (let i = 0; i < classGroups.length; i += 1) {
+ const [className, wrappers] = classGroups[i];
+ for (let j = 0; j < wrappers.length; j += 1) {
+ const wrapper = wrappers[j];
+ const source = wrapper.source;
+ const testName = wrapper.value.testName();
+ // standard
+ let labels = wrapper.value.metricLabels();
+ for (let k = 0; k < labels.length; k += 1) {
+ const label = labels[k];
+ const metric = wrapper.value.metric(label);
+ const charData: ChartData<number> = {
+ values: metric.runs
+ };
+ Transforms.add<number>(
+ standard,
+ className,
+ testName,
+ label,
+ source,
+ charData
+ );
+ }
+ // sampled
+ labels = wrapper.value.sampledLabels();
+ for (let k = 0; k < labels.length; k += 1) {
+ const label = labels[k];
+ const metric = wrapper.value.sampled(label);
+ const charData: ChartData<number[]> = {
+ values: metric.runs
+ };
+ Transforms.add<number[]>(
+ sampled,
+ className,
+ testName,
+ label,
+ source,
+ charData
+ );
+ }
+ }
+ }
+ const metrics: Metrics<number> = {
+ standard: standard,
+ sampled: sampled
+ };
+ return metrics;
+ }
+
+ private static add<T>(
+ metrics: Metric<T>[],
+ className: string,
+ testName: string,
+ label: string,
+ source: string,
+ data: ChartData<T>
+ ) {
+ const metric = Transforms.getOrCreate<T>(metrics, className, testName, label);
+ metric.data[source] = data;
+ }
+
+ private static getOrCreate<T>(
+ metrics: Metric<T>[],
+ className: string,
+ testName: string,
+ label: string
+ ): Metric<T> {
+ let metric: Metric<T> | null = Transforms.find(metrics, className, testName, label);
+ if (metric == null) {
+ const data: Record<string, ChartData<T>> = {};
+ metric = {
+ class: className,
+ benchmark: testName,
+ label: label,
+ data: data
+ }
+ metrics.push(metric);
+ }
+ return metric;
+ }
+
+ private static find<T>(
+ metrics: Metric<T>[],
+ className: string,
+ testName: string,
+ label: string
+ ): Metric<T> | null {
+ for (let i = 0; i < metrics.length; i += 1) {
+ const metric = metrics[i];
+ if (
+ metric.class === className &&
+ metric.benchmark === testName &&
+ metric.label === label
+ ) {
+ return metric;
+ }
+ }
+ return null;
+ }
+}
diff --git a/development/plot-benchmarks/src/transforms/standard-mappers.ts b/development/plot-benchmarks/src/transforms/standard-mappers.ts
new file mode 100644
index 0000000..8b928cd
--- /dev/null
+++ b/development/plot-benchmarks/src/transforms/standard-mappers.ts
@@ -0,0 +1,90 @@
+import type { Point } from "chart.js";
+import type { Series } from "../types/chart.js";
+import type { ChartData, Metric } from "../types/data.js";
+import type { Mapper } from "./data-transforms.js";
+
+function sampledMapper(metric: Metric<number[]>): Series[] {
+ const series: Series[] = [];
+ const data: Record<string, ChartData<number[]>> = metric.data;
+ const entries = Object.entries(data);
+ for (let i = 0; i < entries.length; i += 1) {
+ const [source, chartData] = entries[i];
+ const label = labelFor(metric, source);
+ const points = histogramPoints(chartData.values);
+ series.push({
+ label: label,
+ type: "line",
+ data: points,
+ options: {
+ tension: 0.3
+ }
+ });
+ }
+ return series;
+}
+
+function standardMapper(metric: Metric<number>): Series[] {
+ const series: Series[] = [];
+ const data: Record<string, ChartData<number>> = metric.data;
+ const entries = Object.entries(data);
+ for (let i = 0; i < entries.length; i += 1) {
+ const [source, chartData] = entries[i];
+ const label = labelFor(metric, source);
+ const points = singlePoints(chartData.values);
+ series.push({
+ label: label,
+ type: "line",
+ data: points,
+ options: {
+ tension: 0.3
+ }
+ });
+ }
+ return series;
+}
+
+function histogramPoints(runs: number[][], buckets: number = 10): Point[] {
+ const flattened = runs.flat();
+ // Default comparator coerces types to string !
+ flattened.sort((a, b) => a - b); // in-place
+ const min = flattened[0];
+ const max = flattened[flattened.length - 1];
+ const histogram = new Array(buckets).fill(0);
+ const slots = buckets - 1; // The actual number of slots in the histogram
+ for (let i = 0; i < flattened.length; i += 1) {
+ let n = normalize(flattened[i], min, max);
+ let index = Math.ceil(n * slots);
+ histogram[index] = histogram[index] + 1;
+ }
+ return singlePoints(histogram);
+}
+
+function singlePoints(runs: number[]): Point[] {
+ const points: Point[] = [];
+ for (let i = 0; i < runs.length; i += 1) {
+ points.push({
+ x: i + 1, // 1 based index
+ y: runs[i]
+ });
+ }
+ return points;
+}
+
+function normalize(n: number, min: number, max: number): number {
+ return (n - min) / (max - min + 1e-5);
+}
+
+/**
+ * Generates a series label.
+ */
+function labelFor<T>(metric: Metric<T>, source: string): string {
+ return `${source}[${metric.class} ${metric.benchmark} ${metric.label}]`;
+}
+
+/**
+ * The standard mapper.
+ */
+export const STANDARD_MAPPER: Mapper = {
+ standard: standardMapper,
+ sampled: sampledMapper
+};
diff --git a/development/plot-benchmarks/src/schema.ts b/development/plot-benchmarks/src/types/benchmark.ts
similarity index 100%
rename from development/plot-benchmarks/src/schema.ts
rename to development/plot-benchmarks/src/types/benchmark.ts
diff --git a/development/plot-benchmarks/src/types/chart.ts b/development/plot-benchmarks/src/types/chart.ts
new file mode 100644
index 0000000..0d77b63
--- /dev/null
+++ b/development/plot-benchmarks/src/types/chart.ts
@@ -0,0 +1,27 @@
+import type { ChartDataset, ChartType, Point } from "chart.js";
+
+/**
+ * The chart data container.
+ *
+ * Has relevant default X-Axis labels & corresponding datasets.
+ */
+export interface Data {
+ // X-axis labels.
+ labels: number[];
+ // The corresponding datasets.
+ datasets: ChartDataset[];
+}
+
+/**
+ * A single data-series being rendered in the chart.
+ *
+ * Used by a Mapper for data transformations.
+ */
+export interface Series {
+ label: string;
+ type: ChartType;
+ data: Array<Point>;
+ // Additional series options
+ // For e.g. https://www.chartjs.org/docs/latest/charts/line.html
+ options: object;
+}
diff --git a/development/plot-benchmarks/src/types/data.ts b/development/plot-benchmarks/src/types/data.ts
new file mode 100644
index 0000000..6e6955c
--- /dev/null
+++ b/development/plot-benchmarks/src/types/data.ts
@@ -0,0 +1,28 @@
+/**
+ * A container for raw benchmark data.
+ *
+ * Can be extended to store descriptive statistics about the raw data.
+ */
+export interface ChartData<T> {
+ values: T[];
+}
+
+/**
+ * A container for a Metric.
+ *
+ * This metric has all relevant comparables, in the data keyed by the source.
+ */
+export interface Metric<T> {
+ class: string;
+ benchmark: string;
+ label: string;
+ data: Record<string, ChartData<T>>;
+}
+
+/**
+ * A container for standard and sampled `Metric` instances.
+ */
+export interface Metrics<T = number> {
+ standard?: Metric<T>[];
+ sampled?: Metric<T[]>[];
+}
diff --git a/development/plot-benchmarks/src/types/events.ts b/development/plot-benchmarks/src/types/events.ts
new file mode 100644
index 0000000..48c4777
--- /dev/null
+++ b/development/plot-benchmarks/src/types/events.ts
@@ -0,0 +1,5 @@
+import type { FileMetadata } from "./files.js";
+
+export interface FileMetadataEvent {
+ entries: Array<FileMetadata>;
+}
diff --git a/development/plot-benchmarks/src/types/files.ts b/development/plot-benchmarks/src/types/files.ts
new file mode 100644
index 0000000..930369e
--- /dev/null
+++ b/development/plot-benchmarks/src/types/files.ts
@@ -0,0 +1,10 @@
+import type { Benchmarks } from "./benchmark.js";
+
+/**
+ * File information + associated benchmarks.
+ */
+export interface FileMetadata {
+ file: File;
+ enabled: boolean;
+ container: Benchmarks;
+}
diff --git a/development/plot-benchmarks/src/wrappers/benchmarks.ts b/development/plot-benchmarks/src/wrappers/benchmarks.ts
new file mode 100644
index 0000000..9bbc015
--- /dev/null
+++ b/development/plot-benchmarks/src/wrappers/benchmarks.ts
@@ -0,0 +1,54 @@
+import type { Benchmark, MetricsCollection, Sampled, Standard } from "../types/benchmark.js";
+
+export class BenchmarkWrapper {
+ constructor(
+ private benchmark: Benchmark,
+ private separator: string = '_'
+ ) {
+
+ }
+
+ datasetName(): string {
+ return `${this.className()}${this.separator}${this.benchmark.name}`;
+ }
+
+ metric(label: string): Standard | undefined {
+ return this.benchmark?.metrics[label];
+ }
+
+ sampled(label: string): Sampled | undefined {
+ return this.benchmark?.sampledMetrics[label];
+ }
+
+ metricLabels(): string[] {
+ return BenchmarkWrapper.labels(this.benchmark.metrics);
+ }
+
+ sampledLabels(): string[] {
+ return BenchmarkWrapper.labels(this.benchmark.sampledMetrics);
+ }
+
+ className(): string {
+ const className = this.benchmark.className;
+ const parts = className.split('.');
+ const lastIndex = parts.length - 1;
+ return parts[lastIndex];
+ }
+
+ testName(): string {
+ return this.benchmark.name;
+ }
+
+ private static labels(collection: MetricsCollection | null): string[] {
+ const labels: string[] = [];
+ if (collection) {
+ for (const key in collection) {
+ if (collection.hasOwnProperty(key)) {
+ labels.push(key);
+ }
+ }
+ }
+ return labels;
+ }
+
+}
diff --git a/development/plot-benchmarks/src/wrappers/session.ts b/development/plot-benchmarks/src/wrappers/session.ts
new file mode 100644
index 0000000..2581f92
--- /dev/null
+++ b/development/plot-benchmarks/src/wrappers/session.ts
@@ -0,0 +1,72 @@
+import { type FileMetadata } from "../types/files.js";
+import { BenchmarkWrapper } from "./benchmarks.js";
+
+export interface Indexed<T> {
+ // The source of the benchmark.
+ source: string;
+ // `source` index.
+ index: number;
+ // The underlying type.
+ value: T;
+}
+
+export type IndexedWrapper = Indexed<BenchmarkWrapper>;
+
+/**
+ * A Benchmarking plotting session.
+ */
+export class Session {
+ // className -> BenchmarkWrapper[]
+ public classGroups: Record<string, IndexedWrapper[]>;
+ // BenchmarkWrapper[]
+ public benchmarks: IndexedWrapper[];
+ public fileNames: Set<string>;
+
+ constructor(public files: FileMetadata[]) {
+ this.initialize();
+ }
+
+ private initialize() {
+ this.classGroups = {};
+ this.fileNames = new Set();
+ this.benchmarks = [];
+
+ for (let i = 0; i < this.files.length; i += 1) {
+ const fileIndex = i;
+ const fileMeta = this.files[fileIndex];
+ for (let j = 0; j < fileMeta.container.benchmarks.length; j += 1) {
+ const wrapper = new BenchmarkWrapper(fileMeta.container.benchmarks[j]);
+ const fileGroupKey = wrapper.className();
+ const classGroup = this.classGroups[fileGroupKey] || [];
+ const item = {
+ source: fileMeta.file.name,
+ index: fileIndex,
+ value: wrapper
+ };
+ classGroup.push(item);
+ // Update
+ this.classGroups[fileGroupKey] = classGroup;
+ this.fileNames.add(fileMeta.file.name);
+ this.benchmarks.push(item);
+ }
+ }
+ }
+
+ static datasetNames(wrappers: IndexedWrapper[]): Set<string> {
+ const names = new Set<string>();
+ for (let i = 0; i < wrappers.length; i += 1) {
+ const wrapper = wrappers[i];
+ names.add(wrapper.value.datasetName());
+ }
+ return names;
+ }
+
+ static sources(wrappers: IndexedWrapper[]): Set<string> {
+ const sources = new Set<string>();
+ for (let i = 0; i < wrappers.length; i += 1) {
+ sources.add(wrappers[i].source);
+ }
+ return sources;
+ }
+
+}
