Refactor Plot benchmarks. * Faster rendering. * Flexible mappers for custom series generation. * Make it easier to visually inspect and find benchmark data sources. Change-Id: I37634549e0204fff3c6059717d12566d0abdf630 Test: `npm run-script dev`.
diff --git a/development/plot-benchmarks/src/chart-transforms.ts b/development/plot-benchmarks/src/chart-transforms.ts deleted file mode 100644 index f76049f..0000000 --- a/development/plot-benchmarks/src/chart-transforms.ts +++ /dev/null
@@ -1,32 +0,0 @@ -import type { ChartDataset } from "chart.js"; -import type { Data } from "./transforms.js"; - -export function chartData(container: Array<Data>) { - let xmax = 0; - let datasets = []; - for (let i = 0; i < container.length; i += 1) { - // update xmax - xmax = Math.max(xmax, container[i].data.length); - datasets.push(chartDataset(container[i])); - } - return { - labels: xrange(xmax), - datasets: datasets - }; -} - -function xrange(xmax: number): Array<number> { - let labels = []; - for (let i = 1; i <= xmax; i += 1) { - labels.push(i); - } - return labels; -} - -function chartDataset(data: Data): ChartDataset<'line'> { - return { - label: data.name, - data: data.data, - tension: 0.3 - }; -}
diff --git a/development/plot-benchmarks/src/files.ts b/development/plot-benchmarks/src/files.ts index a5e26ef..6d45cd9 100644 --- a/development/plot-benchmarks/src/files.ts +++ b/development/plot-benchmarks/src/files.ts
@@ -1,14 +1,4 @@ -import type { Benchmarks } from "./schema.js"; - -export interface FileMetadata { - file: File; - enabled: boolean; - benchmarks: Benchmarks; -} - -export interface FileMetadataEvent { - entries: Array<FileMetadata>; -} +import type { Benchmarks } from "./types/benchmark.js"; export async function readBenchmarks(file: File): Promise<Benchmarks> { const contents = await readFile(file);
diff --git a/development/plot-benchmarks/src/lib/App.svelte b/development/plot-benchmarks/src/lib/App.svelte index e50908c..ffa3e57 100644 --- a/development/plot-benchmarks/src/lib/App.svelte +++ b/development/plot-benchmarks/src/lib/App.svelte
@@ -1,23 +1,14 @@ <script lang="ts"> import type { Writable } from "svelte/store"; - import { derived, writable } from "svelte/store"; - import type { FileMetadata } from "../files.js"; - import type { Benchmarks } from "../schema.js"; - import Chart from "./Chart.svelte"; - import Files from "./Files.svelte"; + import { writable } from "svelte/store"; + import type { FileMetadata } from "../types/files.js"; + import Session from "./Session.svelte"; - let entries: Writable<Array<FileMetadata>> = writable([]); - let containers = derived(entries, ($entries) => { - const containers: Array<Benchmarks> = []; - for (let i = 0; i < $entries.length; i += 1) { - let entry = $entries[i]; - containers.push(entry.benchmarks); - } - return containers; - }); + // Stores + let entries: Writable<FileMetadata[]> = writable([]); - function onFileMetadataEvent(event) { - const detail: Array<FileMetadata> = event.detail; + function onFilesChanged(event) { + const detail: FileMetadata[] = event.detail; if (detail) { const enabled = detail.filter((metadata) => metadata.enabled === true); $entries = [...enabled]; @@ -31,8 +22,5 @@ </details> <div class="container"> - <Files on:entries={onFileMetadataEvent} /> - {#if $containers.length > 0} - <Chart {containers} /> - {/if} + <Session fileEntries={$entries} on:entries={onFilesChanged} /> </div>
diff --git a/development/plot-benchmarks/src/lib/Chart.svelte b/development/plot-benchmarks/src/lib/Chart.svelte index 257f228..7a063a8 100644 --- a/development/plot-benchmarks/src/lib/Chart.svelte +++ b/development/plot-benchmarks/src/lib/Chart.svelte
@@ -1,42 +1,33 @@ <script lang="ts"> - import type { LegendItem } from "chart.js"; + import type { ChartType, LegendItem } from "chart.js"; import { Chart } from "chart.js/auto"; import { onMount } from "svelte"; - import type { Readable, Writable } from "svelte/store"; - import { derived, writable } from "svelte/store"; - import { chartData } from "../chart-transforms.js"; - import { saveToClipboard } from "../clipboard.js"; + import { writable, type Writable } from "svelte/store"; + import type { Data } from "../types/chart.js"; import { LegendPlugin } from "../plugins.js"; - import { expressionFilter } from "../regexp.js"; - import type { Benchmarks } from "../schema.js"; - import { benchmarksDataset } from "../transforms.js"; - type FilterFn = (label: string) => boolean; + import Legend from "./Legend.svelte"; + import { saveToClipboard as save } from "../clipboard.js"; - export let containers: Readable<Array<Benchmarks>>; + export let data: Data; + export let chartType: ChartType = "line"; - let canvas: HTMLCanvasElement; - let filter: Writable<FilterFn> = writable((_: string) => true); - - let data = derived([containers, filter], ([$containers, $filter]) => { - return chartData(benchmarksDataset($containers, $filter)); - }); - - let chart: Writable<Chart | undefined> = writable(null); - let legendLabels: Writable<Array<LegendItem> | undefined> = writable(null); - - data.subscribe(($data) => { + $: { if ($chart) { - $chart.data = $data; + $chart.data = data; $chart.update(); } - }); + } + // State + let element: HTMLCanvasElement; + let chart: Writable<Chart | null> = writable(null); + let items: Writable<LegendItem[] | null> = writable(null); + + // Effects onMount(() => { - const onUpdate = (updated: Chart) => { - chart.set(updated); - legendLabels.set( - updated.options.plugins.legend.labels.generateLabels(updated) - ); + const onUpdate = (chart: Chart) => { + $chart = chart; + $items = chart.options.plugins.legend.labels.generateLabels(chart); }; const plugins = { legend: { @@ -46,119 +37,56 @@ onUpdate: onUpdate, }, }; - chart.set( - new Chart(canvas, { - type: "line", - data: $data, - plugins: [LegendPlugin], - options: { - plugins: plugins, - }, - }) - ); + $chart = new Chart(element, { + data: data, + type: chartType, + plugins: [LegendPlugin], + options: { + plugins: plugins, + }, + }); }); - function onItemClick(item: LegendItem) { - return (_: Event) => { - // https://www.chartjs.org/docs/latest/samples/legend/html.html - $chart.setDatasetVisibility( - item.datasetIndex, - !$chart.isDatasetVisible(item.datasetIndex) - ); - // Update chart - $chart.update(); - }; - } - - function onFilterChanged(event: Event) { - const target = event.currentTarget as HTMLInputElement; - const value = target.value; - $filter = expressionFilter(value); - } - - async function onCopy(_: Event) { + // Copy to clip board + async function copy(event: Event) { if ($chart) { - await saveToClipboard($chart); + await save($chart); } } </script> <article> - <button - class="copy outline" - data-tooltip="Copy chart to clipboard." - on:click={onCopy} - > - ⎘ - </button> - <canvas id="chart" class="chart" bind:this={canvas} /> + <div class="toolbar"> + <button + class="btn outline" + data-tooltip="Copy chart to clipboard." + on:click={copy} + > + ⎘ + </button> + </div> + <canvas id="chart" class="chart" bind:this={element} /> </article> -{#if $legendLabels && $legendLabels.length >= 0} - <article> - <div class="filter"> - <label for="metricFilter"> - <input - type="text" - id="metricFilter" - name="metricFilter" - placeholder="Filter metrics (regular expressions)" - autocomplete="off" - on:input={onFilterChanged} - /> - </label> - </div> - <div class="legend"> - {#each $legendLabels as label, index} - <div - class="item" - on:dblclick={onItemClick(label)} - aria-label="legend" - role="listitem" - > - <span - class="box" - style="background: {label.fillStyle}; border-color: {label.strokeStyle}; border-width: {label.lineWidth}px;" - /> - <span - class="label" - style="text-decoration: {label.hidden ? 'line-through' : ''};" - > - {label.text} - </span> - </div> - {/each} - </div> - </article> +{#if $items} + <Legend chart={$chart} items={$items} /> {/if} <style> - .copy { - width: 4%; - position: relative; - padding: 0; - border: none; - /* Looked okay on my machine. */ - right: -52rem; - top: -3rem; - } .chart { width: 100%; } - .legend { - display: flex; - flex-direction: column; - row-gap: 3px; - } - .item { + .toolbar { + padding: 0; display: flex; flex-direction: row; - column-gap: 10px; - align-items: center; + justify-content: flex-end; } - .item .box { - display: inline-block; - width: 20px; - height: 20px; + + .toolbar .btn { + width: auto; + height: auto; + border: none; + padding: 5px; } </style>
diff --git a/development/plot-benchmarks/src/lib/Dataset.svelte b/development/plot-benchmarks/src/lib/Dataset.svelte new file mode 100644 index 0000000..ce37d45 --- /dev/null +++ b/development/plot-benchmarks/src/lib/Dataset.svelte
@@ -0,0 +1,68 @@ +<script lang="ts"> + import { Session, type IndexedWrapper } from "../wrappers/session.js"; + + export let name: string; + export let datasetGroup: IndexedWrapper[]; + + let sources: Set<string>; + let sampledMetrics: Set<string>; + let metrics: Set<string>; + + $: { + sources = Session.sources(datasetGroup); + + let labels = datasetGroup + .map((indexed) => indexed.value.metricLabels()) + .flat(); + + let sampled = datasetGroup + .map((indexed) => indexed.value.sampledLabels()) + .flat(); + + metrics = new Set<string>(labels); + sampledMetrics = new Set<string>(sampled); + } +</script> + +<div class="dataset"> + <hgroup> + <div>{name}</div> + <div class="details"> + <div class="sources"> + {#each sources as source (source)} + <div>📁 <small>{source}</small></div> + {/each} + </div> + {#if metrics.size > 0} + <div class="metrics"> + {#each metrics as metric} + <div>📏 <small>{metric}</small></div> + {/each} + </div> + {/if} + {#if sampledMetrics.size > 0} + <div class="sampled"> + {#each sampledMetrics as metric} + <div>📏 <small>{metric}</small></div> + {/each} + </div> + {/if} + </div> + </hgroup> +</div> + +<style> + .dataset { + margin: 0.5rem; + padding-top: 1rem; + padding-left: 1rem; + border: 1px solid #b3cee5; + } + .details { + padding-left: 1rem; + } + + .details > div { + margin-top: 0.25rem; + } +</style>
diff --git a/development/plot-benchmarks/src/lib/Files.svelte b/development/plot-benchmarks/src/lib/Files.svelte deleted file mode 100644 index ffe09d6..0000000 --- a/development/plot-benchmarks/src/lib/Files.svelte +++ /dev/null
@@ -1,124 +0,0 @@ -<script lang="ts"> - import { createEventDispatcher } from "svelte"; - import type { Writable } from "svelte/store"; - import { writable } from "svelte/store"; - import type { FileMetadata, FileMetadataEvent } from "../files.js"; - import { readBenchmarks } from "../files.js"; - - let active: Writable<boolean> = writable(false); - let entries: Writable<Array<FileMetadata>> = writable([]); - let dispatcher = createEventDispatcher<FileMetadataEvent>(); - - entries.subscribe(($entries) => { - // Dispatch an event anytime files change. - dispatcher("entries", [...$entries]); - }); - - function onDropFile(event: DragEvent) { - handleDropFile(event); // async - active.set(false); - event.preventDefault(); - } - - async function handleDropFile(event: DragEvent) { - const items = [...event.dataTransfer.items]; - const metadata = []; - if (items) { - for (let i = 0; i < items.length; i += 1) { - if (items[i].kind === "file") { - const file = items[i].getAsFile(); - if (file.name.endsWith(".json")) { - const benchmarks = await readBenchmarks(file); - const entry: FileMetadata = { - enabled: true, - file: file, - benchmarks: benchmarks, - }; - metadata.push(entry); - } - } - } - // Deep copy - $entries = [...$entries, ...metadata]; - } - } - - function onDragOver(event: DragEvent) { - active.set(true); - event.preventDefault(); - } - - function onDragLeave(event: DragEvent) { - active.set(false); - event.preventDefault(); - } - - function onChecked(entries: Array<FileMetadata>, index: number) { - return (event: Event) => { - // Deep copy - const copied = [...entries]; - copied[index].enabled = !copied[index].enabled; - $entries = copied; - }; - } -</script> - -<article - id="drop" - class="drop" - class:active={$active} - on:drop={onDropFile} - on:dragover={onDragOver} - on:dragleave={onDragLeave} -> - {#if $entries.length > 0} - <div class="files"> - {#each $entries as entry, index} - <div class="file"> - <input - type="checkbox" - checked={entry.enabled} - on:change={onChecked($entries, index)} - /> - <div class="index"> - <span>{index + 1}</span> - </div> - <span class="label">{entry.file.name}</span> - </div> - {/each} - </div> - {:else} - <h5>Drag and drop benchmark results to get started.</h5> - {/if} -</article> - -<style> - .active { - background-color: #00bfff; - outline: gray; - outline-style: groove; - } - - .files { - display: flex; - flex-direction: column; - row-gap: 3px; - } - - .file { - display: flex; - flex-direction: row; - column-gap: 10px; - align-items: center; - } - - .index { - width: 2rem; - height: 2rem; - display: flex; - flex-direction: row; - align-items: center; - justify-content: center; - border: 2px solid sandybrown; - } -</style>
diff --git a/development/plot-benchmarks/src/lib/Group.svelte b/development/plot-benchmarks/src/lib/Group.svelte new file mode 100644 index 0000000..328e606 --- /dev/null +++ b/development/plot-benchmarks/src/lib/Group.svelte
@@ -0,0 +1,20 @@ +<script lang="ts"> + import { Session, type IndexedWrapper } from "../wrappers/session.js"; + import Dataset from "./Dataset.svelte"; + + export let className: string; + export let datasetGroup: IndexedWrapper[]; + let datasetNames: Set<string>; + $: { + datasetNames = Session.datasetNames(datasetGroup); + } +</script> + +<details open> + <summary>{className}</summary> + <div class="details"> + {#each datasetNames as name (name)} + <Dataset {datasetGroup} {name} /> + {/each} + </div> +</details>
diff --git a/development/plot-benchmarks/src/lib/Legend.svelte b/development/plot-benchmarks/src/lib/Legend.svelte new file mode 100644 index 0000000..ef239f5 --- /dev/null +++ b/development/plot-benchmarks/src/lib/Legend.svelte
@@ -0,0 +1,60 @@ +<script lang="ts"> + import type { Chart, LegendItem } from "chart.js"; + export let chart: Chart; + export let items: LegendItem[]; + + const handlerFactory = (item: LegendItem) => { + return (event: Event) => { + // https://www.chartjs.org/docs/latest/samples/legend/html.html + chart.setDatasetVisibility( + item.datasetIndex, + !chart.isDatasetVisible(item.datasetIndex) + ); + chart.update(); + }; + }; +</script> + +<div class="legend"> + <h6>Legend</h6> + {#each items as item (item)} + <div + class="item" + on:dblclick={handlerFactory(item)} + aria-label="legend" + role="listitem" + > + <span + class="box" + style="background: {item.fillStyle}; border-color: {item.strokeStyle}; border-width: {item.lineWidth}px;" + /> + <span + class="label" + style="text-decoration: {item.hidden ? 'line-through' : ''};" + > + {item.text} + </span> + </div> + {/each} +</div> + +<style> + .legend { + padding: 4px; + display: flex; + flex-direction: column; + row-gap: 3px; + } + + .item { + display: flex; + flex-direction: row; + column-gap: 10px; + align-items: center; + } + .item .box { + display: inline-block; + width: 20px; + height: 20px; + } +</style>
diff --git a/development/plot-benchmarks/src/lib/Session.svelte b/development/plot-benchmarks/src/lib/Session.svelte new file mode 100644 index 0000000..4b0cf57 --- /dev/null +++ b/development/plot-benchmarks/src/lib/Session.svelte
@@ -0,0 +1,117 @@ +<script lang="ts"> + import { createEventDispatcher } from "svelte"; + import type { FileMetadata } from "../types/files.js"; + import { Session, type IndexedWrapper } from "../wrappers/session.js"; + import Group from "./Group.svelte"; + import type { FileMetadataEvent } from "../types/events.js"; + import { writable, type Writable } from "svelte/store"; + import { readBenchmarks } from "../files.js"; + import type { Metrics } from "../types/data.js"; + import { Transforms } from "../transforms/metric-transforms.js"; + import type { Data, Series } from "../types/chart.js"; + import { ChartDataTransforms } from "../transforms/data-transforms.js"; + import { STANDARD_MAPPER } from "../transforms/standard-mappers.js"; + import Chart from "./Chart.svelte"; + + export let fileEntries: FileMetadata[]; + + // State + let eventDispatcher = createEventDispatcher<FileMetadataEvent>(); + let session: Session; + let metrics: Metrics<number>; + let series: Series[]; + let chartData: Data; + let classGroups: Record<string, IndexedWrapper[]>; + let size: number; + + // Stores + let activeDragDrop: Writable<boolean> = writable(false); + + $: { + session = new Session(fileEntries); + metrics = Transforms.buildMetrics(session); + series = ChartDataTransforms.mapToSeries(metrics, STANDARD_MAPPER); + chartData = ChartDataTransforms.mapToDataset(series); + classGroups = session.classGroups; + size = session.fileNames.size; + } + + // Helpers + + function onDropFile(event: DragEvent) { + handleFileDragDrop(event); // async + $activeDragDrop = false; + event.preventDefault(); + } + + function onDragOver(event: DragEvent) { + $activeDragDrop = true; + event.preventDefault(); + } + + function onDragLeave(event: DragEvent) { + $activeDragDrop = false; + event.preventDefault(); + } + + async function handleFileDragDrop(event: DragEvent) { + const items = [...event.dataTransfer.items]; + const newFiles: FileMetadata[] = []; + if (items) { + for (let i = 0; i < items.length; i += 1) { + if (items[i].kind === "file") { + const file = items[i].getAsFile(); + if (file.name.endsWith(".json")) { + const benchmarks = await readBenchmarks(file); + const entry: FileMetadata = { + enabled: true, + file: file, + container: benchmarks, + }; + newFiles.push(entry); + } + } + } + // Deep copy & notify + eventDispatcher("entries", [...fileEntries, ...newFiles]); + } + } +</script> + +{#if size <= 0} + <article + id="drop" + class="drop" + class:active={$activeDragDrop} + on:drop={onDropFile} + on:dragover={onDragOver} + on:dragleave={onDragLeave} + > + <h5>Drag and drop benchmark results to get started.</h5> + </article> +{:else} + <article + id="drop" + class="drop" + class:active={$activeDragDrop} + on:drop={onDropFile} + on:dragover={onDragOver} + on:dragleave={onDragLeave} + > + <h5>Benchmarks</h5> + {#each Object.entries(classGroups) as [className, wrappers]} + <Group {className} datasetGroup={wrappers} /> + {/each} + </article> + + {#if series.length > 0} + <Chart data={chartData} /> + {/if} +{/if} + +<style> + .active { + outline: beige; + outline-style: dashed; + } +</style>
diff --git a/development/plot-benchmarks/src/regexp.ts b/development/plot-benchmarks/src/regexp.ts deleted file mode 100644 index b99513d..0000000 --- a/development/plot-benchmarks/src/regexp.ts +++ /dev/null
@@ -1,17 +0,0 @@ -/** - * Supports regular expressions, while falling back gracefully to substring matching. - */ -export function expressionFilter(expression: string) { - let regExp: RegExp | null = null; - try { - regExp = new RegExp(expression, 'g') - } catch (error) { - // Invalid regular expression. - // Falling back to substring matching. - console.warn(`Invalid regular expression ${expression}. Falling back to substring matching.`) - } - if (regExp) { - return (label: string) => regExp.test(label); - } - return (label: string) => label.indexOf(expression) >= 0; -}
diff --git a/development/plot-benchmarks/src/transforms.ts b/development/plot-benchmarks/src/transforms.ts deleted file mode 100644 index be74e71..0000000 --- a/development/plot-benchmarks/src/transforms.ts +++ /dev/null
@@ -1,123 +0,0 @@ -import type { Point } from "chart.js"; -import type { Benchmark, Benchmarks, Metrics, MetricsCollection } from "./schema.js"; - -/** - * A container for the chart data type. - */ -export interface Data { - name: string; - data: Array<Point>; -} - -export function benchmarksDataset(containers: Array<Benchmarks>, filterFn: (label: string) => boolean): Array<Data> { - const datasets: Array<Data> = []; - for (let i = 0; i < containers.length; i += 1) { - let container = containers[i]; - for (let j = 0; j < container.benchmarks.length; j += 1) { - const benchmark = container.benchmarks[j]; - // 1 based index - const prefix = datasetName(i + 1, benchmark); - const labels = metricLabels(benchmark); - for (let k = 0; k < labels.length; k += 1) { - const name = `${prefix}_${labels[k]}`; - if (filterFn(name)) { - const metrics = metricsData(benchmark, labels[k]); - if (metrics) { - const sampled = metrics.runs && Array.isArray(metrics.runs[0]); - if (sampled) { - // This is always the case for single metrics - const runs = metrics.runs as number[][]; - datasets.push({ - name: name, - // Compute histograms - data: histogramPoints(runs) - }); - } else { - - const runs = metrics.runs as number[]; - datasets.push({ - name: name, - // Might want to make this compatible for scatter plots - data: singlePoints(runs) - }); - } - } - } - } - } - } - return datasets; -} - -function datasetName(index: number, benchmark: Benchmark): string { - const className = benchmark.className; - const parts = className.split('.'); - const lastIndex = parts.length - 1; - return `${index}_${parts[lastIndex]}_${benchmark.name}`; -} - -function metricsData(benchmark: Benchmark, label: string): Metrics | undefined { - let data = metricsDataFor(benchmark.metrics, label); - if (data) { - return data; - } - return metricsDataFor(benchmark.sampledMetrics, label); -} - -function metricsDataFor(collection: MetricsCollection, label: string): Metrics | undefined { - for (const key in collection) { - if (collection.hasOwnProperty(key) && key == label) { - return collection[key]; - } - } - return undefined; -} - -function metricLabels(benchmark: Benchmark): Array<string> { - const labels = labelsFor(benchmark.metrics); - const sampled = labelsFor(benchmark.sampledMetrics); - return [...labels, ...sampled]; -} - -function labelsFor(collection: MetricsCollection): Array<string> { - const labels = []; - if (collection) { - for (const key in collection) { - if (collection.hasOwnProperty(key)) { - labels.push(key); - } - } - } - return labels; -} - -function histogramPoints(runs: Array<number[]>, buckets: number = 10): Array<Point> { - const flattened = runs.flat(); - // Default comparator coerces types to string ! - flattened.sort((a, b) => a - b); // in-place - const min = flattened[0]; - const max = flattened[flattened.length - 1]; - const histogram = new Array(buckets).fill(0); - const slots = buckets - 1; // The actual number of slots in the histogram - for (let i = 0; i < flattened.length; i += 1) { - let n = normalize(flattened[i], min, max); - let index = Math.ceil(n * slots); - histogram[index] = histogram[index] + 1; - } - return singlePoints(histogram); -} - -function normalize(n: number, min: number, max: number): number { - return (n - min) / (max - min + 1e-5); -} - -function singlePoints(runs: Array<number>): Array<Point> { - const points: Array<Point> = []; - for (let i = 0; i < runs.length; i += 1) { - points.push({ - x: i + 1, // 1 based index - y: runs[i] - }); - } - return points; -}
diff --git a/development/plot-benchmarks/src/transforms/data-transforms.ts b/development/plot-benchmarks/src/transforms/data-transforms.ts new file mode 100644 index 0000000..daadebb --- /dev/null +++ b/development/plot-benchmarks/src/transforms/data-transforms.ts
@@ -0,0 +1,66 @@ +import type { ChartDataset, ChartType, Point } from "chart.js"; +import type { Data, Series } from "../types/chart.js"; +import type { Metric, Metrics } from "../types/data.js"; + +export interface Mapper<T = number> { + standard: (value: Metric<T>) => Series[]; + sampled: (value: Metric<T[]>) => Series[]; +} + +/** + * Converts `Metrics` to the corresponding chart data structures. + */ +export class ChartDataTransforms { + + static mapToSeries(metrics: Metrics<number>, mapper: Mapper<number>): Series[] { + const series: Series[] = []; + const standard = metrics.standard; + const sampled = metrics.sampled; + if (standard) { + for (let i = 0; i < standard.length; i += 1) { + const metric = standard[i]; + const mapped = mapper.standard(metric); + series.push(...mapped); + } + } + if (sampled) { + for (let i = 0; i < sampled.length; i += 1) { + const metric = sampled[i]; + const mapped = mapper.sampled(metric); + series.push(...mapped); + } + } + return series; + } + + static mapToDataset(series: Series[]): Data { + let xmax = 0; + let datasets: ChartDataset[] = []; + for (let i = 0; i < series.length; i += 1) { + xmax = Math.max(xmax, series[i].data.length); + datasets.push(ChartDataTransforms.chartDataset(series[i])); + } + const chartData: Data = { + labels: ChartDataTransforms.xrange(xmax), + datasets: datasets + }; + return chartData; + } + + private static chartDataset<T extends ChartType>(series: Series): ChartDataset { + return { + label: series.label, + type: series.type, + data: series.data, + ...series.options + }; + } + + private static xrange(xmax: number): number[] { + let range = []; + for (let i = 1; i <= xmax; i += 1) { + range.push(i); + } + return range; + } +}
diff --git a/development/plot-benchmarks/src/transforms/metric-transforms.ts b/development/plot-benchmarks/src/transforms/metric-transforms.ts new file mode 100644 index 0000000..56a8763 --- /dev/null +++ b/development/plot-benchmarks/src/transforms/metric-transforms.ts
@@ -0,0 +1,115 @@ +import type { ChartData, Metric, Metrics } from "../types/data.js"; +import type { Session } from "../wrappers/session.js"; + +/** + * Helps with transforming benchmark data into something that can be visualized easily. + */ +export class Transforms { + private constructor() { + // static helpers. + } + + static buildMetrics(session: Session): Metrics<number> { + const classGroups = Object.entries(session.classGroups); + const standard: Metric<number>[] = []; + const sampled: Metric<number[]>[] = []; + for (let i = 0; i < classGroups.length; i += 1) { + const [className, wrappers] = classGroups[i]; + for (let j = 0; j < wrappers.length; j += 1) { + const wrapper = wrappers[j]; + const source = wrapper.source; + const testName = wrapper.value.testName(); + // standard + let labels = wrapper.value.metricLabels(); + for (let k = 0; k < labels.length; k += 1) { + const label = labels[k]; + const metric = wrapper.value.metric(label); + const charData: ChartData<number> = { + values: metric.runs + }; + Transforms.add<number>( + standard, + className, + testName, + label, + source, + charData + ); + } + // sampled + labels = wrapper.value.sampledLabels(); + for (let k = 0; k < labels.length; k += 1) { + const label = labels[k]; + const metric = wrapper.value.sampled(label); + const charData: ChartData<number[]> = { + values: metric.runs + }; + Transforms.add<number[]>( + sampled, + className, + testName, + label, + source, + charData + ); + } + } + } + const metrics: Metrics<number> = { + standard: standard, + sampled: sampled + }; + return metrics; + } + + private static add<T>( + metrics: Metric<T>[], + className: string, + testName: string, + label: string, + source: string, + data: ChartData<T> + ) { + const metric = Transforms.getOrCreate<T>(metrics, className, testName, label); + metric.data[source] = data; + } + + private static getOrCreate<T>( + metrics: Metric<T>[], + className: string, + testName: string, + label: string + ): Metric<T> { + let metric: Metric<T> | null = Transforms.find(metrics, className, testName, label); + if (metric == null) { + const data: Record<string, ChartData<T>> = {}; + metric = { + class: className, + benchmark: testName, + label: label, + data: data + } + metrics.push(metric); + } + return metric; + } + + private static find<T>( + metrics: Metric<T>[], + className: string, + testName: string, + label: string + ): Metric<T> | null { + for (let i = 0; i < metrics.length; i += 1) { + const metric = metrics[i]; + if ( + metric.class === className && + metric.benchmark === testName && + metric.label === label + ) { + return metric; + } + } + return null; + } +}
diff --git a/development/plot-benchmarks/src/transforms/standard-mappers.ts b/development/plot-benchmarks/src/transforms/standard-mappers.ts new file mode 100644 index 0000000..8b928cd --- /dev/null +++ b/development/plot-benchmarks/src/transforms/standard-mappers.ts
@@ -0,0 +1,90 @@ +import type { Point } from "chart.js"; +import type { Series } from "../types/chart.js"; +import type { ChartData, Metric } from "../types/data.js"; +import type { Mapper } from "./data-transforms.js"; + +function sampledMapper(metric: Metric<number[]>): Series[] { + const series: Series[] = []; + const data: Record<string, ChartData<number[]>> = metric.data; + const entries = Object.entries(data); + for (let i = 0; i < entries.length; i += 1) { + const [source, chartData] = entries[i]; + const label = labelFor(metric, source); + const points = histogramPoints(chartData.values); + series.push({ + label: label, + type: "line", + data: points, + options: { + tension: 0.3 + } + }); + } + return series; +} + +function standardMapper(metric: Metric<number>): Series[] { + const series: Series[] = []; + const data: Record<string, ChartData<number>> = metric.data; + const entries = Object.entries(data); + for (let i = 0; i < entries.length; i += 1) { + const [source, chartData] = entries[i]; + const label = labelFor(metric, source); + const points = singlePoints(chartData.values); + series.push({ + label: label, + type: "line", + data: points, + options: { + tension: 0.3 + } + }); + } + return series; +} + +function histogramPoints(runs: number[][], buckets: number = 10): Point[] { + const flattened = runs.flat(); + // Default comparator coerces types to string ! + flattened.sort((a, b) => a - b); // in-place + const min = flattened[0]; + const max = flattened[flattened.length - 1]; + const histogram = new Array(buckets).fill(0); + const slots = buckets - 1; // The actual number of slots in the histogram + for (let i = 0; i < flattened.length; i += 1) { + let n = normalize(flattened[i], min, max); + let index = Math.ceil(n * slots); + histogram[index] = histogram[index] + 1; + } + return singlePoints(histogram); +} + +function singlePoints(runs: number[]): Point[] { + const points: Point[] = []; + for (let i = 0; i < runs.length; i += 1) { + points.push({ + x: i + 1, // 1 based index + y: runs[i] + }); + } + return points; +} + +function normalize(n: number, min: number, max: number): number { + return (n - min) / (max - min + 1e-5); +} + +/** + * Generates a series label. + */ +function labelFor<T>(metric: Metric<T>, source: string): string { + return `${source}[${metric.class} ${metric.benchmark} ${metric.label}]`; +} + +/** + * The standard mapper. + */ +export const STANDARD_MAPPER: Mapper = { + standard: standardMapper, + sampled: sampledMapper +};
diff --git a/development/plot-benchmarks/src/schema.ts b/development/plot-benchmarks/src/types/benchmark.ts similarity index 100% rename from development/plot-benchmarks/src/schema.ts rename to development/plot-benchmarks/src/types/benchmark.ts
diff --git a/development/plot-benchmarks/src/types/chart.ts b/development/plot-benchmarks/src/types/chart.ts new file mode 100644 index 0000000..0d77b63 --- /dev/null +++ b/development/plot-benchmarks/src/types/chart.ts
@@ -0,0 +1,27 @@ +import type { ChartDataset, ChartType, Point } from "chart.js"; + +/** + * The chart data container. + * + * Has relevant default X-Axis labels & corresponding datasets. + */ +export interface Data { + // X-axis labels. + labels: number[]; + // The corresponding datasets. + datasets: ChartDataset[]; +} + +/** + * A single data-series being rendered in the chart. + * + * Used by a Mapper for data transformations. + */ +export interface Series { + label: string; + type: ChartType; + data: Array<Point>; + // Additional series options + // For e.g. https://www.chartjs.org/docs/latest/charts/line.html + options: object; +}
diff --git a/development/plot-benchmarks/src/types/data.ts b/development/plot-benchmarks/src/types/data.ts new file mode 100644 index 0000000..6e6955c --- /dev/null +++ b/development/plot-benchmarks/src/types/data.ts
@@ -0,0 +1,28 @@ +/** + * A container for raw benchmark data. + * + * Can be extended to store descriptive statistics about the raw data. + */ +export interface ChartData<T> { + values: T[]; +} + +/** + * A container for a Metric. + * + * This metric has all relevant comparables, in the data keyed by the source. + */ +export interface Metric<T> { + class: string; + benchmark: string; + label: string; + data: Record<string, ChartData<T>>; +} + +/** + * A container for standard and sampled `Metric` instances. + */ +export interface Metrics<T = number> { + standard?: Metric<T>[]; + sampled?: Metric<T[]>[]; +}
diff --git a/development/plot-benchmarks/src/types/events.ts b/development/plot-benchmarks/src/types/events.ts new file mode 100644 index 0000000..48c4777 --- /dev/null +++ b/development/plot-benchmarks/src/types/events.ts
@@ -0,0 +1,5 @@ +import type { FileMetadata } from "./files.js"; + +export interface FileMetadataEvent { + entries: Array<FileMetadata>; +}
diff --git a/development/plot-benchmarks/src/types/files.ts b/development/plot-benchmarks/src/types/files.ts new file mode 100644 index 0000000..930369e --- /dev/null +++ b/development/plot-benchmarks/src/types/files.ts
@@ -0,0 +1,10 @@ +import type { Benchmarks } from "./benchmark.js"; + +/** + * File information + associated benchmarks. + */ +export interface FileMetadata { + file: File; + enabled: boolean; + container: Benchmarks; +}
diff --git a/development/plot-benchmarks/src/wrappers/benchmarks.ts b/development/plot-benchmarks/src/wrappers/benchmarks.ts new file mode 100644 index 0000000..9bbc015 --- /dev/null +++ b/development/plot-benchmarks/src/wrappers/benchmarks.ts
@@ -0,0 +1,54 @@ +import type { Benchmark, MetricsCollection, Sampled, Standard } from "../types/benchmark.js"; + +export class BenchmarkWrapper { + constructor( + private benchmark: Benchmark, + private separator: string = '_' + ) { + + } + + datasetName(): string { + return `${this.className()}${this.separator}${this.benchmark.name}`; + } + + metric(label: string): Standard | undefined { + return this.benchmark?.metrics[label]; + } + + sampled(label: string): Sampled | undefined { + return this.benchmark?.sampledMetrics[label]; + } + + metricLabels(): string[] { + return BenchmarkWrapper.labels(this.benchmark.metrics); + } + + sampledLabels(): string[] { + return BenchmarkWrapper.labels(this.benchmark.sampledMetrics); + } + + className(): string { + const className = this.benchmark.className; + const parts = className.split('.'); + const lastIndex = parts.length - 1; + return parts[lastIndex]; + } + + testName(): string { + return this.benchmark.name; + } + + private static labels(collection: MetricsCollection | null): string[] { + const labels: string[] = []; + if (collection) { + for (const key in collection) { + if (collection.hasOwnProperty(key)) { + labels.push(key); + } + } + } + return labels; + } + +}
diff --git a/development/plot-benchmarks/src/wrappers/session.ts b/development/plot-benchmarks/src/wrappers/session.ts new file mode 100644 index 0000000..2581f92 --- /dev/null +++ b/development/plot-benchmarks/src/wrappers/session.ts
@@ -0,0 +1,72 @@ +import { type FileMetadata } from "../types/files.js"; +import { BenchmarkWrapper } from "./benchmarks.js"; + +export interface Indexed<T> { + // The source of the benchmark. + source: string; + // `source` index. + index: number; + // The underlying type. + value: T; +} + +export type IndexedWrapper = Indexed<BenchmarkWrapper>; + +/** + * A Benchmarking plotting session. + */ +export class Session { + // className -> BenchmarkWrapper[] + public classGroups: Record<string, IndexedWrapper[]>; + // BenchmarkWrapper[] + public benchmarks: IndexedWrapper[]; + public fileNames: Set<string>; + + constructor(public files: FileMetadata[]) { + this.initialize(); + } + + private initialize() { + this.classGroups = {}; + this.fileNames = new Set(); + this.benchmarks = []; + + for (let i = 0; i < this.files.length; i += 1) { + const fileIndex = i; + const fileMeta = this.files[fileIndex]; + for (let j = 0; j < fileMeta.container.benchmarks.length; j += 1) { + const wrapper = new BenchmarkWrapper(fileMeta.container.benchmarks[j]); + const fileGroupKey = wrapper.className(); + const classGroup = this.classGroups[fileGroupKey] || []; + const item = { + source: fileMeta.file.name, + index: fileIndex, + value: wrapper + }; + classGroup.push(item); + // Update + this.classGroups[fileGroupKey] = classGroup; + this.fileNames.add(fileMeta.file.name); + this.benchmarks.push(item); + } + } + } + + static datasetNames(wrappers: IndexedWrapper[]): Set<string> { + const names = new Set<string>(); + for (let i = 0; i < wrappers.length; i += 1) { + const wrapper = wrappers[i]; + names.add(wrapper.value.datasetName()); + } + return names; + } + + static sources(wrappers: IndexedWrapper[]): Set<string> { + const sources = new Set<string>(); + for (let i = 0; i < wrappers.length; i += 1) { + sources.add(wrappers[i].source); + } + return sources; + } + +}