Add plots for statistical significance of measurements.
* This is useful when comparing benchmark runs.
* Inspired by [Statistics for Hackers](https://youtu.be/Iq9DzN6mvYA?t=800)
Change-Id: I4f2bfdf174263e1fa1542e115c0caabcafdb53b3
Test: `npm run-script dev`
diff --git a/development/plot-benchmarks/src/lib/App.svelte b/development/plot-benchmarks/src/lib/App.svelte
index ffa3e57..f1ace88 100644
--- a/development/plot-benchmarks/src/lib/App.svelte
+++ b/development/plot-benchmarks/src/lib/App.svelte
@@ -3,9 +3,17 @@
import { writable } from "svelte/store";
import type { FileMetadata } from "../types/files.js";
import Session from "./Session.svelte";
+ import { wrap } from "comlink";
+ import { StatService } from "../workers/service.js";
// Stores
let entries: Writable<FileMetadata[]> = writable([]);
+ const url = new URL("../workers/worker.ts", import.meta.url);
+ const service = wrap<StatService>(
+ new Worker(url, {
+ type: "module",
+ })
+ );
function onFilesChanged(event) {
const detail: FileMetadata[] = event.detail;
@@ -22,5 +30,5 @@
</details>
<div class="container">
- <Session fileEntries={$entries} on:entries={onFilesChanged} />
+ <Session fileEntries={$entries} {service} on:entries={onFilesChanged} />
</div>
diff --git a/development/plot-benchmarks/src/lib/Chart.svelte b/development/plot-benchmarks/src/lib/Chart.svelte
index 7a063a8..9a224e2 100644
--- a/development/plot-benchmarks/src/lib/Chart.svelte
+++ b/development/plot-benchmarks/src/lib/Chart.svelte
@@ -10,6 +10,7 @@
export let data: Data;
export let chartType: ChartType = "line";
+ export let isExperimental: boolean = false;
$: {
if ($chart) {
@@ -27,7 +28,9 @@
onMount(() => {
const onUpdate = (chart: Chart) => {
$chart = chart;
- $items = chart.options.plugins.legend.labels.generateLabels(chart);
+ // Bad typings.
+ const legend = chart.options.plugins.legend as any;
+ $items = legend.labels.generateLabels(chart);
};
const plugins = {
legend: {
@@ -65,7 +68,14 @@
⎘
</button>
</div>
- <canvas id="chart" class="chart" bind:this={element} />
+ <canvas class="chart" bind:this={element} />
+ {#if isExperimental}
+ <footer class="slim">
+ <section class="experimental">
+ <kbd>Experimental</kbd>
+ </section>
+ </footer>
+ {/if}
</article>
{#if $items}
@@ -89,4 +99,17 @@
border: none;
padding: 5px;
}
+
+ .slim {
+ margin-bottom: 0px;
+ padding: 0;
+ }
+
+ .experimental {
+ display: flex;
+ flex-direction: row;
+ flex-wrap: nowrap;
+ justify-content: center;
+ margin-bottom: 0px;
+ }
</style>
diff --git a/development/plot-benchmarks/src/lib/Dataset.svelte b/development/plot-benchmarks/src/lib/Dataset.svelte
index a19eff0..6e81cc0 100644
--- a/development/plot-benchmarks/src/lib/Dataset.svelte
+++ b/development/plot-benchmarks/src/lib/Dataset.svelte
@@ -1,14 +1,23 @@
<script lang="ts">
import { createEventDispatcher } from "svelte";
import { Session, type IndexedWrapper } from "../wrappers/session.js";
- import type { SelectionEvent, Selection } from "../types/events.js";
+ import type {
+ SelectionEvent,
+ Selection,
+ StatEvent,
+ StatInfo,
+ StatType,
+ } from "../types/events.js";
export let name: string;
export let datasetGroup: IndexedWrapper[];
+ // Dispatchers
+ let selectionDispatcher = createEventDispatcher<SelectionEvent>();
+ let statDispatcher = createEventDispatcher<StatEvent>();
// State
- let dispatcher = createEventDispatcher<SelectionEvent>();
let selected: boolean = true;
+ let compute: boolean = false;
let sources: Set<string>;
let sampledMetrics: Set<string>;
let metrics: Set<string>;
@@ -22,7 +31,21 @@
name: name,
enabled: selected,
};
- dispatcher("selections", [selection]);
+ selectionDispatcher("selections", [selection]);
+ };
+
+ let stat = function (type: StatType) {
+ return function (event: Event) {
+ event.stopPropagation();
+ const target = event.target as HTMLInputElement;
+ compute = target.checked;
+ const stat: StatInfo = {
+ name: name,
+ type: type,
+ enabled: compute
+ };
+ statDispatcher("info", [stat]);
+ };
};
$: {
@@ -45,16 +68,32 @@
<hgroup>
<div class="section">
<span class="item">{name}</span>
- <fieldset class="item">
- <label for="switch">
- <input
- type="checkbox"
- role="switch"
- checked={selected}
- on:change={selection}
- />
- </label>
- </fieldset>
+ <div class="item actions">
+ <fieldset>
+ <label for="switch">
+ Show
+ <input
+ type="checkbox"
+ role="switch"
+ checked={selected}
+ on:change={selection}
+ />
+ </label>
+ </fieldset>
+ {#if sources.size > 1}
+ <fieldset>
+ <label for="switch">
+ P
+ <input
+ type="checkbox"
+ role="switch"
+ checked={compute}
+ on:change={stat("p")}
+ />
+ </label>
+ </fieldset>
+ {/if}
+ </div>
</div>
<div class="details">
<div class="sources">
@@ -103,4 +142,14 @@
.section .item {
margin: 0px 10px;
}
+
+ .actions {
+ display: flex;
+ flex-direction: row;
+ justify-content: flex-end;
+ }
+
+ .actions fieldset {
+ margin-left: 5px;
+ }
</style>
diff --git a/development/plot-benchmarks/src/lib/Group.svelte b/development/plot-benchmarks/src/lib/Group.svelte
index e538e68..6dcee76 100644
--- a/development/plot-benchmarks/src/lib/Group.svelte
+++ b/development/plot-benchmarks/src/lib/Group.svelte
@@ -1,18 +1,27 @@
<script lang="ts">
import { createEventDispatcher } from "svelte";
- import type { Selection, SelectionEvent } from "../types/events.js";
+ import type {
+ Selection,
+ SelectionEvent,
+ StatEvent,
+ StatInfo,
+ } from "../types/events.js";
import { Session, type IndexedWrapper } from "../wrappers/session.js";
import Dataset from "./Dataset.svelte";
export let className: string;
export let datasetGroup: IndexedWrapper[];
- let dispatcher = createEventDispatcher<SelectionEvent>();
+ let selectionDispatcher = createEventDispatcher<SelectionEvent>();
+ let statDispatcher = createEventDispatcher<StatEvent>();
let datasetNames: Set<string>;
+ // Forward events.
let selection = function (event: CustomEvent<Selection[]>) {
- // Forward events.
- dispatcher("selections", event.detail);
+ selectionDispatcher("selections", event.detail);
+ };
+ let stat = function (event: CustomEvent<StatInfo[]>) {
+ statDispatcher("info", event.detail);
};
$: {
@@ -24,7 +33,7 @@
<summary>{className}</summary>
<div class="details">
{#each datasetNames as name (name)}
- <Dataset {datasetGroup} {name} on:selections={selection} />
+ <Dataset {datasetGroup} {name} on:selections={selection} on:info={stat} />
{/each}
</div>
</details>
diff --git a/development/plot-benchmarks/src/lib/Session.svelte b/development/plot-benchmarks/src/lib/Session.svelte
index b751f67..3a0a062 100644
--- a/development/plot-benchmarks/src/lib/Session.svelte
+++ b/development/plot-benchmarks/src/lib/Session.svelte
@@ -1,19 +1,31 @@
<script lang="ts">
import { createEventDispatcher } from "svelte";
- import { writable, type Writable } from "svelte/store";
+ import {
+ writable,
+ type Readable,
+ type Writable,
+ derived,
+ } from "svelte/store";
import { readBenchmarks } from "../files.js";
import { ChartDataTransforms } from "../transforms/data-transforms.js";
import { Transforms } from "../transforms/metric-transforms.js";
import { STANDARD_MAPPER } from "../transforms/standard-mappers.js";
import type { Data, Series } from "../types/chart.js";
import type { Metrics } from "../types/data.js";
- import type { FileMetadataEvent, Selection } from "../types/events.js";
+ import type {
+ FileMetadataEvent,
+ Selection,
+ StatInfo,
+ } from "../types/events.js";
import type { FileMetadata } from "../types/files.js";
import { Session, type IndexedWrapper } from "../wrappers/session.js";
import Chart from "./Chart.svelte";
import Group from "./Group.svelte";
+ import type { StatService } from "../workers/service.js";
+ import type { Remote } from "comlink";
export let fileEntries: FileMetadata[];
+ export let service: Remote<StatService>;
// State
let eventDispatcher = createEventDispatcher<FileMetadataEvent>();
@@ -23,13 +35,23 @@
let chartData: Data;
let classGroups: Record<string, IndexedWrapper[]>;
let size: number;
+ let activeSeries: Promise<Series[]>;
// Stores
let activeDragDrop: Writable<boolean> = writable(false);
let suppressed: Writable<Set<string>> = writable(new Set());
+ let activeStats: Writable<StatInfo[]> = writable([]);
+ let active: Readable<Set<string>> = derived(activeStats, ($activeStats) => {
+ const datasets = [];
+ for (let i = 0; i < $activeStats.length; i += 1) {
+ const activeStat = $activeStats[i];
+ datasets.push(activeStat.name);
+ }
+ return new Set(datasets);
+ });
// Events
- let handler = function (event: CustomEvent<Selection[]>) {
+ let selectionHandler = function (event: CustomEvent<Selection[]>) {
const selections: Selection[] = event.detail;
for (let i = 0; i < selections.length; i += 1) {
const selection = selections[i];
@@ -42,9 +64,28 @@
$suppressed = $suppressed;
};
+ let statHandler = function (event: CustomEvent<StatInfo[]>) {
+ const statistics = event.detail;
+ for (let i = 0; i < statistics.length; i += 1) {
+ const statInfo = statistics[i];
+ if (!statInfo.enabled) {
+ const index = $activeStats.findIndex(
+ (entry) => entry.name == statInfo.name && entry.type == statInfo.type
+ );
+ if (index >= 0) {
+ $activeStats.splice(index, 1);
+ }
+ } else {
+ $activeStats.push(statInfo);
+ }
+ $activeStats = $activeStats;
+ }
+ };
+
$: {
session = new Session(fileEntries);
metrics = Transforms.buildMetrics(session, $suppressed);
+ activeSeries = service.pSeries(metrics, $active);
series = ChartDataTransforms.mapToSeries(metrics, STANDARD_MAPPER);
chartData = ChartDataTransforms.mapToDataset(series);
classGroups = session.classGroups;
@@ -115,13 +156,26 @@
>
<h5>Benchmarks</h5>
{#each Object.entries(classGroups) as [className, wrappers]}
- <Group {className} datasetGroup={wrappers} on:selections={handler} />
+ <Group
+ {className}
+ datasetGroup={wrappers}
+ on:selections={selectionHandler}
+ on:info={statHandler}
+ />
{/each}
</article>
{#if series.length > 0}
<Chart data={chartData} />
{/if}
+
+ {#await activeSeries}
+ <article aria-busy="true" />
+ {:then chartData}
+ {#if chartData.length > 0}
+ <Chart data={ChartDataTransforms.mapToDataset(chartData)} isExperimental={true} />
+ {/if}
+ {/await}
{/if}
<style>
