feat: compute and display median and median absolute deviation (#89)

Signed-off-by: Jérôme Benoit <[email protected]>
tinylibs · Oct 13, 2024 · 46fbd0c · 46fbd0c
1 parent ab467e2
commit 46fbd0c
Show file tree

Hide file tree

Showing 6 changed files with 74 additions and 16 deletions.
diff --git a/README.md b/README.md
@@ -53,12  53,12 @@ await bench.run();
 console.table(bench.table());
 
 // Output:
-// ┌─────────┬───────────────┬──────────┬────────────────────┬───────────┬─────────┐
-// │ (index) │   Task Name   │ ops/sec  │ Average Time (ns)  │  Margin   │ Samples │
-// ├─────────┼───────────────┼──────────┼────────────────────┼───────────┼─────────┤
-// │    0    │ 'faster task' │ '41,621' │ 24025.791819761525 │ '±20.50%' │  4257   │
-// │    1    │ 'slower task' │  '828'   │ 1207382.7838323202 │ '±7.07%'  │   83    │
-// └─────────┴───────────────┴──────────┴────────────────────┴───────────┴─────────┘
 // ┌─────────┬───────────────┬──────────┬──────────────────────┬──────────┬──────────────────────────────────┬─────────┐
 // │ (index) │ Task name     │ ops/sec  │ Average time/op (ns) │ Margin   │ Median time/op (ns)              │ Samples │
 // ├─────────┼───────────────┼──────────┼──────────────────────┼──────────┼──────────────────────────────────┼─────────┤
 // │ 0       │ 'faster task' │ '38,832' │ 25751.297631307978   │ '±3.48%' │ '22016.49999997812±5.5000000145' │ 3884    │
 // │ 1       │ 'slower task' │ '669'    │ 1493338.567164177    │ '±5.98%' │ '1445076.0000000286'             │ 67      │
 // └─────────┴───────────────┴──────────┴──────────────────────┴──────────┴──────────────────────────────────┴─────────┘
 
 console.table(bench.table((task) => ({ 'Task name': task.name })));
 
@@ -282,6  282,16 @@ export type TaskResult = {
    */
   rme: number;
 
   /**
    * median absolute deviation
    */
   mad: number;
 
   /**
    * p50/median percentile
    */
   p50: number;
 
   /**
    * p75 percentile
    */

diff --git a/examples/src/simple.ts b/examples/src/simple.ts
@@ -18,12  18,12 @@ await bench.run();
 console.table(bench.table());
 
 // Output:
-// ┌─────────┬───────────────┬──────────┬────────────────────┬───────────┬─────────┐
-// │ (index) │   Task Name   │ ops/sec  │ Average Time (ns)  │  Margin   │ Samples │
-// ├─────────┼───────────────┼──────────┼────────────────────┼───────────┼─────────┤
-// │    0    │ 'faster task' │ '41,621' │ 24025.791819761525 │ '±20.50%' │  4257   │
-// │    1    │ 'slower task' │  '828'   │ 1207382.7838323202 │ '±7.07%'  │   83    │
-// └─────────┴───────────────┴──────────┴────────────────────┴───────────┴─────────┘
 // ┌─────────┬───────────────┬──────────┬──────────────────────┬──────────┬──────────────────────────────────┬─────────┐
 // │ (index) │ Task name     │ ops/sec  │ Average time/op (ns) │ Margin   │ Median time/op (ns)              │ Samples │
 // ├─────────┼───────────────┼──────────┼──────────────────────┼──────────┼──────────────────────────────────┼─────────┤
 // │ 0       │ 'faster task' │ '38,832' │ 25751.297631307978   │ '±3.48%' │ '22016.49999997812±5.5000000145' │ 3884    │
 // │ 1       │ 'slower task' │ '669'    │ 1493338.567164177    │ '±5.98%' │ '1445076.0000000286'             │ 67      │
 // └─────────┴───────────────┴──────────┴──────────────────────┴──────────┴──────────────────────────────────┴─────────┘
 
 console.table(
   bench.todos.map(({ name }) => ({

diff --git a/src/bench.ts b/src/bench.ts
@@ -242,10  242,11 @@ export default class Bench extends EventTarget {
           throw task.result.error;
         }
         return convert?.(task) || {
-          'Task Name': task.name,
           'Task name': task.name,
           'ops/sec': task.result.error ? 'NaN' : parseInt(task.result.hz.toString(), 10).toLocaleString(),
-          'Average Time (ns)': task.result.error ? 'NaN' : task.result.mean * 1000 * 1000,
           'Average time/op (ns)': task.result.error ? 'NaN' : task.result.mean * 1e6,
           Margin: task.result.error ? 'NaN' : `\xb1${task.result.rme.toFixed(2)}%`,
           'Median time/op (ns)': task.result.error ? 'NaN' : `${task.result.p50 * 1e6}${task.result.mad * 1e6 > 0 ? `\xb1${(task.result.mad * 1e6).toFixed(10)}` : ''}`,
           Samples: task.result.error ? 'NaN' : task.result.samples.length,
         };
       }

diff --git a/src/task.ts b/src/task.ts
@@ -10,7  10,13 @@ import Bench from './bench';
 import tTable from './constants';
 import { createBenchEvent } from './event';
 import { AddEventListenerOptionsArgument, RemoveEventListenerOptionsArgument } from './types';
-import { getVariance, isAsyncTask, quantileSorted } from './utils';
 import {
   absoluteDeviation,
   getVariance,
   isAsyncTask,
   medianSorted,
   quantileSorted,
 } from './utils';
 
 /**
  * A class that represents each benchmark task in Tinybench. It keeps track of the
@@ -152,6  158,9 @@ export default class Task extends EventTarget {
       const moe = sem * critical;
       const rme = (moe / mean) * 100;
 
       const mad = absoluteDeviation(samples, medianSorted);
 
       const p50 = medianSorted(samples);
       const p75 = quantileSorted(samples, 0.75);
       const p99 = quantileSorted(samples, 0.99);
       const p995 = quantileSorted(samples, 0.995);
@@ -176,6  185,8 @@ export default class Task extends EventTarget {
         critical,
         moe,
         rme,
         mad,
         p50,
         p75,
         p99,
         p995,

diff --git a/src/types.ts b/src/types.ts
@@ -105,6  105,16 @@ export type TaskResult = {
    */
   rme: number;
 
   /**
    * median absolute deviation
    */
   mad: number;
 
   /**
    * p50/median percentile
    */
   p50: number;
 
   /**
    * p75 percentile
    */

diff --git a/src/utils.ts b/src/utils.ts
@@ -19,7  19,7 @@ function isPromiseLike<T>(maybePromiseLike: any): maybePromiseLike is PromiseLik
 const AsyncFunctionConstructor = (async () => {}).constructor;
 
 /**
- * an async function check method only consider runtime support async syntax
  * An async function check method only consider runtime support async syntax
  */
 export const isAsyncFunction = (fn: Fn) => fn.constructor === AsyncFunctionConstructor;
 
@@ -113,3  113,29 @@ export const quantileSorted = (samples: number[], q: number) => {
   }
   return samples[baseIndex];
 };
 
 /**
  * Computes the median of a sorted sample.
  *
  * @param samples the sorted sample
  * @returns the median of the sample
  */
 export const medianSorted = (samples: number[]) => quantileSorted(samples, 0.5);
 
 /**
  * Computes the absolute deviation of a sample given an aggregation.
  *
  * @param samples the sample
  * @param aggFn the aggregation function to use
  * @returns the absolute deviation of the sample given the aggregation
  */
 export const absoluteDeviation = (samples: number[], aggFn: (arr: number[]) => number | undefined) => {
   const value = aggFn(samples);
   const absoluteDeviations: number[] = [];
 
   for (const sample of samples) {
     absoluteDeviations.push(Math.abs(sample - value!));
   }
 
   return aggFn(absoluteDeviations);
 };