benchmarks/functional_autograd_benchmark/compare.py

import argparse
from collections import defaultdict

from utils import from_markdown_table, to_markdown_table


def main():
    parser = argparse.ArgumentParser(
        "Main script to compare results from the benchmarks"
    )
    parser.add_argument(
        "--before",
        type=str,
        default="before.txt",
        help="Text file containing the times to use as base",
    )
    parser.add_argument(
        "--after",
        type=str,
        default="after.txt",
        help="Text file containing the times to use as new version",
    )
    parser.add_argument(
        "--output", type=str, default="", help="Text file where to write the output"
    )
    args = parser.parse_args()

    with open(args.before) as f:
        content = f.read()
    res_before = from_markdown_table(content)

    with open(args.after) as f:
        content = f.read()
    res_after = from_markdown_table(content)

    diff = defaultdict(defaultdict)
    for model in res_before:
        for task in res_before[model]:
            mean_before, var_before = res_before[model][task]
            if task not in res_after[model]:
                diff[model][task] = (None, mean_before, var_before, None, None)
            else:
                mean_after, var_after = res_after[model][task]
                diff[model][task] = (
                    mean_before / mean_after,
                    mean_before,
                    var_before,
                    mean_after,
                    var_after,
                )
    for model in res_after:
        for task in res_after[model]:
            if task not in res_before[model]:
                mean_after, var_after = res_after[model][task]
                diff[model][task] = (None, None, None, mean_after, var_after)

    header = (
        "model",
        "task",
        "speedup",
        "mean (before)",
        "var (before)",
        "mean (after)",
        "var (after)",
    )
    out = to_markdown_table(diff, header=header)

    print(out)
    if args.output:
        with open(args.output, "w") as f:
            f.write(out)


if __name__ == "__main__":
    main()