New versions of twoway container benchmark graphs.
Remove site-specific details from nightly.py.
Replace uplevel vars of nested functions by class object.
Signed-off-by: Duane Sand <[email protected]>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@3593 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/tko/nightly.py b/tko/nightly.py
index c431829..5a08608 100644
--- a/tko/nightly.py
+++ b/tko/nightly.py
@@ -1,4 +1,4 @@
-# tko/nightly.py code shared by various *_nightly.cgi scripts
+# tko/nightly.py code shared by various tko/*.cgi graphing scripts
import cgi, cgitb
import os, sys
@@ -7,370 +7,780 @@
from autotest_lib.client.common_lib import kernel_versions
-def nightly_views(suite_notes, kernel_legend, benchmarks,
- released_kernel_series,
- nightly_kernel_series,
- smp = 'smp',
- test_group='Kernel_Qual_Containers',
- tko_mysql_server='autotest', results_server='autotest',
- max_rel_kernels=8, max_dev_kernels=5):
-
- test_runs = {} # kernel --> (platform --> list of test runs)
- job_table = {} # kernel id --> list of job idxs
- kernel_dates = {} # Kernel id --> date of nightly test
+def add_kernel_jobs(label_pattern):
+ cmd = "select job_idx from jobs where label like '%s'" % label_pattern
+ nrows = perf.db_cur.execute(cmd)
+ return [row[0] for row in perf.db_cur.fetchall()]
- def add_kernel_jobs(label_pattern):
- cmd = "select job_idx from jobs where label like '%s'" % label_pattern
- nrows = perf.db_cur.execute(cmd)
- return [row[0] for row in perf.db_cur.fetchall()]
+def is_filtered_platform(platform, platforms_filter):
+ if not platforms_filter:
+ return True
+ for p in platforms_filter:
+ if platform.startswith(p):
+ return True
+ return False
- def is_filtered_platform(platform):
- for p in platforms_filter.split(','):
- if platform.startswith(p):
- return True
- return False
+def get_test_attributes(testrunx):
+ cmd = ( "select attribute, value from test_attributes"
+ " where test_idx = %d" % testrunx )
+ nrows = perf.db_cur.execute(cmd)
+ return dict(perf.db_cur.fetchall())
- def collect_testruns(jobs, regressed_platforms, test):
- # get test_runs run #s for 1 test on 1 kernel and some platforms
- # TODO: Is jobs list short enough to use directly in 1 sql cmd?
- runs = {} # platform --> list of test runs
- for jobx in jobs:
- cmd = ( "select test_idx, machine_idx from tests"
- " where job_idx = %s and test = %s" )
- args = [jobx, test]
- nrows = perf.db_cur.execute(cmd, args)
- for testrunx, machx in perf.db_cur.fetchall():
- platform, host = perf.machine_idx_to_platform_host(machx)
- if by_hosts:
- platform += '.'+host
- if ( is_filtered_platform(platform) and
- (regressed_platforms is None
- or platform in regressed_platforms) ):
- runs.setdefault(platform, []).append(testrunx)
- return runs
+def get_antag(testrunx):
+ attrs = get_test_attributes(testrunx)
+ return attrs.get('antag', None)
- def collect_all_testruns(trimmed_kernels, regressed_platforms, test):
- # get test_runs run #s for 1 test on some kernels and platforms
- for kernel in trimmed_kernels:
- runs = collect_testruns(job_table[kernel], regressed_platforms, test)
- if runs:
- test_runs[kernel] = runs
+def matching_test_attributes(attrs, required_test_attributes):
+ if not required_test_attributes:
+ return True
+ matches = [attrs[key] == required_test_attributes[key]
+ for key in attrs if key in required_test_attributes]
+ return min(matches+[True]) # True if all jointly-existing keys matched
- def collect_raw_scores(runs, metric):
- # get unscaled scores of test runs for 1 test on certain jobs
- # arrange them by platform type
- platform_scores = {} # platform --> list of perf scores
- for platform in runs:
- vals = perf.get_metric_at_point(runs[platform], metric)
+def collect_testruns(jobs, test, test_attributes,
+ platforms_filter, by_hosts, no_antag):
+ # get test_runs run #s for 1 test on 1 kernel and some platforms
+ # TODO: Is jobs list short enough to use directly in 1 sql cmd?
+ # TODO: add filtering on test series?
+ runs = {} # platform --> list of test runs
+ for jobx in jobs:
+ cmd = ( "select test_idx, machine_idx from tests"
+ " where job_idx = %s and test = %s" )
+ args = [jobx, test]
+ nrows = perf.db_cur.execute(cmd, args)
+ for testrunx, machx in perf.db_cur.fetchall():
+ platform, host = perf.machine_idx_to_platform_host(machx)
+ if by_hosts:
+ platform += '.'+host
+ if ( is_filtered_platform(platform, platforms_filter) and
+ matching_test_attributes(get_test_attributes(testrunx),
+ test_attributes) and
+ (not no_antag or get_antag(testrunx) == '') ):
+ runs.setdefault(platform, []).append(testrunx)
+ return runs
+
+
+def all_tested_platforms(test_runs):
+ # extract list of all tested platforms from test_runs table
+ platforms = set()
+ for kernel in test_runs:
+ platforms.update(set(test_runs[kernel].keys()))
+ return sorted(platforms)
+
+
+def divide_twoway_testruns(test_runs, platform):
+ # partition all twoway runs based on name of antagonist progs
+ twoway_runs = {}
+ antagonists = set()
+ for kernel in test_runs:
+ runs = {}
+ for testrunx in test_runs[kernel].get(platform, []):
+ antag = get_antag(testrunx)
+ if antag is not None:
+ runs.setdefault(antag, []).append(testrunx)
+ antagonists.add(antag)
+ twoway_runs[kernel] = runs
+ return twoway_runs, sorted(antagonists)
+
+
+def collect_raw_scores(runs, metric):
+ # get unscaled scores of test runs for 1 test on certain jobs
+ # arrange them by platform type
+ platform_scores = {} # platform --> list of perf scores
+ for platform in runs:
+ vals = perf.get_metric_at_point(runs[platform], metric)
+ if vals:
+ platform_scores[platform] = vals
+ return platform_scores
+
+
+def collect_scaled_scores(metric, test_runs, regressed_platforms, relative):
+ # get scores of test runs for 1 test on some kernels and platforms
+ # optionally make relative to oldest (?) kernel on that platform
+ # arrange by plotline (ie platform) for gnuplot
+ plot_data = {} # platform --> (kernel --> list of perf scores)
+ baseline = {}
+ for kernel in sorted(test_runs.keys()):
+ for platform in test_runs[kernel]:
+ if not (regressed_platforms is None or
+ platform in regressed_platforms):
+ continue # delete results for uninteresting platforms
+ vals = perf.get_metric_at_point(test_runs[kernel][platform],
+ metric)
if vals:
- platform_scores[platform] = vals
- return platform_scores
+ if relative:
+ if platform not in baseline:
+ baseline[platform], std = plotgraph.avg_dev(vals)
+ vals = [v/baseline[platform] for v in vals]
+ pdp = plot_data.setdefault(platform, {})
+ pdp.setdefault(kernel, []).extend(vals)
+ return plot_data
- def collect_scaled_scores(metric):
- # get scores of test runs for 1 test on some kernels and platforms
- # optionally make relative to first kernel on that platform
- # arrange by plotline (ie platform) for gnuplot
- plot_data = {} # platform --> (kernel --> list of perf scores)
- baseline = {}
- for kernel in sorted(test_runs.keys()):
- for platform in test_runs[kernel]:
- vals = perf.get_metric_at_point(test_runs[kernel][platform], metric)
- if vals:
- if relative:
- if platform not in baseline:
- baseline[platform], std = plotgraph.avg_dev(vals)
- vals = [v/baseline[platform] for v in vals]
- pdp = plot_data.setdefault(platform, {})
- pdp.setdefault(kernel, []).extend(vals)
- return plot_data
-
-
- def find_regressions(kernels, test, metric):
- # A test is regressed on some platform if its latest results are
- # definitely lower than on the reference kernel.
- # Runs for the latest kernel may be underway and incomplete.
- # In that case, selectively use next-latest kernel.
- if not regress:
- return None
- ref = kernels[0]
- latest = kernels[-1]
- prev = kernels[-2:][0]
- scores = {} # kernel --> (platform --> list of perf scores)
- for k in [ref, prev, latest]:
- runs = collect_testruns(job_table[k], None, test)
- scores[k] = collect_raw_scores(runs, metric)
- regressed_platforms = []
- for platform in scores[ref]:
- k = latest
- if platform not in scores[k]:
- k = prev
- if platform not in scores[k]:
- continue # perhaps due to decay of test machines
- ref_avg, ref_std = plotgraph.avg_dev(scores[ref][platform])
- avg, std = plotgraph.avg_dev(scores[ k ][platform])
- if avg+std < ref_avg-ref_std:
- regressed_platforms.append(platform)
- return sorted(regressed_platforms)
-
-
- def select_dev_kernels():
- # collect table of all tested kernels' test runs
- kernels = []
- for series in released_kernel_series:
- kernels += survey_all_kernels_tested(perf.db_cur, series+'.',
- '', smp, test_group,
- max_rel_kernels,
- job_table, kernel_dates)
- for series in nightly_kernel_series:
- kernels += survey_all_kernels_tested(perf.db_cur,
- '2.6.26-%s-' % series,
- series, smp, test_group,
- max_dev_kernels,
- job_table, kernel_dates)
- kernels = sort_kernels(kernels)
- return kernels # sorted subset of kernels in job_table
-
-
- def graph_1_test(test, metric, size):
- # generate graph image for one benchmark, showing avg and
- # std dev of one metric, over various kernels (X columns)
- # and various platform types (graphed lines)
- title = test.capitalize() + suite_notes
- if regress:
- title += ', Regressions Only'
+def collect_twoway_scores(metric, antagonists, twoway_runs, relative):
+ alone = ''
+ plot_data = {}
+ for kernel in twoway_runs:
+ for test2 in antagonists:
+ runs = twoway_runs[kernel].get(test2, [])
+ vals = perf.get_metric_at_point(runs, metric)
+ plot_data.setdefault(test2, {})
+ if vals:
+ plot_data[test2][kernel] = vals
if relative:
- ylegend = 'Relative '
- ymin = 0.9
- else:
- ylegend = ''
- ymin = None
- ylegend += metric.capitalize()
- graph = plotgraph.gnuplot(title, kernel_legend, ylegend, size=size,
- xsort=sort_kernels)
- for platform in platforms:
- graph.add_dataset(platform, plot_data[platform])
- graph.plot(cgi_header=True, ymin=ymin, dark=dark)
+ vals = plot_data[alone].get(kernel, [])
+ if vals:
+ baseline = perf.average(vals)
+ for test2 in antagonists:
+ vals = plot_data[test2].get(kernel, [])
+ vals = [val/baseline for val in vals]
+ if vals:
+ plot_data[test2][kernel] = vals
+ else:
+ for test2 in antagonists:
+ if kernel in plot_data[test2]:
+ del plot_data[test2][kernel]
+ return plot_data
- def table_for_1_test(test, metric):
- # generate detailed html page with graph plus numeric table for 1 benchmark
- print "Content-Type: text/html\n\n<html><body>"
- heading = "%s %s:  %s%s" % (test_group, kernel_legend,
- test.capitalize(), suite_notes)
- if regress:
- heading += ", Regressions Only"
- print "<h2> %s </h2>" % heading
- print "<img src='%s?%s'>" % (myself, '&'.join(passthru))
-
- heading = "%s %s metric" % (test.capitalize(), metric)
- if relative: heading += ", relative"
- print "<p><p> <h3> %s: </h3>" % heading
-
- ref_thresholds = {}
- print "<table border=1, cellpadding=3>"
- print "<tr> <td><b> Kernel </b></td>",
- for platform in platforms:
- p = platform.replace("_", "_<br>").replace(".", "<br>")
- print "<td><b>", p, "</b></td>"
- print "</tr>"
- for kernel in kernels:
- print "<tr> <td><b>", kernel, "</b><br><small>",
- print kernel_dates[kernel], "</small></td>"
- for platform in platforms:
- print "<td",
- vals = plot_data[platform].get(kernel, [])
- if vals:
- (avg, std_dev) = plotgraph.avg_dev(vals)
- if platform not in ref_thresholds:
- ref_thresholds[platform] = avg - std_dev
- if avg+std_dev < ref_thresholds[platform]:
- print "bgcolor=pink",
- print ( "> <a href='%s?test=%s&metric=%s"
- "&platforms=%s&runs&kernel=%s'>"
- % (myself, test, metric, platform, kernel) )
- print "<b>%.4g</b>" % avg, "</a><br>",
- print " <small> %dr </small>" % len(vals),
- print " <small> %.3g </small>" % std_dev,
- else:
- print "> ?",
- print "</td>"
- print "</tr>\n"
- print "</table>"
- print "<p> <b>Bold value:</b> Average of this metric, then <br>"
- print "number of good test runs, then standard deviation of those runs"
- print "<br> Pink if regressed from reference kernel"
- print "</body></html>"
+def find_regressions(kernels, test_runs, metric):
+ # A test is regressed on some platform if its latest results are
+ # definitely lower than on the reference kernel.
+ # Runs for the latest kernel may be underway and incomplete.
+ # In that case, selectively use next-latest kernel.
+ # TODO: the next-latest method hurts if latest run is not sorted last,
+ # or if there are several dev threads
+ ref = kernels[0]
+ latest = kernels[-1]
+ prev = kernels[-2:][0]
+ scores = {} # kernel --> (platform --> list of perf scores)
+ for k in [ref, prev, latest]:
+ if k in test_runs:
+ scores[k] = collect_raw_scores(test_runs[k], metric)
+ regressed_platforms = []
+ for platform in scores[ref]:
+ if latest in scores and platform in scores[latest]:
+ k = latest
+ elif prev in scores and platform in scores[prev]:
+ k = prev
+ else: # perhaps due to decay of test machines
+ k = ref # no regression info avail
+ ref_avg, ref_std = plotgraph.avg_dev(scores[ref][platform])
+ avg, std = plotgraph.avg_dev(scores[ k ][platform])
+ if avg+std < ref_avg-ref_std:
+ regressed_platforms.append(platform)
+ return sorted(regressed_platforms)
- def get_testrun_context(testrun):
- cmd = ( 'select jobs.tag, tests.subdir,'
- ' jobs.label, tests.started_time'
- ' from jobs, tests'
- ' where jobs.job_idx = tests.job_idx'
- ' and tests.test_idx = %d' % testrun )
- nrows = perf.db_cur.execute(cmd)
- assert nrows == 1
- row = perf.db_cur.fetchone()
- row = (row[0], row[1], row[2], row[3].strftime('%m/%d/%y %H:%M'))
- return row
+def get_testrun_context(testrun):
+ cmd = ( 'select jobs.label, jobs.tag, tests.subdir,'
+ ' tests.started_time'
+ ' from jobs, tests'
+ ' where jobs.job_idx = tests.job_idx'
+ ' and tests.test_idx = %d' % testrun )
+ nrows = perf.db_cur.execute(cmd)
+ assert nrows == 1
+ row = perf.db_cur.fetchone()
+ row = [row[0], row[1], row[2], row[3].strftime('%m/%d/%y %H:%M')]
+ return row
- def testrun_details_for_1_test_kernel_platform(test, metric, platform):
- kernel = form.getvalue('kernel')
- show_attrs = 'attrs' in form
- print "Content-Type: text/html\n\n<html><body>"
- print "<h3> %s %s: %s%s, Kernel %s on %s </h3>" % (
- test_group, kernel_legend, test.capitalize(), suite_notes, kernel, platform)
- print "<table border=1 cellpadding=4>"
- print "<tr><td> Test_idx </td> <td> %s metric </td>" % metric
- print "<td> Job tag </td> <td> Subdir </td> <td> Job label </td>"
- print "<td> Started_time </td>"
- if show_attrs:
- print "<td> Test attributes </td>"
- print "</tr>\n"
+def html_top():
+ print "Content-Type: text/html\n\n<html><body>"
- for testrunx in test_runs[kernel][platform]:
- print "<tr> <td>", testrunx, "</td>"
- print "<td>",
- vals = perf.get_metric_at_point([testrunx], metric)
- for v in vals:
- print "%.4g" % v,
+
+def abs_rel_link(myurl, passthru):
+ # link redraws current page with opposite absolute/relative choice
+ mod_passthru = passthru[:]
+ if 'absolute' in passthru:
+ mod_passthru.remove('absolute')
+ opposite = 'relative'
+ else:
+ mod_passthru.append('absolute')
+ opposite = 'absolute'
+ url = '%s?%s' % (myurl, '&'.join(mod_passthru))
+ return "<a href='%s'> %s </a>" % (url, opposite)
+
+
+def table_1_metric_all_kernels(plot_data, columns, column_argname,
+ kernels, kernel_dates,
+ myurl, filtered_passthru):
+ # generate html table of graph's numbers
+ # for 1 benchmark metric over all kernels (rows),
+ # over various platforms or various antagonists etc (cols).
+ ref_thresholds = {}
+ print "<table border=1 cellpadding=3 cellspacing=0>"
+ print "<tr> <td><b> Kernel </b></td>",
+ for label in columns:
+ if not label and column_argname == 'antag':
+ label = 'no antag'
+ print "<td><b>", label.replace('_', '<br>_'), "</b></td>"
+ print "</tr>"
+ for kernel in kernels:
+ print "<tr> <td><b>", kernel, "</b>",
+ if kernel in kernel_dates:
+ print "<br><small>", kernel_dates[kernel], "</small>"
+ print "</td>"
+ for col in columns:
+ print "<td",
+ vals = plot_data[col].get(kernel, [])
+ if not vals:
+ print "> ?",
+ else:
+ (avg, std_dev) = plotgraph.avg_dev(vals)
+ if col not in ref_thresholds:
+ ref_thresholds[col] = avg - std_dev
+ if avg+std_dev < ref_thresholds[col]:
+ print "bgcolor=pink",
+ print "> ",
+ args = filtered_passthru[:]
+ perf.append_cgi_args(args,
+ {column_argname:col, 'kernel':kernel})
+ print "<a href='%s?%s&runs&attrs'>" % (myurl,
+ '&'.join(args))
+ print "<b>%.4g</b>" % avg, "</a><br>",
+ print " <small> %dr </small>" % len(vals),
+ print " <small> %.3g </small>" % std_dev,
print "</td>"
- row = get_testrun_context(testrunx)
- print ( "<td> <a href='//%s/results/%s/%s/results'> %s </a></td>"
- % (results_server, row[0], row[1], row[0]) )
- for v in row[1:]:
- print "<td> %s </td>" % v
- if show_attrs:
- attrs = perf.get_test_attributes(testrunx)
- print "<td>",
- for attr in attrs:
- # if attr == "sysinfo-cmdline": continue
- # if attr[:4] == "svs-": continue
- val = attrs[attr]
- if len(val) > 40:
- val = val[:40-3] + "..."
- print "%s=%s" % (attr, val)
- print "</td>"
- print "</tr>\n"
- print "</table>"
+ print "</tr>\n"
+ print "</table>"
+ print "<p> <b>Bold value:</b> Average of this metric, then <br>"
+ print "number of good test runs, then standard deviation of those runs"
+ print "<br> Pink if regressed from reference kernel"
+
+
+def table_all_metrics_1_platform(test_runs, platform, relative):
+ # TODO: show std dev in cells
+ # can't mark regressions, since some metrics improve downwards
+ kernels = perf.sort_kernels(test_runs.keys())
+ scores = {}
+ attrs = set()
+ for kernel in kernels:
+ testruns = test_runs[kernel].get(platform, [])
+ if testruns:
+ d = perf.collect_all_metrics_scores(testruns)
+ scores[kernel] = d
+ attrs.update(set(d.keys()))
+ else:
+ print "No runs completed on", kernel, "<br>"
+ attrs = sorted(list(attrs))[:100]
+
+ print "<table border=1 cellpadding=4 cellspacing=0>"
+ print "<tr><td> Metric </td>"
+ for kernel in kernels:
+ kernel = kernel.replace("_", "_<br>")
+ print "<td>", kernel, "</td>"
+ print "</tr>"
+ for attr in attrs:
+ print "<tr>"
+ print "<td>", attr, "</td>"
+ baseline = None
+ for kernel in kernels:
+ print "<td>",
+ if kernel in scores and attr in scores[kernel]:
+ (avg, dev) = plotgraph.avg_dev(scores[kernel][attr])
+ if baseline and relative:
+ percent = (avg/baseline - 1)*100
+ print "%+.1f%%" % percent,
+ else:
+ baseline = avg
+ print "%.4g" % avg,
+ else:
+ print "?"
+ print "</td>"
+ print "</tr>"
+ print "</table>"
+
+
+def table_variants_all_tests(plot_data, columns, colkeys, benchmarks,
+ myurl, filtered_passthru):
+ # generate html table of graph's numbers
+ # for primary metric over all benchmarks (rows),
+ # on one platform and one kernel,
+ # over various combos of test run attribute constraints (cols).
+ ref_thresholds = {}
+ print "<table border=1 cellpadding=3 cellspacing=0>"
+ print "<tr> <td><b> Benchmark </b></td>",
+ for col in columns:
+ print "<td><b>", colkeys[col].replace(',', ',<br>'), "</b></td>"
+ print "</tr>"
+ for benchmark in benchmarks:
+ print "<tr> <td><b>", benchmark, "</b></td>"
+ for col in columns:
+ print "<td>",
+ vals = plot_data[col].get(benchmark, [])
+ if not vals:
+ print "?",
+ else:
+ (avg, std_dev) = plotgraph.avg_dev(vals)
+ args = filtered_passthru[:]
+ perf.append_cgi_args(args, {'test':benchmark})
+ for keyval in colkeys[col].split(','):
+ key, val = keyval.split('=', 1)
+ perf.append_cgi_args(args, {key:val})
+ print "<a href='%s?%s&runs&attrs'>" % (myurl,
+ '&'.join(args))
+ print "<b>%.4g</b>" % avg, "</a><br>",
+ print " <small> %dr </small>" % len(vals),
+ print " <small> %.3g </small>" % std_dev,
+ print "</td>"
+ print "</tr>\n"
+ print "</table>"
+ print "<p> <b>Bold value:</b> Average of this metric, then <br>"
+ print "number of good test runs, then standard deviation of those runs"
+
+
+def table_testrun_details(runs, metric, tko_server, show_attrs):
+ print "<table border=1 cellpadding=4 cellspacing=0>"
+ print "<tr><td> %s metric </td>" % metric
+ print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>"
+ print "<td> Started_time </td>"
+ if show_attrs:
+ print "<td> Test attributes </td>"
+ print "</tr>\n"
+
+ for testrunx in runs:
+ print "<tr> <td>",
+ vals = perf.get_metric_at_point([testrunx], metric)
+ for v in vals:
+ print "%.4g " % v,
+ print "</td>"
+ row = get_testrun_context(testrunx)
+ row[2] = ( "<a href='//%s/results/%s/%s/results/keyval'> %s </a>"
+ % (tko_server, row[1], row[2], row[2]) )
+ for v in row:
+ print "<td> %s </td>" % v
+ if show_attrs:
+ attrs = get_test_attributes(testrunx)
+ print "<td>",
+ for attr in sorted(attrs.keys()):
+ if attr == "sysinfo-cmdline": continue
+ if attr[:4] == "svs-": continue
+ val = attrs[attr]
+ if len(val) > 40:
+ val = val[:40-3] + "..."
+ print "%s=%s " % (attr, val)
+ print "</td>"
+ print "</tr>\n"
+ print "</table>"
+
+
+def overview_thumb(test, metric, myurl, passthru):
+ pass_ = passthru + ['test=%s' % test]
+ if metric:
+ pass_ += ['metric=%s' % metric]
+ pass_ = '&'.join(pass_)
+ print "<a href='%s?%s&table'>" % (myurl, pass_)
+ print " <img src='%s?%s&size=450,500'> </a>" % (myurl, pass_)
+ # embedded graphs fit 3 across on 1400x1050 laptop
+
+
+def graph_1_test(title, metric, plot_data, line_argname, lines,
+ kernel_legend, relative, size, dark=False):
+ # generate graph image for one benchmark, showing avg and
+ # std dev of one metric, over various kernels (X columns),
+ # over various platforms or antagonists etc (graphed lines)
+ xlegend = kernel_legend
+ ylegend = metric.capitalize()
+ if relative:
+ ylegend += ', Relative'
+ ymin = 0.8
+ else:
+ ymin = None
+ if len(lines) > 1:
+ keytitle = line_argname.capitalize() + ':'
+ else:
+ keytitle = ''
+ graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size,
+ xsort=perf.sort_kernels, keytitle=keytitle)
+ for line in lines:
+ label = line
+ if not label and line_argname == 'antag':
+ label = 'no antag'
+ graph.add_dataset(label, plot_data[line])
+ graph.plot(cgi_header=True, ymin=ymin, dark=dark)
+
+
+def graph_variants_all_tests(title, plot_data, linekeys, size, dark):
+ # generate graph image showing all benchmarks
+ # on one platform and one kernel,
+ # over various combos of test run attribute constraints (lines).
+ xlegend = "Benchmark"
+ ylegend = "Relative Perf"
+ graph = plotgraph.gnuplot(title, xlegend, ylegend, size=size)
+ for i in linekeys:
+ graph.add_dataset(linekeys[i], plot_data[i])
+ graph.plot(cgi_header=True, dark=dark, ymin=0.8)
+
+
+class generate_views(object):
+
+
+ def __init__(self, kernel_legend, benchmarks, test_group,
+ site_benchmark_metrics, tko_server,
+ jobs_selector, no_antag):
+ self.kernel_legend = kernel_legend
+ self.benchmarks = benchmarks
+ self.test_group = test_group
+ self.tko_server = tko_server
+ self.jobs_selector = jobs_selector
+ self.no_antag = no_antag
+
+ cgitb.enable()
+ test, antagonists = self.parse_most_cgi_args()
+
+ perf.init(tko_server=tko_server)
+ for b in site_benchmark_metrics:
+ perf.add_benchmark_main_metric(b, site_benchmark_metrics[b])
+
+ self.test_runs = {} # kernel --> (platform --> list of test runs)
+ self.job_table = {} # kernel id --> list of job idxs
+ self.kernel_dates = {} # kernel id --> date of nightly test
+
+ vary = self.cgiform.getlist('vary')
+ if vary:
+ platform = self.platforms_filter[0]
+ self.analyze_variants_all_tests_1_platform(platform, vary)
+ elif test:
+ self.analyze_1_test(test, antagonists)
+ else:
+ self.overview_page_all_tests(self.benchmarks, antagonists)
+
+
+ def collect_all_testruns(self, trimmed_kernels, test):
+ # get test_runs run #s for 1 test on some kernels and platforms
+ for kernel in trimmed_kernels:
+ runs = collect_testruns(self.job_table[kernel], test,
+ self.test_attributes, self.platforms_filter,
+ 'by_hosts' in self.toggles, self.no_antag)
+ if runs:
+ self.test_runs[kernel] = runs
+
+
+ def table_for_graph_1_test(self, title, metric, plot_data,
+ column_argname, columns, filtered_passthru):
+ # generate detailed html page with 1 graph and corresp numbers
+ # for 1 benchmark metric over all kernels (rows),
+ # over various platforms or various antagonists etc (cols).
+ html_top()
+ print '<h3> %s </h3>' % title
+ print ('%s, machine group %s on //%s server <br>' %
+ (self.kernel_legend, self.test_group, self.tko_server))
+ if self.test_tag:
+ print '%s test script series <br>' % self.test_tag[1:]
+
+ print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru))
+
+ link = abs_rel_link(self.myurl, self.passthru+['table'])
+ print "<p><p> <h4> Redraw this with %s performance? </h4>" % link
+
+ heading = "%s, %s metric" % (title, metric)
+ if self.relative:
+ heading += ", relative"
+ print "<p><p> <h3> %s: </h3>" % heading
+ table_1_metric_all_kernels(plot_data, columns, column_argname,
+ self.kernels, self.kernel_dates,
+ self.myurl, filtered_passthru)
print "</body></html>"
- def overview_thumb(test, metric=[]):
- pass_ = passthru + ['test=%s' % test]
- if metric:
- pass_ += ['metric=%s' % metric]
- pass_ = '&'.join(pass_)
- print "<a href='%s?%s&table'>" % (myself, pass_)
- print " <img src='%s?%s&size=650,600'> </a>" % (myself, pass_)
- # embedded graphs fit 2 across on 1400x1050 laptop
+ def graph_1_test_all_platforms(self, test, metric, platforms, plot_data):
+ # generate graph image for one benchmark
+ title = test.capitalize()
+ if 'regress' in self.toggles:
+ title += ' Regressions'
+ if 'table' in self.cgiform:
+ self.table_for_graph_1_test(title, metric, plot_data,
+ 'platforms', platforms,
+ filtered_passthru=self.passthru)
+ else:
+ graph_1_test(title, metric, plot_data, 'platforms', platforms,
+ self.kernel_legend, self.relative,
+ self.size, 'dark' in self.toggles)
- def overview_page(benchmarks):
+ def testrun_details(self, title, runs, metric):
+ html_top()
+ print '<h3> %s </h3>' % title
+ print ('%s, machine group %s on //%s server' %
+ (self.kernel_legend, self.test_group, self.tko_server))
+ if self.test_tag:
+ print '<br> %s test script series' % self.test_tag[1:]
+ print '<p>'
+ table_testrun_details(runs, metric,
+ self.tko_server, 'attrs' in self.cgiform)
+ print "</body></html>"
+
+
+ def testrun_details_for_1_test_kernel_platform(self, test,
+ metric, platform):
+ default_kernel = min(self.test_runs.keys())
+ kernel = self.cgiform.getvalue('kernel', default_kernel)
+ title = '%s on %s using %s' % (test.capitalize(), platform, kernel)
+ runs = self.test_runs[kernel].get(platform, [])
+ self.testrun_details(title, runs, metric)
+
+
+ def analyze_1_metric_all_platforms(self, test, metric):
+ if 'regress' in self.toggles:
+ regressed_platforms = find_regressions(self.kernels, self.test_runs,
+ metric)
+ else:
+ regressed_platforms = None
+ plot_data = collect_scaled_scores(metric, self.test_runs,
+ regressed_platforms, self.relative)
+ platforms = sorted(plot_data.keys())
+ if not plot_data:
+ html_top()
+ print 'No runs'
+ elif 'runs' in self.cgiform:
+ self.testrun_details_for_1_test_kernel_platform(test, metric,
+ platforms[0])
+ else:
+ self.graph_1_test_all_platforms(test, metric, platforms, plot_data)
+
+
+ def analyze_all_metrics_1_platform(self, test, platform):
+ # TODO: show #runs in header
+ html_top()
+ heading = "%s %s:  %s" % (self.test_group, self.kernel_legend,
+ test.capitalize())
+ print "<h2> %s </h2>" % heading
+ print "platform=%s <br>" % platform
+ for attr in self.test_attributes:
+ print "%s=%s " % (attr, self.test_attributes[attr])
+ print "<p>"
+ table_all_metrics_1_platform(self.test_runs, platform, self.relative)
+ print "</body></html>"
+
+
+ def table_for_variants_all_tests(self, title, plot_data, colkeys, columns,
+ filtered_passthru, test_tag):
+ # generate detailed html page with 1 graph and corresp numbers
+ # for primary metric over all benchmarks (rows),
+ # on one platform and one kernel,
+ # over various combos of test run attribute constraints (cols).
+ html_top()
+ print '<h3> %s </h3>' % title
+ print ('%s, machine group %s on //%s server <br>' %
+ (self.kernel_legend, self.test_group, self.tko_server))
+ if test_tag:
+ print '%s test script series <br>' % test_tag[1:]
+
+ varies = ['vary='+colkeys[col] for col in columns]
+ print "<img src='%s?%s'>" % (self.myurl, '&'.join(self.passthru+varies))
+
+ print "<p><p> <h3> %s: </h3>" % title
+ table_variants_all_tests(plot_data, columns, colkeys, self.benchmarks,
+ self.myurl, filtered_passthru)
+ print "</body></html>"
+
+
+ def analyze_variants_all_tests_1_platform(self, platform, vary):
+ # generate one graph image for results of all benchmarks
+ # on one platform and one kernel, comparing effects of
+ # two or more combos of kernel options (test run attributes)
+ # (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
+ kernel = self.cgiform.getvalue('kernel', 'some_kernel')
+ self.passthru.append('kernel=%s' % kernel)
+
+ # two or more vary_groups, one for each plotted line,
+ # each group begins with vary= and ends with next &
+ # each group has comma-separated list of test attribute key=val pairs
+ # eg vary=keyval1,keyval2&vary=keyval3,keyval4
+ vary_groups = [dict(pair.split('=',1) for pair
+ in vary_group.split(','))
+ for vary_group in vary]
+
+ test = self.benchmarks[0] # pick any test in all jobs
+ kernels, test_tag = self.jobs_selector(test, self.job_table,
+ self.kernel_dates)
+
+ linekeys = {}
+ plot_data = {}
+ baselines = {}
+ for i, vary_group in enumerate(vary_groups):
+ group_attributes = self.test_attributes.copy()
+ group_attributes.update(vary_group)
+ linekey = ','.join('%s=%s' % (attr, vary_group[attr])
+ for attr in vary_group)
+ linekeys[i] = linekey
+ data = {}
+ for benchmark in self.benchmarks:
+ metric = perf.benchmark_main_metric(benchmark)
+ runs = collect_testruns(self.job_table[kernel],
+ benchmark+test_tag,
+ group_attributes,
+ self.platforms_filter,
+ 'by_hosts' in self.toggles,
+ self.no_antag)
+ vals = []
+ for testrunx in runs[platform]:
+ vals += perf.get_metric_at_point([testrunx], metric)
+ if vals:
+ if benchmark not in baselines:
+ baselines[benchmark], stddev = plotgraph.avg_dev(vals)
+ vals = [val/baselines[benchmark] for val in vals]
+ data[benchmark] = vals
+ plot_data[i] = data
+
+ title = "%s on %s" % (kernel, platform)
+ for attr in self.test_attributes:
+ title += ', %s=%s' % (attr, self.test_attributes[attr])
+ if 'table' in self.cgiform:
+ self.table_for_variants_all_tests(title, plot_data, linekeys,
+ range(len(linekeys)),
+ filtered_passthru=self.passthru,
+ test_tag=test_tag)
+ else:
+ graph_variants_all_tests(title, plot_data, linekeys,
+ self.size, 'dark' in self.toggles)
+
+
+ def graph_twoway_antagonists_1_test_1_platform(
+ self, test, metric, platform, antagonists, twoway_runs):
+ # generate graph of one benchmark's performance paired with
+ # various antagonists, with one plotted line per antagonist,
+ # over most kernels (X axis), all on one machine type
+ # performance is relative to the no-antag baseline case
+ plot_data = collect_twoway_scores(metric, antagonists,
+ twoway_runs, self.relative)
+ title = "%s vs. an Antagonist on %s:" % (test.capitalize(), platform)
+ if 'table' in self.cgiform:
+ filtered_passthru = [arg for arg in self.passthru
+ if not arg.startswith('antag=')]
+ self.table_for_graph_1_test(title, metric, plot_data,
+ 'antag', antagonists,
+ filtered_passthru=filtered_passthru)
+ else:
+ graph_1_test(title, metric, plot_data, 'antag', antagonists,
+ self.kernel_legend, self.relative,
+ self.size, 'dark' in self.toggles)
+
+
+ def testrun_details_for_twoway_test(self, test, metric, platform,
+ antagonist, twoway_runs):
+ default_kernel = min(twoway_runs.keys())
+ kernel = self.cgiform.getvalue('kernel', default_kernel)
+ title = '%s vs. Antagonist %s on %s using %s' % (
+ test.capitalize(), antagonist.capitalize(), platform, kernel)
+ runs = twoway_runs[kernel].get(antagonist, [])
+ self.testrun_details(title, runs, metric)
+
+
+ def analyze_twoway_antagonists_1_test_1_platform(
+ self, test, metric, platform, antagonists):
+ twoway_runs, all_antagonists = divide_twoway_testruns(self.test_runs,
+ platform)
+ if antagonists == ['*']:
+ antagonists = all_antagonists
+ if not twoway_runs:
+ html_top()
+ print 'No runs'
+ elif 'runs' in self.cgiform:
+ self.testrun_details_for_twoway_test(
+ test, metric, platform, antagonists[0], twoway_runs)
+ else:
+ self.graph_twoway_antagonists_1_test_1_platform(
+ test, metric, platform, antagonists, twoway_runs)
+
+
+ def get_twoway_default_platform(self):
+ if self.platforms_filter:
+ return self.platforms_filter[0]
+ test = 'unixbench'
+ kernels, test_tag = self.jobs_selector(test, self.job_table,
+ self.kernel_dates)
+ self.collect_all_testruns(kernels, test+test_tag)
+ return all_tested_platforms(self.test_runs)[0]
+
+
+ def overview_page_all_tests(self, benchmarks, antagonists):
# generate overview html page with small graphs for each benchmark
# linking to detailed html page for that benchmark
# recursively link to this same cgi to generate each image
- print "Content-Type: text/html\n\n<html><body>"
- heading = "%s %s" % (test_group, kernel_legend)
- if regress:
+ html_top()
+ if antagonists is not None:
+ heading = ('Twoway Container Isolation using %s on %s' %
+ (self.kernel_legend, self.get_twoway_default_platform()))
+ else:
+ heading = '%s, %s Benchmarks' % (self.kernel_legend,
+ self.test_group)
+ if 'regress' in self.toggles:
heading += ", Regressions Only"
- print "<h2> %s </h2>" % heading
+ print "<h3> %s </h3>" % heading
for test in benchmarks:
- overview_thumb(test)
+ overview_thumb(test, '', self.myurl, self.passthru)
if test == 'unixbench':
- overview_thumb('unixbench', 'Process_creation')
+ overview_thumb('unixbench', 'Process_creation',
+ self.myurl, self.passthru)
+
+ link = abs_rel_link(self.myurl, self.passthru)
+ print "<p><p> <h4> Redraw this with %s performance? </h4>" % link
print "</body></html>"
- # body of nightly_views():
- cgitb.enable()
- form = cgi.FieldStorage(keep_blank_values=True)
- test = form.getvalue('test', '')
- relative = 'relative' in form
- regress = 'regress' in form
- dark = 'dark' in form
- platforms_filter = form.getvalue('platforms', '')
- by_hosts = 'by_hosts' in form or '.' in platforms_filter
- passthru = []
- if relative:
- passthru += ['relative']
- if regress:
- passthru += ['regress']
- if dark:
- passthru += ['dark']
- if by_hosts:
- passthru += ['by_hosts']
- if platforms_filter:
- passthru += ['platforms=%s' % platforms_filter]
- myself = os.path.basename(sys.argv[0])
- if test:
- passthru += ['test=%s' % test]
- metric = form.getvalue('metric', '')
+ def analyze_1_test(self, test, antagonists):
+ self.passthru.append('test=%s' % test)
+ metric = self.cgiform.getvalue('metric', '')
if metric:
- passthru += ['metric=%s' % metric]
+ self.passthru.append('metric=%s' % metric)
else:
metric = perf.benchmark_main_metric(test)
assert metric, "no default metric for test %s" % test
- # perf.init()
- perf.db_cur = db.db(host=tko_mysql_server,
- user='nobody', password='').cur
- kernels = select_dev_kernels()
- regressed_platforms = find_regressions(kernels, test, metric)
- collect_all_testruns(kernels, regressed_platforms, test)
- plot_data = collect_scaled_scores(metric)
- platforms = sorted(plot_data.keys())
- if 'runs' in form:
- testrun_details_for_1_test_kernel_platform(test, metric,
- platforms[0])
- elif 'table' in form:
- table_for_1_test(test, metric)
+ self.kernels, self.test_tag = self.jobs_selector(test, self.job_table,
+ self.kernel_dates)
+ self.collect_all_testruns(self.kernels, test+self.test_tag)
+ if not self.platforms_filter and (metric == '*' or
+ antagonists is not None):
+ # choose default platform
+ self.platforms_filter = all_tested_platforms(self.test_runs)[0:1]
+ self.passthru.append('platforms=%s' %
+ ','.join(self.platforms_filter))
+ if antagonists is not None:
+ antagonists = antagonists.split(',')
+ if len(antagonists) == 1 and antagonists != ['*']:
+ self.relative = False
+ self.analyze_twoway_antagonists_1_test_1_platform(
+ test, metric, self.platforms_filter[0], antagonists)
+ elif metric == '*':
+ platform = self.platforms_filter[0]
+ self.analyze_all_metrics_1_platform(test, platform)
else:
- size = form.getvalue('size', '1200,850' )
- graph_1_test(test, metric, size)
- else:
- overview_page(benchmarks)
+ self.analyze_1_metric_all_platforms(test, metric)
-def sort_kernels(kernels):
- return sorted(kernels, key=kernel_versions.version_encode)
-
-
-def survey_all_kernels_tested(db_cur, kernel_series, kname_prefix, smp,
- test_group, maxkernels,
- kernel_jobs, kernel_dates):
- kernels = set()
- # script run's job label has form
- # 'Kernel_Qual_Containers : 2.6.26-300.8-jilee : smp : 2009-05-15'
- # or 'Kernel_Qual_Containers : 2.6.26-DEV-4099999 : smp : 2009-05-15'
- job_label = ('%s : %s%% : %s : %%'
- % (test_group, kernel_series, smp))
- # find names and job#s of all matching perf runs
- cmd = ( "select job_idx, label, tag from jobs"
- " where label like '%s' order by label desc" % job_label )
- nrows = db_cur.execute(cmd)
- for jobx, joblabel, tag in db_cur.fetchall():
- cols = joblabel.split(' : ')
- kernvers = cols[1].split('-') # 2.6.26 300.8 jilee
- # or 2.6.26 DEV 4099999
- if kname_prefix: # nightly build, eg 'DEV' or '300'
- changelist = kernvers[2] # build's CL number
- testdate = cols[3]
- kernel = '%s_%s' % (kname_prefix, changelist)
- else: # release candidates
- if len(kernvers) > 2: # reject jobs with -qual suffix
- continue
- kernel = kernvers[1] # 300.8
- testdate = ''
- kernel_jobs.setdefault(kernel, [])
- kernel_jobs[kernel].append(jobx)
- kernel_dates[kernel] = testdate
- kernels.add(kernel)
- kernels = sort_kernels(kernels)[-maxkernels:]
- return kernels
+ def parse_most_cgi_args(self):
+ self.myurl = os.path.basename(sys.argv[0])
+ self.cgiform = cgi.FieldStorage(keep_blank_values=True)
+ self.size = self.cgiform.getvalue('size', '1200,850')
+ all_toggles = set(('absolute', 'regress', 'dark', 'by_hosts'))
+ self.toggles = set(tog for tog in all_toggles if tog in self.cgiform)
+ platforms = self.cgiform.getvalue('platforms', '')
+ if '.' in platforms:
+ self.toggles.add('by_hosts')
+ self.passthru = list(self.toggles)
+ self.relative = 'absolute' not in self.toggles
+ if platforms:
+ self.passthru.append('platforms=%s' % platforms)
+ self.platforms_filter = platforms.split(',')
+ else:
+ self.platforms_filter = []
+ self.test_attributes = perf.parse_test_attr_args(self.cgiform)
+ perf.append_cgi_args(self.passthru, self.test_attributes)
+ test = self.cgiform.getvalue('test', '')
+ if 'antag' in self.cgiform:
+ antagonists = ','.join(self.cgiform.getlist('antag'))
+ # antag=*
+ # or antag=test1,test2,test3,...
+ # or antag=test1&antag=test2&...
+ # testN is empty for solo case of no antagonist
+ self.passthru.append('antag=%s' % antagonists)
+ else:
+ antagonists = None # not same as ''
+ return test, antagonists
