mirror of
git://git.yoctoproject.org/poky
synced 2026-09-17 01:13:40 +00:00
b25e586458
When comparing branches, correctly filter the revisions corresponding to the specific branch specified. Also use the commit numbers as a way to gauge spatially related commits for comparision meaning comparisions for out of order build revisions becomes meaninful. This should improve the reporting for autobuilder generated builds. Also improve the branch option help text. (From OE-Core rev: 9f6f4ab6eec9dca07af7f53da5f737a6167bfb38) (From OE-Core rev: d6645c4c931565d62df85db0fa71f15d51265828) Signed-off-by: Richard Purdie <richard.purdie@linuxfoundation.org>
682 lines
26 KiB
Python
Executable File
682 lines
26 KiB
Python
Executable File
#!/usr/bin/python3
|
|
#
|
|
# Examine build performance test results
|
|
#
|
|
# Copyright (c) 2017, Intel Corporation.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify it
|
|
# under the terms and conditions of the GNU General Public License,
|
|
# version 2, as published by the Free Software Foundation.
|
|
#
|
|
# This program is distributed in the hope it will be useful, but WITHOUT
|
|
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
|
|
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
|
|
# more details.
|
|
#
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
from collections import namedtuple, OrderedDict
|
|
from operator import attrgetter
|
|
from xml.etree import ElementTree as ET
|
|
|
|
# Import oe libs
|
|
scripts_path = os.path.dirname(os.path.realpath(__file__))
|
|
sys.path.append(os.path.join(scripts_path, 'lib'))
|
|
import scriptpath
|
|
from build_perf import print_table
|
|
from build_perf.report import (metadata_xml_to_json, results_xml_to_json,
|
|
aggregate_data, aggregate_metadata, measurement_stats,
|
|
AggregateTestData)
|
|
from build_perf import html
|
|
from buildstats import BuildStats, diff_buildstats, BSVerDiff
|
|
|
|
scriptpath.add_oe_lib_path()
|
|
|
|
from oeqa.utils.git import GitRepo, GitError
|
|
|
|
|
|
# Setup logging
|
|
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
|
log = logging.getLogger('oe-build-perf-report')
|
|
|
|
|
|
# Container class for tester revisions
|
|
TestedRev = namedtuple('TestedRev', 'commit commit_number tags')
|
|
|
|
|
|
def get_test_runs(repo, tag_name, **kwargs):
|
|
"""Get a sorted list of test runs, matching given pattern"""
|
|
# First, get field names from the tag name pattern
|
|
field_names = [m.group(1) for m in re.finditer(r'{(\w+)}', tag_name)]
|
|
undef_fields = [f for f in field_names if f not in kwargs.keys()]
|
|
|
|
# Fields for formatting tag name pattern
|
|
str_fields = dict([(f, '*') for f in field_names])
|
|
str_fields.update(kwargs)
|
|
|
|
# Get a list of all matching tags
|
|
tag_pattern = tag_name.format(**str_fields)
|
|
tags = repo.run_cmd(['tag', '-l', tag_pattern]).splitlines()
|
|
log.debug("Found %d tags matching pattern '%s'", len(tags), tag_pattern)
|
|
|
|
# Parse undefined fields from tag names
|
|
str_fields = dict([(f, r'(?P<{}>[\w\-.()]+)'.format(f)) for f in field_names])
|
|
str_fields['branch'] = r'(?P<branch>[\w\-.()/]+)'
|
|
str_fields['commit'] = '(?P<commit>[0-9a-f]{7,40})'
|
|
str_fields['commit_number'] = '(?P<commit_number>[0-9]{1,7})'
|
|
str_fields['tag_number'] = '(?P<tag_number>[0-9]{1,5})'
|
|
# escape parenthesis in fields in order to not messa up the regexp
|
|
fixed_fields = dict([(k, v.replace('(', r'\(').replace(')', r'\)')) for k, v in kwargs.items()])
|
|
str_fields.update(fixed_fields)
|
|
tag_re = re.compile(tag_name.format(**str_fields))
|
|
|
|
# Parse fields from tags
|
|
revs = []
|
|
for tag in tags:
|
|
m = tag_re.match(tag)
|
|
groups = m.groupdict()
|
|
revs.append([groups[f] for f in undef_fields] + [tag])
|
|
|
|
# Return field names and a sorted list of revs
|
|
return undef_fields, sorted(revs)
|
|
|
|
def list_test_revs(repo, tag_name, verbosity, **kwargs):
|
|
"""Get list of all tested revisions"""
|
|
valid_kwargs = dict([(k, v) for k, v in kwargs.items() if v is not None])
|
|
|
|
fields, revs = get_test_runs(repo, tag_name, **valid_kwargs)
|
|
ignore_fields = ['tag_number']
|
|
if verbosity < 2:
|
|
extra_fields = ['COMMITS', 'TEST RUNS']
|
|
ignore_fields.extend(['commit_number', 'commit'])
|
|
else:
|
|
extra_fields = ['TEST RUNS']
|
|
|
|
print_fields = [i for i, f in enumerate(fields) if f not in ignore_fields]
|
|
|
|
# Sort revs
|
|
rows = [[fields[i].upper() for i in print_fields] + extra_fields]
|
|
|
|
prev = [''] * len(print_fields)
|
|
prev_commit = None
|
|
commit_cnt = 0
|
|
commit_field = fields.index('commit')
|
|
for rev in revs:
|
|
# Only use fields that we want to print
|
|
cols = [rev[i] for i in print_fields]
|
|
|
|
|
|
if cols != prev:
|
|
commit_cnt = 1
|
|
test_run_cnt = 1
|
|
new_row = [''] * (len(print_fields) + len(extra_fields))
|
|
|
|
for i in print_fields:
|
|
if cols[i] != prev[i]:
|
|
break
|
|
new_row[i:-len(extra_fields)] = cols[i:]
|
|
rows.append(new_row)
|
|
else:
|
|
if rev[commit_field] != prev_commit:
|
|
commit_cnt += 1
|
|
test_run_cnt += 1
|
|
|
|
if verbosity < 2:
|
|
new_row[-2] = commit_cnt
|
|
new_row[-1] = test_run_cnt
|
|
prev = cols
|
|
prev_commit = rev[commit_field]
|
|
|
|
print_table(rows)
|
|
|
|
def get_test_revs(repo, tag_name, **kwargs):
|
|
"""Get list of all tested revisions"""
|
|
fields, runs = get_test_runs(repo, tag_name, **kwargs)
|
|
|
|
revs = {}
|
|
commit_i = fields.index('commit')
|
|
commit_num_i = fields.index('commit_number')
|
|
for run in runs:
|
|
commit = run[commit_i]
|
|
commit_num = run[commit_num_i]
|
|
tag = run[-1]
|
|
if not commit in revs:
|
|
revs[commit] = TestedRev(commit, commit_num, [tag])
|
|
else:
|
|
assert commit_num == revs[commit].commit_number, "Commit numbers do not match"
|
|
revs[commit].tags.append(tag)
|
|
|
|
# Return in sorted table
|
|
revs = sorted(revs.values(), key=attrgetter('commit_number'))
|
|
log.debug("Found %d tested revisions:\n %s", len(revs),
|
|
"\n ".join(['{} ({})'.format(rev.commit_number, rev.commit) for rev in revs]))
|
|
return revs
|
|
|
|
def rev_find(revs, attr, val):
|
|
"""Search from a list of TestedRev"""
|
|
for i, rev in enumerate(revs):
|
|
if getattr(rev, attr) == val:
|
|
return i
|
|
raise ValueError("Unable to find '{}' value '{}'".format(attr, val))
|
|
|
|
def is_xml_format(repo, commit):
|
|
"""Check if the commit contains xml (or json) data"""
|
|
if repo.rev_parse(commit + ':results.xml'):
|
|
log.debug("Detected report in xml format in %s", commit)
|
|
return True
|
|
else:
|
|
log.debug("No xml report in %s, assuming json formatted results", commit)
|
|
return False
|
|
|
|
def read_results(repo, tags, xml=True):
|
|
"""Read result files from repo"""
|
|
|
|
def parse_xml_stream(data):
|
|
"""Parse multiple concatenated XML objects"""
|
|
objs = []
|
|
xml_d = ""
|
|
for line in data.splitlines():
|
|
if xml_d and line.startswith('<?xml version='):
|
|
objs.append(ET.fromstring(xml_d))
|
|
xml_d = line
|
|
else:
|
|
xml_d += line
|
|
objs.append(ET.fromstring(xml_d))
|
|
return objs
|
|
|
|
def parse_json_stream(data):
|
|
"""Parse multiple concatenated JSON objects"""
|
|
objs = []
|
|
json_d = ""
|
|
for line in data.splitlines():
|
|
if line == '}{':
|
|
json_d += '}'
|
|
objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
|
|
json_d = '{'
|
|
else:
|
|
json_d += line
|
|
objs.append(json.loads(json_d, object_pairs_hook=OrderedDict))
|
|
return objs
|
|
|
|
num_revs = len(tags)
|
|
|
|
# Optimize by reading all data with one git command
|
|
log.debug("Loading raw result data from %d tags, %s...", num_revs, tags[0])
|
|
if xml:
|
|
git_objs = [tag + ':metadata.xml' for tag in tags] + [tag + ':results.xml' for tag in tags]
|
|
data = parse_xml_stream(repo.run_cmd(['show'] + git_objs + ['--']))
|
|
return ([metadata_xml_to_json(e) for e in data[0:num_revs]],
|
|
[results_xml_to_json(e) for e in data[num_revs:]])
|
|
else:
|
|
git_objs = [tag + ':metadata.json' for tag in tags] + [tag + ':results.json' for tag in tags]
|
|
data = parse_json_stream(repo.run_cmd(['show'] + git_objs + ['--']))
|
|
return data[0:num_revs], data[num_revs:]
|
|
|
|
|
|
def get_data_item(data, key):
|
|
"""Nested getitem lookup"""
|
|
for k in key.split('.'):
|
|
data = data[k]
|
|
return data
|
|
|
|
|
|
def metadata_diff(metadata_l, metadata_r):
|
|
"""Prepare a metadata diff for printing"""
|
|
keys = [('Hostname', 'hostname', 'hostname'),
|
|
('Branch', 'branch', 'layers.meta.branch'),
|
|
('Commit number', 'commit_num', 'layers.meta.commit_count'),
|
|
('Commit', 'commit', 'layers.meta.commit'),
|
|
('Number of test runs', 'testrun_count', 'testrun_count')
|
|
]
|
|
|
|
def _metadata_diff(key):
|
|
"""Diff metadata from two test reports"""
|
|
try:
|
|
val1 = get_data_item(metadata_l, key)
|
|
except KeyError:
|
|
val1 = '(N/A)'
|
|
try:
|
|
val2 = get_data_item(metadata_r, key)
|
|
except KeyError:
|
|
val2 = '(N/A)'
|
|
return val1, val2
|
|
|
|
metadata = OrderedDict()
|
|
for title, key, key_json in keys:
|
|
value_l, value_r = _metadata_diff(key_json)
|
|
metadata[key] = {'title': title,
|
|
'value_old': value_l,
|
|
'value': value_r}
|
|
return metadata
|
|
|
|
|
|
def print_diff_report(metadata_l, data_l, metadata_r, data_r):
|
|
"""Print differences between two data sets"""
|
|
|
|
# First, print general metadata
|
|
print("\nTEST METADATA:\n==============")
|
|
meta_diff = metadata_diff(metadata_l, metadata_r)
|
|
rows = []
|
|
row_fmt = ['{:{wid}} ', '{:<{wid}} ', '{:<{wid}}']
|
|
rows = [['', 'CURRENT COMMIT', 'COMPARING WITH']]
|
|
for key, val in meta_diff.items():
|
|
# Shorten commit hashes
|
|
if key == 'commit':
|
|
rows.append([val['title'] + ':', val['value'][:20], val['value_old'][:20]])
|
|
else:
|
|
rows.append([val['title'] + ':', val['value'], val['value_old']])
|
|
print_table(rows, row_fmt)
|
|
|
|
|
|
# Print test results
|
|
print("\nTEST RESULTS:\n=============")
|
|
|
|
tests = list(data_l['tests'].keys())
|
|
# Append tests that are only present in 'right' set
|
|
tests += [t for t in list(data_r['tests'].keys()) if t not in tests]
|
|
|
|
# Prepare data to be printed
|
|
rows = []
|
|
row_fmt = ['{:8}', '{:{wid}}', '{:{wid}}', ' {:>{wid}}', ' {:{wid}} ', '{:{wid}}',
|
|
' {:>{wid}}', ' {:>{wid}}']
|
|
num_cols = len(row_fmt)
|
|
for test in tests:
|
|
test_l = data_l['tests'][test] if test in data_l['tests'] else None
|
|
test_r = data_r['tests'][test] if test in data_r['tests'] else None
|
|
pref = ' '
|
|
if test_l is None:
|
|
pref = '+'
|
|
elif test_r is None:
|
|
pref = '-'
|
|
descr = test_l['description'] if test_l else test_r['description']
|
|
heading = "{} {}: {}".format(pref, test, descr)
|
|
|
|
rows.append([heading])
|
|
|
|
# Generate the list of measurements
|
|
meas_l = test_l['measurements'] if test_l else {}
|
|
meas_r = test_r['measurements'] if test_r else {}
|
|
measurements = list(meas_l.keys())
|
|
measurements += [m for m in list(meas_r.keys()) if m not in measurements]
|
|
|
|
for meas in measurements:
|
|
m_pref = ' '
|
|
if meas in meas_l:
|
|
stats_l = measurement_stats(meas_l[meas], 'l.')
|
|
else:
|
|
stats_l = measurement_stats(None, 'l.')
|
|
m_pref = '+'
|
|
if meas in meas_r:
|
|
stats_r = measurement_stats(meas_r[meas], 'r.')
|
|
else:
|
|
stats_r = measurement_stats(None, 'r.')
|
|
m_pref = '-'
|
|
stats = stats_l.copy()
|
|
stats.update(stats_r)
|
|
|
|
absdiff = stats['val_cls'](stats['r.mean'] - stats['l.mean'])
|
|
reldiff = "{:+.1f} %".format(absdiff * 100 / stats['l.mean'])
|
|
if stats['r.mean'] > stats['l.mean']:
|
|
absdiff = '+' + str(absdiff)
|
|
else:
|
|
absdiff = str(absdiff)
|
|
rows.append(['', m_pref, stats['name'] + ' ' + stats['quantity'],
|
|
str(stats['l.mean']), '->', str(stats['r.mean']),
|
|
absdiff, reldiff])
|
|
rows.append([''] * num_cols)
|
|
|
|
print_table(rows, row_fmt)
|
|
|
|
print()
|
|
|
|
|
|
class BSSummary(object):
|
|
def __init__(self, bs1, bs2):
|
|
self.tasks = {'count': bs2.num_tasks,
|
|
'change': '{:+d}'.format(bs2.num_tasks - bs1.num_tasks)}
|
|
self.top_consumer = None
|
|
self.top_decrease = None
|
|
self.top_increase = None
|
|
self.ver_diff = OrderedDict()
|
|
|
|
tasks_diff = diff_buildstats(bs1, bs2, 'cputime')
|
|
|
|
# Get top consumers of resources
|
|
tasks_diff = sorted(tasks_diff, key=attrgetter('value2'))
|
|
self.top_consumer = tasks_diff[-5:]
|
|
|
|
# Get biggest increase and decrease in resource usage
|
|
tasks_diff = sorted(tasks_diff, key=attrgetter('absdiff'))
|
|
self.top_decrease = tasks_diff[0:5]
|
|
self.top_increase = tasks_diff[-5:]
|
|
|
|
# Compare recipe versions and prepare data for display
|
|
ver_diff = BSVerDiff(bs1, bs2)
|
|
if ver_diff:
|
|
if ver_diff.new:
|
|
self.ver_diff['New recipes'] = [(n, r.evr) for n, r in ver_diff.new.items()]
|
|
if ver_diff.dropped:
|
|
self.ver_diff['Dropped recipes'] = [(n, r.evr) for n, r in ver_diff.dropped.items()]
|
|
if ver_diff.echanged:
|
|
self.ver_diff['Epoch changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.echanged.items()]
|
|
if ver_diff.vchanged:
|
|
self.ver_diff['Version changed'] = [(n, "{} → {}".format(r.left.version, r.right.version)) for n, r in ver_diff.vchanged.items()]
|
|
if ver_diff.rchanged:
|
|
self.ver_diff['Revision changed'] = [(n, "{} → {}".format(r.left.evr, r.right.evr)) for n, r in ver_diff.rchanged.items()]
|
|
|
|
|
|
def print_html_report(data, id_comp, buildstats):
|
|
"""Print report in html format"""
|
|
# Handle metadata
|
|
metadata = metadata_diff(data[id_comp].metadata, data[-1].metadata)
|
|
|
|
# Generate list of tests
|
|
tests = []
|
|
for test in data[-1].results['tests'].keys():
|
|
test_r = data[-1].results['tests'][test]
|
|
new_test = {'name': test_r['name'],
|
|
'description': test_r['description'],
|
|
'status': test_r['status'],
|
|
'measurements': [],
|
|
'err_type': test_r.get('err_type'),
|
|
}
|
|
# Limit length of err output shown
|
|
if 'message' in test_r:
|
|
lines = test_r['message'].splitlines()
|
|
if len(lines) > 20:
|
|
new_test['message'] = '...\n' + '\n'.join(lines[-20:])
|
|
else:
|
|
new_test['message'] = test_r['message']
|
|
|
|
|
|
# Generate the list of measurements
|
|
for meas in test_r['measurements'].keys():
|
|
meas_r = test_r['measurements'][meas]
|
|
meas_type = 'time' if meas_r['type'] == 'sysres' else 'size'
|
|
new_meas = {'name': meas_r['name'],
|
|
'legend': meas_r['legend'],
|
|
'description': meas_r['name'] + ' ' + meas_type,
|
|
}
|
|
samples = []
|
|
|
|
# Run through all revisions in our data
|
|
for meta, test_data in data:
|
|
if (not test in test_data['tests'] or
|
|
not meas in test_data['tests'][test]['measurements']):
|
|
samples.append(measurement_stats(None))
|
|
continue
|
|
test_i = test_data['tests'][test]
|
|
meas_i = test_i['measurements'][meas]
|
|
commit_num = get_data_item(meta, 'layers.meta.commit_count')
|
|
samples.append(measurement_stats(meas_i))
|
|
samples[-1]['commit_num'] = commit_num
|
|
|
|
absdiff = samples[-1]['val_cls'](samples[-1]['mean'] - samples[id_comp]['mean'])
|
|
new_meas['absdiff'] = absdiff
|
|
new_meas['absdiff_str'] = str(absdiff) if absdiff < 0 else '+' + str(absdiff)
|
|
new_meas['reldiff'] = "{:+.1f} %".format(absdiff * 100 / samples[id_comp]['mean'])
|
|
new_meas['samples'] = samples
|
|
new_meas['value'] = samples[-1]
|
|
new_meas['value_type'] = samples[-1]['val_cls']
|
|
|
|
# Compare buildstats
|
|
bs_key = test + '.' + meas
|
|
rev = metadata['commit_num']['value']
|
|
comp_rev = metadata['commit_num']['value_old']
|
|
if (rev in buildstats and bs_key in buildstats[rev] and
|
|
comp_rev in buildstats and bs_key in buildstats[comp_rev]):
|
|
new_meas['buildstats'] = BSSummary(buildstats[comp_rev][bs_key],
|
|
buildstats[rev][bs_key])
|
|
|
|
|
|
new_test['measurements'].append(new_meas)
|
|
tests.append(new_test)
|
|
|
|
# Chart options
|
|
chart_opts = {'haxis': {'min': get_data_item(data[0][0], 'layers.meta.commit_count'),
|
|
'max': get_data_item(data[-1][0], 'layers.meta.commit_count')}
|
|
}
|
|
|
|
print(html.template.render(title="Build Perf Test Report",
|
|
metadata=metadata, test_data=tests,
|
|
chart_opts=chart_opts))
|
|
|
|
|
|
def get_buildstats(repo, notes_ref, revs, outdir=None):
|
|
"""Get the buildstats from git notes"""
|
|
full_ref = 'refs/notes/' + notes_ref
|
|
if not repo.rev_parse(full_ref):
|
|
log.error("No buildstats found, please try running "
|
|
"'git fetch origin %s:%s' to fetch them from the remote",
|
|
full_ref, full_ref)
|
|
return
|
|
|
|
missing = False
|
|
buildstats = {}
|
|
log.info("Parsing buildstats from 'refs/notes/%s'", notes_ref)
|
|
for rev in revs:
|
|
buildstats[rev.commit_number] = {}
|
|
log.debug('Dumping buildstats for %s (%s)', rev.commit_number,
|
|
rev.commit)
|
|
for tag in rev.tags:
|
|
log.debug(' %s', tag)
|
|
try:
|
|
bs_all = json.loads(repo.run_cmd(['notes', '--ref', notes_ref,
|
|
'show', tag + '^0']))
|
|
except GitError:
|
|
log.warning("Buildstats not found for %s", tag)
|
|
bs_all = {}
|
|
missing = True
|
|
|
|
for measurement, bs in bs_all.items():
|
|
# Write out onto disk
|
|
if outdir:
|
|
tag_base, run_id = tag.rsplit('/', 1)
|
|
tag_base = tag_base.replace('/', '_')
|
|
bs_dir = os.path.join(outdir, measurement, tag_base)
|
|
if not os.path.exists(bs_dir):
|
|
os.makedirs(bs_dir)
|
|
with open(os.path.join(bs_dir, run_id + '.json'), 'w') as f:
|
|
json.dump(bs, f, indent=2)
|
|
|
|
# Read buildstats into a dict
|
|
_bs = BuildStats.from_json(bs)
|
|
if measurement not in buildstats[rev.commit_number]:
|
|
buildstats[rev.commit_number][measurement] = _bs
|
|
else:
|
|
buildstats[rev.commit_number][measurement].aggregate(_bs)
|
|
|
|
if missing:
|
|
log.info("Buildstats were missing for some test runs, please "
|
|
"run 'git fetch origin %s:%s' and try again",
|
|
full_ref, full_ref)
|
|
|
|
return buildstats
|
|
|
|
|
|
def auto_args(repo, args):
|
|
"""Guess arguments, if not defined by the user"""
|
|
# Get the latest commit in the repo
|
|
log.debug("Guessing arguments from the latest commit")
|
|
msg = repo.run_cmd(['log', '-1', '--branches', '--remotes', '--format=%b'])
|
|
for line in msg.splitlines():
|
|
split = line.split(':', 1)
|
|
if len(split) != 2:
|
|
continue
|
|
|
|
key = split[0]
|
|
val = split[1].strip()
|
|
if key == 'hostname' and not args.hostname:
|
|
log.debug("Using hostname %s", val)
|
|
args.hostname = val
|
|
elif key == 'branch' and not args.branch:
|
|
log.debug("Using branch %s", val)
|
|
args.branch = val
|
|
|
|
|
|
def parse_args(argv):
|
|
"""Parse command line arguments"""
|
|
description = """
|
|
Examine build performance test results from a Git repository"""
|
|
parser = argparse.ArgumentParser(
|
|
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
|
|
description=description)
|
|
|
|
parser.add_argument('--debug', '-d', action='store_true',
|
|
help="Verbose logging")
|
|
parser.add_argument('--repo', '-r', required=True,
|
|
help="Results repository (local git clone)")
|
|
parser.add_argument('--list', '-l', action='count',
|
|
help="List available test runs")
|
|
parser.add_argument('--html', action='store_true',
|
|
help="Generate report in html format")
|
|
group = parser.add_argument_group('Tag and revision')
|
|
group.add_argument('--tag-name', '-t',
|
|
default='{hostname}/{branch}/{machine}/{commit_number}-g{commit}/{tag_number}',
|
|
help="Tag name (pattern) for finding results")
|
|
group.add_argument('--hostname', '-H')
|
|
group.add_argument('--branch', '-B', default='master', help="Branch to find commit in")
|
|
group.add_argument('--branch2', help="Branch to find comparision revisions in")
|
|
group.add_argument('--machine', default='qemux86')
|
|
group.add_argument('--history-length', default=25, type=int,
|
|
help="Number of tested revisions to plot in html report")
|
|
group.add_argument('--commit',
|
|
help="Revision to search for")
|
|
group.add_argument('--commit-number',
|
|
help="Revision number to search for, redundant if "
|
|
"--commit is specified")
|
|
group.add_argument('--commit2',
|
|
help="Revision to compare with")
|
|
group.add_argument('--commit-number2',
|
|
help="Revision number to compare with, redundant if "
|
|
"--commit2 is specified")
|
|
parser.add_argument('--dump-buildstats', nargs='?', const='.',
|
|
help="Dump buildstats of the tests")
|
|
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def main(argv=None):
|
|
"""Script entry point"""
|
|
args = parse_args(argv)
|
|
if args.debug:
|
|
log.setLevel(logging.DEBUG)
|
|
|
|
repo = GitRepo(args.repo)
|
|
|
|
if args.list:
|
|
list_test_revs(repo, args.tag_name, args.list, hostname=args.hostname)
|
|
return 0
|
|
|
|
# Determine hostname which to use
|
|
if not args.hostname:
|
|
auto_args(repo, args)
|
|
|
|
revs = get_test_revs(repo, args.tag_name, hostname=args.hostname,
|
|
branch=args.branch, machine=args.machine)
|
|
if args.branch2:
|
|
revs2 = get_test_revs(repo, args.tag_name, hostname=args.hostname,
|
|
branch=args.branch2, machine=args.machine)
|
|
if not len(revs2):
|
|
log.error("No revisions found to compare against")
|
|
return 1
|
|
if not len(revs):
|
|
log.error("No revision to report on found")
|
|
return 1
|
|
else:
|
|
if len(revs) < 2:
|
|
log.error("Only %d tester revisions found, unable to generate report" % len(revs))
|
|
return 1
|
|
|
|
# Pick revisions
|
|
if args.commit:
|
|
if args.commit_number:
|
|
log.warning("Ignoring --commit-number as --commit was specified")
|
|
index1 = rev_find(revs, 'commit', args.commit)
|
|
elif args.commit_number:
|
|
index1 = rev_find(revs, 'commit_number', args.commit_number)
|
|
else:
|
|
index1 = len(revs) - 1
|
|
|
|
if args.branch2:
|
|
revs2.append(revs[index1])
|
|
index1 = len(revs2) - 1
|
|
revs = revs2
|
|
|
|
if args.commit2:
|
|
if args.commit_number2:
|
|
log.warning("Ignoring --commit-number2 as --commit2 was specified")
|
|
index2 = rev_find(revs, 'commit', args.commit2)
|
|
elif args.commit_number2:
|
|
index2 = rev_find(revs, 'commit_number', args.commit_number2)
|
|
else:
|
|
if index1 > 0:
|
|
index2 = index1 - 1
|
|
# Find the closest matching commit number for comparision
|
|
# In future we could check the commit is a common ancestor and
|
|
# continue back if not but this good enough for now
|
|
while index2 > 0 and revs[index2].commit_number > revs[index1].commit_number:
|
|
index2 = index2 - 1
|
|
else:
|
|
log.error("Unable to determine the other commit, use "
|
|
"--commit2 or --commit-number2 to specify it")
|
|
return 1
|
|
|
|
index_l = min(index1, index2)
|
|
index_r = max(index1, index2)
|
|
|
|
rev_l = revs[index_l]
|
|
rev_r = revs[index_r]
|
|
log.debug("Using 'left' revision %s (%s), %s test runs:\n %s",
|
|
rev_l.commit_number, rev_l.commit, len(rev_l.tags),
|
|
'\n '.join(rev_l.tags))
|
|
log.debug("Using 'right' revision %s (%s), %s test runs:\n %s",
|
|
rev_r.commit_number, rev_r.commit, len(rev_r.tags),
|
|
'\n '.join(rev_r.tags))
|
|
|
|
# Check report format used in the repo (assume all reports in the same fmt)
|
|
xml = is_xml_format(repo, revs[index_r].tags[-1])
|
|
|
|
if args.html:
|
|
index_0 = max(0, min(index_l, index_r - args.history_length))
|
|
rev_range = range(index_0, index_r + 1)
|
|
else:
|
|
# We do not need range of commits for text report (no graphs)
|
|
index_0 = index_l
|
|
rev_range = (index_l, index_r)
|
|
|
|
# Read raw data
|
|
log.debug("Reading %d revisions, starting from %s (%s)",
|
|
len(rev_range), revs[index_0].commit_number, revs[index_0].commit)
|
|
raw_data = [read_results(repo, revs[i].tags, xml) for i in rev_range]
|
|
|
|
data = []
|
|
for raw_m, raw_d in raw_data:
|
|
data.append(AggregateTestData(aggregate_metadata(raw_m),
|
|
aggregate_data(raw_d)))
|
|
|
|
# Read buildstats only when needed
|
|
buildstats = None
|
|
if args.dump_buildstats or args.html:
|
|
outdir = 'oe-build-perf-buildstats' if args.dump_buildstats else None
|
|
notes_ref = 'buildstats/{}/{}/{}'.format(args.hostname, args.branch,
|
|
args.machine)
|
|
buildstats = get_buildstats(repo, notes_ref, [rev_l, rev_r], outdir)
|
|
|
|
# Print report
|
|
if not args.html:
|
|
print_diff_report(data[0].metadata, data[0].results,
|
|
data[1].metadata, data[1].results)
|
|
else:
|
|
# Re-map 'left' list index to the data table where index_0 maps to 0
|
|
print_html_report(data, index_l - index_0, buildstats)
|
|
|
|
return 0
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|