Update NDK prebuilts to build 3622456. Taken from branch aosp-master. Change-Id: I7a93804765cb9125a5a71ede1fbb2398d3464f1e
diff --git a/annotate.py b/annotate.py new file mode 100644 index 0000000..b49942f --- /dev/null +++ b/annotate.py
@@ -0,0 +1,599 @@ +#!/usr/bin/env python +# +# Copyright (C) 2016 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +"""annotate.py: annotate source files based on perf.data. +""" + + +import argparse +import os +import os.path +import shutil +import subprocess +import sys + +from simpleperf_report_lib import * +from utils import * + +# TODO: using addr2line can't convert from function_start_address to +# source_file:line very well for java code. Because in .debug_line section, +# there is some distance between function_start_address and the address +# of the first instruction which can be mapped to source line. +class Addr2Line(object): + """collect information of how to map [dso_name,vaddr] to [source_file:line]. + """ + def __init__(self, annotator, addr2line_path): + self.dso_dict = dict() + self.annotator = annotator + self.addr2line_path = addr2line_path + + + def add_addr(self, dso_name, addr): + dso = self.dso_dict.get(dso_name) + if dso is None: + self.dso_dict[dso_name] = dso = dict() + if not dso.has_key(addr): + dso[addr] = None + + + def convert_addrs_to_lines(self): + # store a list of source files + self.file_list = [] + # map from file to id with file_list[id] == file + self.file_dict = {} + + for dso_name in self.dso_dict.keys(): + self._convert_addrs_to_lines(dso_name, self.dso_dict[dso_name]) + self._combine_source_files() + + + def _convert_addrs_to_lines(self, dso_name, dso): + dso_path = self.annotator.find_dso_path(dso_name) + if dso_path is None: + log_warning("can't find dso '%s'" % dso_name) + dso.clear() + return + addrs = sorted(dso.keys()); + addr_str = [] + for addr in addrs: + addr_str.append('0x%x' % addr) + addr_str = '\n'.join(addr_str) + subproc = subprocess.Popen([self.addr2line_path, '-e', dso_path], + stdin=subprocess.PIPE, stdout=subprocess.PIPE) + (stdoutdata, _) = subproc.communicate(addr_str) + stdoutdata = stdoutdata.split('\n') + if len(stdoutdata) < len(addrs): + log_fatal("addr2line didn't output enough lines") + for i in range(len(addrs)): + strs = stdoutdata[i].split(':') + if len(strs) == 2 and strs[0].find('?') == -1: + file = strs[0].strip() + if strs[1].find('?') == -1: + line = 0 + for c in strs[1]: + if c.isdigit(): + line = line * 10 + ord(c) - ord('0') + else: + break + else: + line = None + id = self.file_dict.get(file) + if id is None: + id = len(self.file_list) + self.file_list.append(file) + self.file_dict[file] = id + dso[addrs[i]] = (id, line) + + + def _combine_source_files(self): + """It is possible that addr2line gives us different names for the same + file, like: + /usr/local/.../src/main/jni/sudo-game-jni.cpp + sudo-game-jni.cpp + We'd better combine these two files. We can do it by combining + source files with no conflicts in path. + """ + # Collect files having the same filename. + filename_dict = dict() + for file in self.file_list: + index = max(file.rfind('/'), file.rfind(os.sep)) + filename = file[index+1:] + entry = filename_dict.get(filename) + if entry is None: + filename_dict[filename] = entry = [] + entry.append(file) + + # Combine files having the same filename and having no conflicts in path. + for filename in filename_dict.keys(): + files = filename_dict[filename] + if len(files) == 1: + continue + for file in files: + to_file = file + # Test if we can merge files[i] with another file having longer + # path. + for f in files: + if len(f) > len(to_file) and f.find(file) != -1: + to_file = f + if to_file != file: + from_id = self.file_dict[file] + to_id = self.file_dict[to_file] + self.file_list[from_id] = self.file_list[to_id] + + + def get_source(self, dso_name, addr): + dso = self.dso_dict.get(dso_name) + if dso is None: + return (None, None) + item = dso.get(addr) + if item is None: + return (None, None) + return (self.file_list[item[0]], item[1]) + + +class Period(object): + """event count information. It can be used to represent event count + of a line, a function, a source file, or a binary. It contains two + parts: period and acc_period. + When used for a line, period is the event count occurred when running + that line, acc_period is the accumulated event count occurred when + running that line and functions called by that line. Same thing applies + when it is used for a function, a source file, or a binary. + """ + def __init__(self, period=0, acc_period=0): + self.period = period + self.acc_period = acc_period + + + def __iadd__(self, other): + self.period += other.period + self.acc_period += other.acc_period + return self + + +class DsoPeriod(object): + """Period for each shared library""" + def __init__(self, dso_name): + self.dso_name = dso_name + self.period = Period() + + + def add_period(self, period): + self.period += period + + +class FilePeriod(object): + """Period for each source file""" + def __init__(self, file): + self.file = file + self.period = Period() + # Period for each line in the file. + self.line_dict = {} + # Period for each function in the source file. + self.function_dict = {} + + + def add_period(self, period): + self.period += period + + + def add_line_period(self, line, period): + a = self.line_dict.get(line) + if a is None: + self.line_dict[line] = a = Period() + a += period + + + def add_function_period(self, function_name, function_start_line, period): + a = self.function_dict.get(function_name) + if a is None: + if function_start_line is None: + function_start_line = -1 + self.function_dict[function_name] = a = [function_start_line, Period()] + a[1] += period + + +class SourceFileAnnotator(object): + """group code for annotating source files""" + def __init__(self, config): + # check config variables + config_names = ['perf_data_list', 'symfs_dir', 'source_dirs', + 'annotate_dest_dir', 'comm_filters', 'pid_filters', + 'tid_filters', 'dso_filters', 'addr2line_path'] + for name in config_names: + if not config.has_key(name): + log_fatal('config [%s] is missing' % name) + symfs_dir = config['symfs_dir'] + if symfs_dir and not os.path.isdir(symfs_dir): + log_fatal('[symfs_dir] "%s" is not a dir' % symfs_dir) + kallsyms = config['kallsyms'] + if kallsyms and not os.path.isfile(kallsyms): + log_fatal('[kallsyms] "%s" is not a file' % kallsyms) + source_dirs = config['source_dirs'] + for dir in source_dirs: + if not os.path.isdir(dir): + log_fatal('[source_dirs] "%s" is not a dir' % dir) + + # init member variables + self.config = config + self.symfs_dir = None + self.kallsyms = None + self.comm_filter = None + self.pid_filter = None + self.tid_filter = None + self.dso_filter = None + symfs_dir = config['symfs_dir'] + if symfs_dir: + self.symfs_dir = symfs_dir + kallsyms = config['kallsyms'] + if kallsyms: + self.kallsyms = kallsyms + comm_filter = config['comm_filters'] + if comm_filter: + self.comm_filter = set(comm_filter) + pid_filter = config['pid_filters'] + if pid_filter: + self.pid_filter = set() + for pid in pid_filter: + self.pid_filter.add(int(pid)) + tid_filter = config['tid_filters'] + if tid_filter: + self.tid_filter = set() + for tid in tid_filter: + self.tid_filter.add(int(tid)) + dso_filter = config['dso_filters'] + if dso_filter: + self.dso_filter = set(dso_filter) + + output_dir = config['annotate_dest_dir'] + if os.path.isdir(output_dir): + shutil.rmtree(output_dir) + os.makedirs(output_dir) + + self.addr2line = Addr2Line(self, self.config['addr2line_path']) + + + def annotate(self): + self._collect_addrs() + self._convert_addrs_to_lines() + self._generate_periods() + self._write_summary() + self._collect_source_files() + self._annotate_files() + + + def _collect_addrs(self): + """Read perf.data, collect all addresses we need to convert to + source file:line. + """ + for perf_data in self.config['perf_data_list']: + lib = ReportLib() + if self.symfs_dir is not None: + lib.SetSymfs(self.symfs_dir) + if self.kallsyms is not None: + lib.SetKallsymsFile(self.kallsyms) + while True: + sample = lib.GetNextSample() + if sample is None: + lib.Close() + break + if not self._filter_sample(sample): + continue + symbols = [] + symbols.append(lib.GetSymbolOfCurrentSample()) + callchain = lib.GetCallChainOfCurrentSample() + for i in range(callchain.nr): + symbols.append(callchain.entries[i].symbol) + for symbol in symbols: + if self._filter_symbol(symbol): + self.addr2line.add_addr(symbol.dso_name, symbol.vaddr_in_file) + self.addr2line.add_addr(symbol.dso_name, symbol.symbol_addr) + + + def _filter_sample(self, sample): + """Return true if the sample can be used.""" + if self.comm_filter is not None: + if sample.thread_comm not in self.comm_filter: + return False + if self.pid_filter is not None: + if sample.pid not in self.pid_filter: + return False + if self.tid_filter is not None: + if sample.tid not in self.tid_filter: + return False + return True + + + def _filter_symbol(self, symbol): + if self.dso_filter is None or symbol.dso_name in self.dso_filter: + return True + return False + + + def _convert_addrs_to_lines(self): + self.addr2line.convert_addrs_to_lines() + + + def find_dso_path(self, dso): + if dso[0] != '/' or dso == '//anon': + return None + if self.symfs_dir is not None: + dso_path = os.path.join(self.symfs_dir, dso[1:]) + if os.path.isfile(dso_path): + return dso_path + if os.path.isfile(dso): + return dso + return None + + + def _generate_periods(self): + """read perf.data, collect Period for all types: + binaries, source files, functions, lines. + """ + self.period = 0 + self.dso_periods = dict() + self.file_periods = dict() + for perf_data in self.config['perf_data_list']: + lib = ReportLib() + if self.symfs_dir is not None: + lib.SetSymfs(self.symfs_dir) + if self.kallsyms is not None: + lib.SetKallsymsFile(self.kallsyms) + while True: + sample = lib.GetNextSample() + if sample is None: + lib.Close() + break + if not self._filter_sample(sample): + continue + symbols = [] + symbols.append(lib.GetSymbolOfCurrentSample()) + callchain = lib.GetCallChainOfCurrentSample() + for i in range(callchain.nr): + symbols.append(callchain.entries[i].symbol) + # Each sample has a callchain, but its period is only used once + # to add period for each function/source_line/source_file/binary. + # For example, if more than one entry in the callchain hits a + # function, the event count of that function is only increased once. + # Otherwise, we may get periods > 100%. + is_sample_used = False + used_dso_dict = dict() + used_file_dict = dict() + used_function_dict = dict() + used_line_dict = dict() + period = Period(sample.period, sample.period) + for i in range(len(symbols)): + symbol = symbols[i] + if i == 1: + period = Period(0, sample.period) + if not self._filter_symbol(symbol): + continue + is_sample_used = True + # Add period to dso. + self._add_dso_period(symbol.dso_name, period, used_dso_dict) + # Add period to source file. + source = self.addr2line.get_source(symbol.dso_name, symbol.vaddr_in_file) + if source[0] is not None: + self._add_file_period(source[0], period, used_file_dict) + # Add period to line. + if source[1] is not None: + self._add_line_period(source, period, used_line_dict) + # Add period to function. + source = self.addr2line.get_source(symbol.dso_name, symbol.symbol_addr) + if source[0] is not None: + self._add_file_period(source[0], period, used_file_dict) + self._add_function_period(source, symbol.symbol_name, period, + used_function_dict) + + if is_sample_used: + self.period += sample.period + + + def _add_dso_period(self, dso_name, period, used_dso_dict): + if not used_dso_dict.has_key(dso_name): + used_dso_dict[dso_name] = True + dso_period = self.dso_periods.get(dso_name) + if dso_period is None: + dso_period = self.dso_periods[dso_name] = DsoPeriod(dso_name) + dso_period.add_period(period) + + + def _add_file_period(self, file, period, used_file_dict): + if not used_file_dict.has_key(file): + used_file_dict[file] = True + file_period = self.file_periods.get(file) + if file_period is None: + file_period = self.file_periods[file] = FilePeriod(file) + file_period.add_period(period) + + + def _add_line_period(self, source, period, used_line_dict): + if not used_line_dict.has_key(source): + used_line_dict[source] = True + file_period = self.file_periods[source[0]] + file_period.add_line_period(source[1], period) + + + def _add_function_period(self, source, function_name, period, used_function_dict): + if not used_function_dict.has_key((source[0], function_name)): + used_function_dict[(source[0], function_name)] = True + file_period = self.file_periods[source[0]] + file_period.add_function_period(function_name, source[1], period) + + + def _write_summary(self): + summary = os.path.join(self.config['annotate_dest_dir'], 'summary') + with open(summary, 'w') as f: + f.write('total period: %d\n\n' % self.period) + dso_periods = sorted(self.dso_periods.values(), + cmp=lambda x, y: cmp(y.period.acc_period, x.period.acc_period)) + for dso_period in dso_periods: + f.write('dso %s: %s\n' % (dso_period.dso_name, + self._get_percentage_str(dso_period.period))) + f.write('\n') + + file_periods = sorted(self.file_periods.values(), + cmp=lambda x, y: cmp(y.period.acc_period, x.period.acc_period)) + for file_period in file_periods: + f.write('file %s: %s\n' % (file_period.file, + self._get_percentage_str(file_period.period))) + for file_period in file_periods: + f.write('\n\n%s: %s\n' % (file_period.file, + self._get_percentage_str(file_period.period))) + values = [] + for func_name in file_period.function_dict.keys(): + func_start_line, period = file_period.function_dict[func_name] + values.append((func_name, func_start_line, period)) + values = sorted(values, + cmp=lambda x, y: cmp(y[2].acc_period, x[2].acc_period)) + for value in values: + func = file_period.function_dict[func_name] + f.write('\tfunction (%s): line %d, %s\n' % ( + value[0], value[1], self._get_percentage_str(value[2]))) + f.write('\n') + for line in sorted(file_period.line_dict.keys()): + f.write('\tline %d: %s\n' % ( + line, self._get_percentage_str(file_period.line_dict[line]))) + + + def _get_percentage_str(self, period, short=False): + s = 'acc_p: %f%%, p: %f%%' if short else 'accumulated_period: %f%%, period: %f%%' + return s % self._get_percentage(period) + + + def _get_percentage(self, period): + if self.period == 0: + return (0, 0) + acc_p = 100.0 * period.acc_period / self.period + p = 100.0 * period.period / self.period + return (acc_p, p) + + + def _collect_source_files(self): + self.source_file_dict = dict() + source_file_suffix = ['h', 'c', 'cpp', 'cc', 'java'] + for source_dir in self.config['source_dirs']: + for root, _, files in os.walk(source_dir): + for file in files: + if file[file.rfind('.')+1:] in source_file_suffix: + entry = self.source_file_dict.get(file) + if entry is None: + entry = self.source_file_dict[file] = [] + entry.append(os.path.join(root, file)) + + + def _find_source_file(self, file): + filename = file[file.rfind(os.sep)+1:] + source_files = self.source_file_dict.get(filename) + if source_files is None: + return None + match_count = 0 + result = None + for path in source_files: + if path.find(file) != -1: + match_count += 1 + result = path + if match_count > 1: + log_warning('multiple source for %s, select %s' % (file, result)) + return result + + + def _annotate_files(self): + """Annotate Source files: add acc_period/period for each source file. + 1. Annotate java source files, which have $JAVA_SRC_ROOT prefix. + 2. Annotate c++ source files. + """ + dest_dir = self.config['annotate_dest_dir'] + for key in self.file_periods.keys(): + is_java = False + if key.startswith('$JAVA_SRC_ROOT/'): + path = key[len('$JAVA_SRC_ROOT/'):] + items = path.split('/') + path = os.sep.join(items) + from_path = self._find_source_file(path) + to_path = os.path.join(dest_dir, 'java', path) + is_java = True + elif key.startswith('/') and os.path.isfile(key): + path = key + from_path = path + to_path = os.path.join(dest_dir, path[1:]) + else: + path = key[1:] if key.startswith('/') else key + # Change path on device to path on host + path = os.sep.join(path.split('/')) + from_path = self._find_source_file(path) + to_path = os.path.join(dest_dir, path) + if from_path is None: + log_warning("can't find source file for path %s" % key) + continue + self._annotate_file(from_path, to_path, self.file_periods[key], is_java) + + + def _annotate_file(self, from_path, to_path, file_period, is_java): + """Annotate a source file. + + Annotate a source file in three steps: + 1. In the first line, show periods of this file. + 2. For each function, show periods of this function. + 3. For each line not hitting the same line as functions, show + line periods. + """ + log_info('annotate file %s' % from_path) + with open(from_path, 'r') as rf: + lines = rf.readlines() + + annotates = dict() + for line in file_period.line_dict.keys(): + annotates[line] = self._get_percentage_str(file_period.line_dict[line], True) + for func_name in file_period.function_dict.keys(): + func_start_line, period = file_period.function_dict[func_name] + if func_start_line == -1: + continue + line = func_start_line - 1 if is_java else func_start_line + annotates[line] = '[func] ' + self._get_percentage_str(period, True) + annotates[1] = '[file] ' + self._get_percentage_str(file_period.period, True) + + max_annotate_cols = 0 + for key in annotates.keys(): + max_annotate_cols = max(max_annotate_cols, len(annotates[key])) + + empty_annotate = ' ' * (max_annotate_cols + 6) + + dirname = os.path.dirname(to_path) + if not os.path.isdir(dirname): + os.makedirs(dirname) + with open(to_path, 'w') as wf: + for line in range(1, len(lines) + 1): + annotate = annotates.get(line) + if annotate is None: + annotate = empty_annotate + else: + annotate = '/* ' + annotate + ( + ' ' * (max_annotate_cols - len(annotate))) + ' */' + wf.write(annotate) + wf.write(lines[line-1]) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description='Annotate based on perf.data. See configurations in annotate.config.') + parser.add_argument('--config', default='annotate.config', + help='Set configuration file. Default is annotate.config.') + args = parser.parse_args() + config = load_config(args.config) + annotator = SourceFileAnnotator(config) + annotator.annotate() \ No newline at end of file