| #!/usr/bin/env python | |
| # | |
| # Copyright (C) 2016 The Android Open Source Project | |
| # | |
| # Licensed under the Apache License, Version 2.0 (the "License"); | |
| # you may not use this file except in compliance with the License. | |
| # You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| """annotate.py: annotate source files based on perf.data. | |
| """ | |
| import argparse | |
| import os | |
| import os.path | |
| import shutil | |
| import subprocess | |
| import sys | |
| from simpleperf_report_lib import * | |
| from utils import * | |
| class SourceLine(object): | |
| def __init__(self, file, function, line): | |
| self.file = file | |
| self.function = function | |
| self.line = line | |
| @property | |
| def file_key(self): | |
| return self.file | |
| @property | |
| def function_key(self): | |
| return (self.file, self.function) | |
| @property | |
| def line_key(self): | |
| return (self.file, self.line) | |
| # TODO: using addr2line can't convert from function_start_address to | |
| # source_file:line very well for java code. Because in .debug_line section, | |
| # there is some distance between function_start_address and the address | |
| # of the first instruction which can be mapped to source line. | |
| class Addr2Line(object): | |
| """collect information of how to map [dso_name,vaddr] to [source_file:line]. | |
| """ | |
| def __init__(self, addr2line_path, symfs_dir=None): | |
| self.dso_dict = dict() | |
| self.addr2line_path = addr2line_path | |
| self.symfs_dir = symfs_dir | |
| def add_addr(self, dso_name, addr): | |
| dso = self.dso_dict.get(dso_name) | |
| if dso is None: | |
| self.dso_dict[dso_name] = dso = dict() | |
| if addr not in dso: | |
| dso[addr] = None | |
| def convert_addrs_to_lines(self): | |
| # store a list of source files | |
| self.file_list = [] | |
| # map from file to id with file_list[id] == file | |
| self.file_dict = {} | |
| self.file_list.append('') | |
| self.file_dict[''] = 0 | |
| for dso_name in self.dso_dict.keys(): | |
| self._convert_addrs_to_lines(dso_name, self.dso_dict[dso_name]) | |
| self._combine_source_files() | |
| def _convert_addrs_to_lines(self, dso_name, dso): | |
| dso_path = self._find_dso_path(dso_name) | |
| if dso_path is None: | |
| log_warning("can't find dso '%s'" % dso_name) | |
| dso.clear() | |
| return | |
| addrs = sorted(dso.keys()) | |
| addr_str = [] | |
| for addr in addrs: | |
| addr_str.append('0x%x' % addr) | |
| addr_str = '\n'.join(addr_str) | |
| subproc = subprocess.Popen([self.addr2line_path, '-e', dso_path, '-aifC'], | |
| stdin=subprocess.PIPE, stdout=subprocess.PIPE) | |
| (stdoutdata, _) = subproc.communicate(str_to_bytes(addr_str)) | |
| stdoutdata = bytes_to_str(stdoutdata) | |
| stdoutdata = stdoutdata.strip().split('\n') | |
| if len(stdoutdata) < len(addrs): | |
| log_fatal("addr2line didn't output enough lines") | |
| addr_pos = 0 | |
| out_pos = 0 | |
| while addr_pos < len(addrs) and out_pos < len(stdoutdata): | |
| addr_line = stdoutdata[out_pos] | |
| out_pos += 1 | |
| assert addr_line[:2] == "0x" | |
| assert out_pos < len(stdoutdata) | |
| source_lines = [] | |
| while out_pos < len(stdoutdata) and stdoutdata[out_pos][:2] != "0x": | |
| function = stdoutdata[out_pos] | |
| out_pos += 1 | |
| assert out_pos < len(stdoutdata) | |
| # Handle lines like "C:\Users\...\file:32". | |
| items = stdoutdata[out_pos].rsplit(':', 1) | |
| if len(items) != 2: | |
| continue | |
| (file, line) = items | |
| line = line.split()[0] # Remove comments after line number | |
| out_pos += 1 | |
| if file.find('?') != -1: | |
| file = 0 | |
| else: | |
| file = self._get_file_id(file) | |
| if line.find('?') != -1: | |
| line = 0 | |
| else: | |
| line = int(line) | |
| source_lines.append(SourceLine(file, function, line)) | |
| dso[addrs[addr_pos]] = source_lines | |
| addr_pos += 1 | |
| assert addr_pos == len(addrs) | |
| def _get_file_id(self, file): | |
| id = self.file_dict.get(file) | |
| if id is None: | |
| id = len(self.file_list) | |
| self.file_list.append(file) | |
| self.file_dict[file] = id | |
| return id | |
| def _combine_source_files(self): | |
| """It is possible that addr2line gives us different names for the same | |
| file, like: | |
| /usr/local/.../src/main/jni/sudo-game-jni.cpp | |
| sudo-game-jni.cpp | |
| We'd better combine these two files. We can do it by combining | |
| source files with no conflicts in path. | |
| """ | |
| # Collect files having the same filename. | |
| filename_dict = dict() | |
| for file in self.file_list: | |
| index = max(file.rfind('/'), file.rfind(os.sep)) | |
| filename = file[index+1:] | |
| entry = filename_dict.get(filename) | |
| if entry is None: | |
| filename_dict[filename] = entry = [] | |
| entry.append(file) | |
| # Combine files having the same filename and having no conflicts in path. | |
| for filename in filename_dict.keys(): | |
| files = filename_dict[filename] | |
| if len(files) == 1: | |
| continue | |
| for file in files: | |
| to_file = file | |
| # Test if we can merge files[i] with another file having longer | |
| # path. | |
| for f in files: | |
| if len(f) > len(to_file) and f.find(file) != -1: | |
| to_file = f | |
| if to_file != file: | |
| from_id = self.file_dict[file] | |
| to_id = self.file_dict[to_file] | |
| self.file_list[from_id] = self.file_list[to_id] | |
| def get_sources(self, dso_name, addr): | |
| dso = self.dso_dict.get(dso_name) | |
| if dso is None: | |
| return [] | |
| item = dso.get(addr, []) | |
| source_lines = [] | |
| for source in item: | |
| source_lines.append(SourceLine(self.file_list[source.file], | |
| source.function, source.line)) | |
| return source_lines | |
| def _find_dso_path(self, dso): | |
| if dso[0] != '/' or dso == '//anon': | |
| return None | |
| if self.symfs_dir: | |
| dso_path = os.path.join(self.symfs_dir, dso[1:]) | |
| if os.path.isfile(dso_path): | |
| return dso_path | |
| if os.path.isfile(dso): | |
| return dso | |
| return None | |
| class Period(object): | |
| """event count information. It can be used to represent event count | |
| of a line, a function, a source file, or a binary. It contains two | |
| parts: period and acc_period. | |
| When used for a line, period is the event count occurred when running | |
| that line, acc_period is the accumulated event count occurred when | |
| running that line and functions called by that line. Same thing applies | |
| when it is used for a function, a source file, or a binary. | |
| """ | |
| def __init__(self, period=0, acc_period=0): | |
| self.period = period | |
| self.acc_period = acc_period | |
| def __iadd__(self, other): | |
| self.period += other.period | |
| self.acc_period += other.acc_period | |
| return self | |
| class DsoPeriod(object): | |
| """Period for each shared library""" | |
| def __init__(self, dso_name): | |
| self.dso_name = dso_name | |
| self.period = Period() | |
| def add_period(self, period): | |
| self.period += period | |
| class FilePeriod(object): | |
| """Period for each source file""" | |
| def __init__(self, file): | |
| self.file = file | |
| self.period = Period() | |
| # Period for each line in the file. | |
| self.line_dict = {} | |
| # Period for each function in the source file. | |
| self.function_dict = {} | |
| def add_period(self, period): | |
| self.period += period | |
| def add_line_period(self, line, period): | |
| a = self.line_dict.get(line) | |
| if a is None: | |
| self.line_dict[line] = a = Period() | |
| a += period | |
| def add_function_period(self, function_name, function_start_line, period): | |
| a = self.function_dict.get(function_name) | |
| if not a: | |
| if function_start_line is None: | |
| function_start_line = -1 | |
| self.function_dict[function_name] = a = [function_start_line, Period()] | |
| a[1] += period | |
| class SourceFileAnnotator(object): | |
| """group code for annotating source files""" | |
| def __init__(self, config): | |
| # check config variables | |
| config_names = ['perf_data_list', 'symfs_dir', 'source_dirs', | |
| 'annotate_dest_dir', 'comm_filters', 'pid_filters', | |
| 'tid_filters', 'dso_filters', 'addr2line_path'] | |
| for name in config_names: | |
| if name not in config: | |
| log_fatal('config [%s] is missing' % name) | |
| symfs_dir = config['symfs_dir'] | |
| if symfs_dir and not os.path.isdir(symfs_dir): | |
| log_fatal('[symfs_dir] "%s" is not a dir' % symfs_dir) | |
| kallsyms = config['kallsyms'] | |
| if kallsyms and not os.path.isfile(kallsyms): | |
| log_fatal('[kallsyms] "%s" is not a file' % kallsyms) | |
| source_dirs = config['source_dirs'] | |
| for dir in source_dirs: | |
| if not os.path.isdir(dir): | |
| log_fatal('[source_dirs] "%s" is not a dir' % dir) | |
| # init member variables | |
| self.config = config | |
| self.symfs_dir = config.get('symfs_dir') | |
| self.kallsyms = config.get('kallsyms') | |
| self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None | |
| if config.get('pid_filters'): | |
| self.pid_filter = {int(x) for x in config['pid_filters']} | |
| else: | |
| self.pid_filter = None | |
| if config.get('tid_filters'): | |
| self.tid_filter = {int(x) for x in config['tid_filters']} | |
| else: | |
| self.tid_filter = None | |
| self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None | |
| output_dir = config['annotate_dest_dir'] | |
| if os.path.isdir(output_dir): | |
| shutil.rmtree(output_dir) | |
| os.makedirs(output_dir) | |
| self.addr2line = Addr2Line(self.config['addr2line_path'], symfs_dir) | |
| def annotate(self): | |
| self._collect_addrs() | |
| self._convert_addrs_to_lines() | |
| self._generate_periods() | |
| self._write_summary() | |
| self._collect_source_files() | |
| self._annotate_files() | |
| def _collect_addrs(self): | |
| """Read perf.data, collect all addresses we need to convert to | |
| source file:line. | |
| """ | |
| for perf_data in self.config['perf_data_list']: | |
| lib = ReportLib() | |
| lib.SetRecordFile(perf_data) | |
| if self.symfs_dir: | |
| lib.SetSymfs(self.symfs_dir) | |
| if self.kallsyms: | |
| lib.SetKallsymsFile(self.kallsyms) | |
| while True: | |
| sample = lib.GetNextSample() | |
| if sample is None: | |
| lib.Close() | |
| break | |
| if not self._filter_sample(sample): | |
| continue | |
| symbols = [] | |
| symbols.append(lib.GetSymbolOfCurrentSample()) | |
| callchain = lib.GetCallChainOfCurrentSample() | |
| for i in range(callchain.nr): | |
| symbols.append(callchain.entries[i].symbol) | |
| for symbol in symbols: | |
| if self._filter_symbol(symbol): | |
| self.addr2line.add_addr(symbol.dso_name, symbol.vaddr_in_file) | |
| self.addr2line.add_addr(symbol.dso_name, symbol.symbol_addr) | |
| def _filter_sample(self, sample): | |
| """Return true if the sample can be used.""" | |
| if self.comm_filter: | |
| if sample.thread_comm not in self.comm_filter: | |
| return False | |
| if self.pid_filter: | |
| if sample.pid not in self.pid_filter: | |
| return False | |
| if self.tid_filter: | |
| if sample.tid not in self.tid_filter: | |
| return False | |
| return True | |
| def _filter_symbol(self, symbol): | |
| if not self.dso_filter or symbol.dso_name in self.dso_filter: | |
| return True | |
| return False | |
| def _convert_addrs_to_lines(self): | |
| self.addr2line.convert_addrs_to_lines() | |
| def _generate_periods(self): | |
| """read perf.data, collect Period for all types: | |
| binaries, source files, functions, lines. | |
| """ | |
| self.period = 0 | |
| self.dso_periods = dict() | |
| self.file_periods = dict() | |
| for perf_data in self.config['perf_data_list']: | |
| lib = ReportLib() | |
| lib.SetRecordFile(perf_data) | |
| if self.symfs_dir: | |
| lib.SetSymfs(self.symfs_dir) | |
| if self.kallsyms: | |
| lib.SetKallsymsFile(self.kallsyms) | |
| while True: | |
| sample = lib.GetNextSample() | |
| if sample is None: | |
| lib.Close() | |
| break | |
| if not self._filter_sample(sample): | |
| continue | |
| symbols = [] | |
| symbols.append(lib.GetSymbolOfCurrentSample()) | |
| callchain = lib.GetCallChainOfCurrentSample() | |
| for i in range(callchain.nr): | |
| symbols.append(callchain.entries[i].symbol) | |
| # Each sample has a callchain, but its period is only used once | |
| # to add period for each function/source_line/source_file/binary. | |
| # For example, if more than one entry in the callchain hits a | |
| # function, the event count of that function is only increased once. | |
| # Otherwise, we may get periods > 100%. | |
| is_sample_used = False | |
| used_dso_dict = dict() | |
| used_file_dict = dict() | |
| used_function_dict = dict() | |
| used_line_dict = dict() | |
| period = Period(sample.period, sample.period) | |
| for i in range(len(symbols)): | |
| symbol = symbols[i] | |
| if i == 1: | |
| period = Period(0, sample.period) | |
| if not self._filter_symbol(symbol): | |
| continue | |
| is_sample_used = True | |
| # Add period to dso. | |
| self._add_dso_period(symbol.dso_name, period, used_dso_dict) | |
| # Add period to source file. | |
| sources = self.addr2line.get_sources(symbol.dso_name, symbol.vaddr_in_file) | |
| for source in sources: | |
| if source.file: | |
| self._add_file_period(source, period, used_file_dict) | |
| # Add period to line. | |
| if source.line: | |
| self._add_line_period(source, period, used_line_dict) | |
| # Add period to function. | |
| sources = self.addr2line.get_sources(symbol.dso_name, symbol.symbol_addr) | |
| for source in sources: | |
| if source.file: | |
| self._add_file_period(source, period, used_file_dict) | |
| if source.function: | |
| self._add_function_period(source, period, used_function_dict) | |
| if is_sample_used: | |
| self.period += sample.period | |
| def _add_dso_period(self, dso_name, period, used_dso_dict): | |
| if dso_name not in used_dso_dict: | |
| used_dso_dict[dso_name] = True | |
| dso_period = self.dso_periods.get(dso_name) | |
| if dso_period is None: | |
| dso_period = self.dso_periods[dso_name] = DsoPeriod(dso_name) | |
| dso_period.add_period(period) | |
| def _add_file_period(self, source, period, used_file_dict): | |
| if source.file_key not in used_file_dict: | |
| used_file_dict[source.file_key] = True | |
| file_period = self.file_periods.get(source.file) | |
| if file_period is None: | |
| file_period = self.file_periods[source.file] = FilePeriod(source.file) | |
| file_period.add_period(period) | |
| def _add_line_period(self, source, period, used_line_dict): | |
| if source.line_key not in used_line_dict: | |
| used_line_dict[source.line_key] = True | |
| file_period = self.file_periods[source.file] | |
| file_period.add_line_period(source.line, period) | |
| def _add_function_period(self, source, period, used_function_dict): | |
| if source.function_key not in used_function_dict: | |
| used_function_dict[source.function_key] = True | |
| file_period = self.file_periods[source.file] | |
| file_period.add_function_period(source.function, source.line, period) | |
| def _write_summary(self): | |
| summary = os.path.join(self.config['annotate_dest_dir'], 'summary') | |
| with open(summary, 'w') as f: | |
| f.write('total period: %d\n\n' % self.period) | |
| dso_periods = sorted(self.dso_periods.values(), | |
| key=lambda x: x.period.acc_period, reverse=True) | |
| for dso_period in dso_periods: | |
| f.write('dso %s: %s\n' % (dso_period.dso_name, | |
| self._get_percentage_str(dso_period.period))) | |
| f.write('\n') | |
| file_periods = sorted(self.file_periods.values(), | |
| key=lambda x: x.period.acc_period, reverse=True) | |
| for file_period in file_periods: | |
| f.write('file %s: %s\n' % (file_period.file, | |
| self._get_percentage_str(file_period.period))) | |
| for file_period in file_periods: | |
| f.write('\n\n%s: %s\n' % (file_period.file, | |
| self._get_percentage_str(file_period.period))) | |
| values = [] | |
| for func_name in file_period.function_dict.keys(): | |
| func_start_line, period = file_period.function_dict[func_name] | |
| values.append((func_name, func_start_line, period)) | |
| values = sorted(values, key=lambda x: x[2].acc_period, reverse=True) | |
| for value in values: | |
| f.write('\tfunction (%s): line %d, %s\n' % ( | |
| value[0], value[1], self._get_percentage_str(value[2]))) | |
| f.write('\n') | |
| for line in sorted(file_period.line_dict.keys()): | |
| f.write('\tline %d: %s\n' % ( | |
| line, self._get_percentage_str(file_period.line_dict[line]))) | |
| def _get_percentage_str(self, period, short=False): | |
| s = 'acc_p: %f%%, p: %f%%' if short else 'accumulated_period: %f%%, period: %f%%' | |
| return s % self._get_percentage(period) | |
| def _get_percentage(self, period): | |
| if self.period == 0: | |
| return (0, 0) | |
| acc_p = 100.0 * period.acc_period / self.period | |
| p = 100.0 * period.period / self.period | |
| return (acc_p, p) | |
| def _collect_source_files(self): | |
| self.source_file_dict = dict() | |
| source_file_suffix = ['h', 'c', 'cpp', 'cc', 'java', 'kt'] | |
| for source_dir in self.config['source_dirs']: | |
| for root, _, files in os.walk(source_dir): | |
| for file in files: | |
| if file[file.rfind('.')+1:] in source_file_suffix: | |
| entry = self.source_file_dict.get(file) | |
| if entry is None: | |
| entry = self.source_file_dict[file] = [] | |
| entry.append(os.path.join(root, file)) | |
| def _find_source_file(self, file): | |
| filename = file[file.rfind(os.sep)+1:] | |
| source_files = self.source_file_dict.get(filename) | |
| if source_files is None: | |
| return None | |
| match_count = 0 | |
| result = None | |
| for path in source_files: | |
| if path.find(file) != -1: | |
| match_count += 1 | |
| result = path | |
| if match_count > 1: | |
| log_warning('multiple source for %s, select %s' % (file, result)) | |
| return result | |
| def _annotate_files(self): | |
| """Annotate Source files: add acc_period/period for each source file. | |
| 1. Annotate java source files, which have $JAVA_SRC_ROOT prefix. | |
| 2. Annotate c++ source files. | |
| """ | |
| dest_dir = self.config['annotate_dest_dir'] | |
| for key in self.file_periods.keys(): | |
| is_java = False | |
| if key.startswith('$JAVA_SRC_ROOT/'): | |
| path = key[len('$JAVA_SRC_ROOT/'):] | |
| items = path.split('/') | |
| path = os.sep.join(items) | |
| from_path = self._find_source_file(path) | |
| to_path = os.path.join(dest_dir, 'java', path) | |
| is_java = True | |
| elif key.startswith('/') and os.path.isfile(key): | |
| path = key | |
| from_path = path | |
| to_path = os.path.join(dest_dir, path[1:]) | |
| elif is_windows() and key.find(':\\') != -1 and os.path.isfile(key): | |
| from_path = key | |
| to_path = os.path.join(dest_dir, key.replace(':\\', '\\')) | |
| else: | |
| path = key[1:] if key.startswith('/') else key | |
| # Change path on device to path on host | |
| path = os.sep.join(path.split('/')) | |
| from_path = self._find_source_file(path) | |
| to_path = os.path.join(dest_dir, path) | |
| if from_path is None: | |
| log_warning("can't find source file for path %s" % key) | |
| continue | |
| self._annotate_file(from_path, to_path, self.file_periods[key], is_java) | |
| def _annotate_file(self, from_path, to_path, file_period, is_java): | |
| """Annotate a source file. | |
| Annotate a source file in three steps: | |
| 1. In the first line, show periods of this file. | |
| 2. For each function, show periods of this function. | |
| 3. For each line not hitting the same line as functions, show | |
| line periods. | |
| """ | |
| log_info('annotate file %s' % from_path) | |
| with open(from_path, 'r') as rf: | |
| lines = rf.readlines() | |
| annotates = dict() | |
| for line in file_period.line_dict.keys(): | |
| annotates[line] = self._get_percentage_str(file_period.line_dict[line], True) | |
| for func_name in file_period.function_dict.keys(): | |
| func_start_line, period = file_period.function_dict[func_name] | |
| if func_start_line == -1: | |
| continue | |
| line = func_start_line - 1 if is_java else func_start_line | |
| annotates[line] = '[func] ' + self._get_percentage_str(period, True) | |
| annotates[1] = '[file] ' + self._get_percentage_str(file_period.period, True) | |
| max_annotate_cols = 0 | |
| for key in annotates.keys(): | |
| max_annotate_cols = max(max_annotate_cols, len(annotates[key])) | |
| empty_annotate = ' ' * (max_annotate_cols + 6) | |
| dirname = os.path.dirname(to_path) | |
| if not os.path.isdir(dirname): | |
| os.makedirs(dirname) | |
| with open(to_path, 'w') as wf: | |
| for line in range(1, len(lines) + 1): | |
| annotate = annotates.get(line) | |
| if annotate is None: | |
| if not lines[line-1].strip(): | |
| annotate = '' | |
| else: | |
| annotate = empty_annotate | |
| else: | |
| annotate = '/* ' + annotate + ( | |
| ' ' * (max_annotate_cols - len(annotate))) + ' */' | |
| wf.write(annotate) | |
| wf.write(lines[line-1]) | |
| if __name__ == '__main__': | |
| parser = argparse.ArgumentParser( | |
| description='Annotate based on perf.data. See configurations in annotate.config.') | |
| parser.add_argument('--config', default='annotate.config', | |
| help='Set configuration file. Default is annotate.config.') | |
| args = parser.parse_args() | |
| config = load_config(args.config) | |
| annotator = SourceFileAnnotator(config) | |
| annotator.annotate() | |
| log_info('annotate finish successfully, please check result in annotated_files/.') |