annotate.py

#!/usr/bin/env python3
#
# Copyright (C) 2016 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""annotate.py: annotate source files based on perf.data.
"""

import logging
import os
import os.path
import shutil

from simpleperf_report_lib import ReportLib
from simpleperf_utils import (
    Addr2Nearestline, BaseArgumentParser, BinaryFinder, extant_dir, flatten_arg_list, is_windows,
    log_exit, ReadElf, SourceFileSearcher)


class SourceLine(object):
    def __init__(self, file_id, function, line):
        self.file = file_id
        self.function = function
        self.line = line

    @property
    def file_key(self):
        return self.file

    @property
    def function_key(self):
        return (self.file, self.function)

    @property
    def line_key(self):
        return (self.file, self.line)


class Addr2Line(object):
    """collect information of how to map [dso_name, vaddr] to [source_file:line].
    """

    def __init__(self, ndk_path, binary_cache_path, source_dirs):
        binary_finder = BinaryFinder(binary_cache_path, ReadElf(ndk_path))
        self.addr2line = Addr2Nearestline(ndk_path, binary_finder, True)
        self.source_searcher = SourceFileSearcher(source_dirs)

    def add_addr(self, dso_path: str, build_id: str, func_addr: int, addr: int):
        self.addr2line.add_addr(dso_path, build_id, func_addr, addr)

    def convert_addrs_to_lines(self):
        self.addr2line.convert_addrs_to_lines(jobs=os.cpu_count())

    def get_sources(self, dso_path, addr):
        dso = self.addr2line.get_dso(dso_path)
        if not dso:
            return []
        source = self.addr2line.get_addr_source(dso, addr)
        if not source:
            return []
        result = []
        for (source_file, source_line, function_name) in source:
            source_file_path = self.source_searcher.get_real_path(source_file)
            if not source_file_path:
                source_file_path = source_file
            result.append(SourceLine(source_file_path, function_name, source_line))
        return result


class Period(object):
    """event count information. It can be used to represent event count
       of a line, a function, a source file, or a binary. It contains two
       parts: period and acc_period.
       When used for a line, period is the event count occurred when running
       that line, acc_period is the accumulated event count occurred when
       running that line and functions called by that line. Same thing applies
       when it is used for a function, a source file, or a binary.
    """

    def __init__(self, period=0, acc_period=0):
        self.period = period
        self.acc_period = acc_period

    def __iadd__(self, other):
        self.period += other.period
        self.acc_period += other.acc_period
        return self


class DsoPeriod(object):
    """Period for each shared library"""

    def __init__(self, dso_name):
        self.dso_name = dso_name
        self.period = Period()

    def add_period(self, period):
        self.period += period


class FilePeriod(object):
    """Period for each source file"""

    def __init__(self, file_id):
        self.file = file_id
        self.period = Period()
        # Period for each line in the file.
        self.line_dict = {}
        # Period for each function in the source file.
        self.function_dict = {}

    def add_period(self, period):
        self.period += period

    def add_line_period(self, line, period):
        a = self.line_dict.get(line)
        if a is None:
            self.line_dict[line] = a = Period()
        a += period

    def add_function_period(self, function_name, function_start_line, period):
        a = self.function_dict.get(function_name)
        if not a:
            if function_start_line is None:
                function_start_line = -1
            self.function_dict[function_name] = a = [function_start_line, Period()]
        a[1] += period


class SourceFileAnnotator(object):
    """group code for annotating source files"""

    def __init__(self, config):
        # check config variables
        config_names = ['perf_data_list', 'source_dirs', 'comm_filters',
                        'pid_filters', 'tid_filters', 'dso_filters', 'ndk_path']
        for name in config_names:
            if name not in config:
                log_exit('config [%s] is missing' % name)
        symfs_dir = 'binary_cache'
        if not os.path.isdir(symfs_dir):
            symfs_dir = None
        kallsyms = 'binary_cache/kallsyms'
        if not os.path.isfile(kallsyms):
            kallsyms = None

        # init member variables
        self.config = config
        self.symfs_dir = symfs_dir
        self.kallsyms = kallsyms
        self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None
        if config.get('pid_filters'):
            self.pid_filter = {int(x) for x in config['pid_filters']}
        else:
            self.pid_filter = None
        if config.get('tid_filters'):
            self.tid_filter = {int(x) for x in config['tid_filters']}
        else:
            self.tid_filter = None
        self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None

        config['annotate_dest_dir'] = 'annotated_files'
        output_dir = config['annotate_dest_dir']
        if os.path.isdir(output_dir):
            shutil.rmtree(output_dir)
        os.makedirs(output_dir)

        self.addr2line = Addr2Line(self.config['ndk_path'], symfs_dir, config.get('source_dirs'))
        self.period = 0
        self.dso_periods = {}
        self.file_periods = {}

    def annotate(self):
        self._collect_addrs()
        self._convert_addrs_to_lines()
        self._generate_periods()
        self._write_summary()
        self._annotate_files()

    def _collect_addrs(self):
        """Read perf.data, collect all addresses we need to convert to
           source file:line.
        """
        for perf_data in self.config['perf_data_list']:
            lib = ReportLib()
            lib.SetRecordFile(perf_data)
            if self.symfs_dir:
                lib.SetSymfs(self.symfs_dir)
            if self.kallsyms:
                lib.SetKallsymsFile(self.kallsyms)
            while True:
                sample = lib.GetNextSample()
                if sample is None:
                    lib.Close()
                    break
                if not self._filter_sample(sample):
                    continue
                symbols = []
                symbols.append(lib.GetSymbolOfCurrentSample())
                callchain = lib.GetCallChainOfCurrentSample()
                for i in range(callchain.nr):
                    symbols.append(callchain.entries[i].symbol)
                for symbol in symbols:
                    if self._filter_symbol(symbol):
                        build_id = lib.GetBuildIdForPath(symbol.dso_name)
                        self.addr2line.add_addr(symbol.dso_name, build_id, symbol.symbol_addr,
                                                symbol.vaddr_in_file)
                        self.addr2line.add_addr(symbol.dso_name, build_id, symbol.symbol_addr,
                                                symbol.symbol_addr)

    def _filter_sample(self, sample):
        """Return true if the sample can be used."""
        if self.comm_filter:
            if sample.thread_comm not in self.comm_filter:
                return False
        if self.pid_filter:
            if sample.pid not in self.pid_filter:
                return False
        if self.tid_filter:
            if sample.tid not in self.tid_filter:
                return False
        return True

    def _filter_symbol(self, symbol):
        if not self.dso_filter or symbol.dso_name in self.dso_filter:
            return True
        return False

    def _convert_addrs_to_lines(self):
        self.addr2line.convert_addrs_to_lines()

    def _generate_periods(self):
        """read perf.data, collect Period for all types:
            binaries, source files, functions, lines.
        """
        for perf_data in self.config['perf_data_list']:
            lib = ReportLib()
            lib.SetRecordFile(perf_data)
            if self.symfs_dir:
                lib.SetSymfs(self.symfs_dir)
            if self.kallsyms:
                lib.SetKallsymsFile(self.kallsyms)
            while True:
                sample = lib.GetNextSample()
                if sample is None:
                    lib.Close()
                    break
                if not self._filter_sample(sample):
                    continue
                self._generate_periods_for_sample(lib, sample)

    def _generate_periods_for_sample(self, lib, sample):
        symbols = []
        symbols.append(lib.GetSymbolOfCurrentSample())
        callchain = lib.GetCallChainOfCurrentSample()
        for i in range(callchain.nr):
            symbols.append(callchain.entries[i].symbol)
        # Each sample has a callchain, but its period is only used once
        # to add period for each function/source_line/source_file/binary.
        # For example, if more than one entry in the callchain hits a
        # function, the event count of that function is only increased once.
        # Otherwise, we may get periods > 100%.
        is_sample_used = False
        used_dso_dict = {}
        used_file_dict = {}
        used_function_dict = {}
        used_line_dict = {}
        period = Period(sample.period, sample.period)
        for j, symbol in enumerate(symbols):
            if j == 1:
                period = Period(0, sample.period)
            if not self._filter_symbol(symbol):
                continue
            is_sample_used = True
            # Add period to dso.
            self._add_dso_period(symbol.dso_name, period, used_dso_dict)
            # Add period to source file.
            sources = self.addr2line.get_sources(symbol.dso_name, symbol.vaddr_in_file)
            for source in sources:
                if source.file:
                    self._add_file_period(source, period, used_file_dict)
                    # Add period to line.
                    if source.line:
                        self._add_line_period(source, period, used_line_dict)
            # Add period to function.
            sources = self.addr2line.get_sources(symbol.dso_name, symbol.symbol_addr)
            for source in sources:
                if source.file:
                    self._add_file_period(source, period, used_file_dict)
                    if source.function:
                        self._add_function_period(source, period, used_function_dict)

        if is_sample_used:
            self.period += sample.period

    def _add_dso_period(self, dso_name, period, used_dso_dict):
        if dso_name not in used_dso_dict:
            used_dso_dict[dso_name] = True
            dso_period = self.dso_periods.get(dso_name)
            if dso_period is None:
                dso_period = self.dso_periods[dso_name] = DsoPeriod(dso_name)
            dso_period.add_period(period)

    def _add_file_period(self, source, period, used_file_dict):
        if source.file_key not in used_file_dict:
            used_file_dict[source.file_key] = True
            file_period = self.file_periods.get(source.file)
            if file_period is None:
                file_period = self.file_periods[source.file] = FilePeriod(source.file)
            file_period.add_period(period)

    def _add_line_period(self, source, period, used_line_dict):
        if source.line_key not in used_line_dict:
            used_line_dict[source.line_key] = True
            file_period = self.file_periods[source.file]
            file_period.add_line_period(source.line, period)

    def _add_function_period(self, source, period, used_function_dict):
        if source.function_key not in used_function_dict:
            used_function_dict[source.function_key] = True
            file_period = self.file_periods[source.file]
            file_period.add_function_period(source.function, source.line, period)

    def _write_summary(self):
        summary = os.path.join(self.config['annotate_dest_dir'], 'summary')
        with open(summary, 'w') as f:
            f.write('total period: %d\n\n' % self.period)
            dso_periods = sorted(self.dso_periods.values(),
                                 key=lambda x: x.period.acc_period, reverse=True)
            for dso_period in dso_periods:
                f.write('dso %s: %s\n' % (dso_period.dso_name,
                                          self._get_percentage_str(dso_period.period)))
            f.write('\n')

            file_periods = sorted(self.file_periods.values(),
                                  key=lambda x: x.period.acc_period, reverse=True)
            for file_period in file_periods:
                f.write('file %s: %s\n' % (file_period.file,
                                           self._get_percentage_str(file_period.period)))
            for file_period in file_periods:
                f.write('\n\n%s: %s\n' % (file_period.file,
                                          self._get_percentage_str(file_period.period)))
                values = []
                for func_name in file_period.function_dict.keys():
                    func_start_line, period = file_period.function_dict[func_name]
                    values.append((func_name, func_start_line, period))
                values = sorted(values, key=lambda x: x[2].acc_period, reverse=True)
                for value in values:
                    f.write('\tfunction (%s): line %d, %s\n' % (
                        value[0], value[1], self._get_percentage_str(value[2])))
                f.write('\n')
                for line in sorted(file_period.line_dict.keys()):
                    f.write('\tline %d: %s\n' % (
                        line, self._get_percentage_str(file_period.line_dict[line])))

    def _get_percentage_str(self, period, short=False):
        s = 'acc_p: %f%%, p: %f%%' if short else 'accumulated_period: %f%%, period: %f%%'
        return s % self._get_percentage(period)

    def _get_percentage(self, period):
        if self.period == 0:
            return (0, 0)
        acc_p = 100.0 * period.acc_period / self.period
        p = 100.0 * period.period / self.period
        return (acc_p, p)

    def _annotate_files(self):
        """Annotate Source files: add acc_period/period for each source file.
           1. Annotate java source files, which have $JAVA_SRC_ROOT prefix.
           2. Annotate c++ source files.
        """
        dest_dir = self.config['annotate_dest_dir']
        for key in self.file_periods:
            from_path = key
            if not os.path.isfile(from_path):
                logging.warning("can't find source file for path %s" % from_path)
                continue
            if from_path.startswith('/'):
                to_path = os.path.join(dest_dir, from_path[1:])
            elif is_windows() and ':\\' in from_path:
                to_path = os.path.join(dest_dir, from_path.replace(':\\', os.sep))
            else:
                to_path = os.path.join(dest_dir, from_path)
            is_java = from_path.endswith('.java')
            self._annotate_file(from_path, to_path, self.file_periods[key], is_java)

    def _annotate_file(self, from_path, to_path, file_period, is_java):
        """Annotate a source file.

        Annotate a source file in three steps:
          1. In the first line, show periods of this file.
          2. For each function, show periods of this function.
          3. For each line not hitting the same line as functions, show
             line periods.
        """
        logging.info('annotate file %s' % from_path)
        with open(from_path, 'r') as rf:
            lines = rf.readlines()

        annotates = {}
        for line in file_period.line_dict.keys():
            annotates[line] = self._get_percentage_str(file_period.line_dict[line], True)
        for func_name in file_period.function_dict.keys():
            func_start_line, period = file_period.function_dict[func_name]
            if func_start_line == -1:
                continue
            line = func_start_line - 1 if is_java else func_start_line
            annotates[line] = '[func] ' + self._get_percentage_str(period, True)
        annotates[1] = '[file] ' + self._get_percentage_str(file_period.period, True)

        max_annotate_cols = 0
        for key in annotates:
            max_annotate_cols = max(max_annotate_cols, len(annotates[key]))

        empty_annotate = ' ' * (max_annotate_cols + 6)

        dirname = os.path.dirname(to_path)
        if not os.path.isdir(dirname):
            os.makedirs(dirname)
        with open(to_path, 'w') as wf:
            for line in range(1, len(lines) + 1):
                annotate = annotates.get(line)
                if annotate is None:
                    if not lines[line-1].strip():
                        annotate = ''
                    else:
                        annotate = empty_annotate
                else:
                    annotate = '/* ' + annotate + (
                        ' ' * (max_annotate_cols - len(annotate))) + ' */'
                wf.write(annotate)
                wf.write(lines[line-1])


def main():
    parser = BaseArgumentParser(description="""
        Annotate source files based on profiling data. It reads line information from binary_cache
        generated by app_profiler.py or binary_cache_builder.py, and generate annotated source
        files in annotated_files directory.""")
    parser.add_argument('-i', '--perf_data_list', nargs='+', action='append', help="""
        The paths of profiling data. Default is perf.data.""")
    parser.add_argument('-s', '--source_dirs', type=extant_dir, nargs='+', action='append', help="""
        Directories to find source files.""")
    parser.add_argument('--comm', nargs='+', action='append', help="""
        Use samples only in threads with selected names.""")
    parser.add_argument('--pid', nargs='+', action='append', help="""
        Use samples only in processes with selected process ids.""")
    parser.add_argument('--tid', nargs='+', action='append', help="""
        Use samples only in threads with selected thread ids.""")
    parser.add_argument('--dso', nargs='+', action='append', help="""
        Use samples only in selected binaries.""")
    parser.add_argument('--ndk_path', type=extant_dir, help='Set the path of a ndk release.')

    args = parser.parse_args()
    config = {}
    config['perf_data_list'] = flatten_arg_list(args.perf_data_list)
    if not config['perf_data_list']:
        config['perf_data_list'].append('perf.data')
    config['source_dirs'] = flatten_arg_list(args.source_dirs)
    config['comm_filters'] = flatten_arg_list(args.comm)
    config['pid_filters'] = flatten_arg_list(args.pid)
    config['tid_filters'] = flatten_arg_list(args.tid)
    config['dso_filters'] = flatten_arg_list(args.dso)
    config['ndk_path'] = args.ndk_path

    annotator = SourceFileAnnotator(config)
    annotator.annotate()
    logging.info('annotate finish successfully, please check result in annotated_files/.')


if __name__ == '__main__':
    main()