| #!/usr/bin/env python3 |
| # |
| # Copyright (C) 2021 The Android Open Source Project |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| # |
| |
| """gecko_profile_generator.py: converts perf.data to Gecko Profile Format, |
| which can be read by https://profiler.firefox.com/. |
| |
| Example: |
| ./app_profiler.py |
| ./gecko_profile_generator.py | gzip > gecko-profile.json.gz |
| |
| Then open gecko-profile.json.gz in https://profiler.firefox.com/ |
| """ |
| |
| from collections import Counter |
| from dataclasses import dataclass, field |
| from enum import Enum, unique |
| import json |
| import logging |
| import sys |
| from typing import List, Dict, Optional, NamedTuple, Tuple |
| |
| from simpleperf_report_lib import GetReportLib |
| from simpleperf_utils import BaseArgumentParser, ReportLibOptions |
| |
| |
| StringID = int |
| StackID = int |
| FrameID = int |
| CategoryID = int |
| Milliseconds = float |
| GeckoProfile = Dict |
| |
| |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 |
| class GeckoFrame(NamedTuple): |
| string_id: StringID |
| relevantForJS: bool |
| innerWindowID: int |
| implementation: None |
| optimizations: None |
| line: None |
| column: None |
| category: CategoryID |
| subcategory: int |
| |
| |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 |
| class GeckoStack(NamedTuple): |
| prefix_id: Optional[StackID] |
| frame_id: FrameID |
| category_id: CategoryID |
| |
| |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 |
| class GeckoSample(NamedTuple): |
| stack_id: Optional[StackID] |
| time_ms: Milliseconds |
| responsiveness: int |
| complete_stack: bool |
| |
| def to_json(self): |
| return [self.stack_id, self.time_ms, self.responsiveness] |
| |
| |
| # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/profile.js#L425 |
| # Colors must be defined in: |
| # https://github.com/firefox-devtools/profiler/blob/50124adbfa488adba6e2674a8f2618cf34b59cd2/res/css/categories.css |
| @unique |
| class Category(Enum): |
| # Follow Brendan Gregg's Flamegraph convention: yellow for userland |
| # https://github.com/brendangregg/FlameGraph/blob/810687f180f3c4929b5d965f54817a5218c9d89b/flamegraph.pl#L419 |
| USER = 0, 'User', 'yellow' |
| # Follow Brendan Gregg's Flamegraph convention: orange for kernel |
| # https://github.com/brendangregg/FlameGraph/blob/810687f180f3c4929b5d965f54817a5218c9d89b/flamegraph.pl#L417 |
| KERNEL = 1, 'Kernel', 'orange' |
| # Follow Brendan Gregg's Flamegraph convention: yellow for userland |
| # https://github.com/brendangregg/FlameGraph/blob/810687f180f3c4929b5d965f54817a5218c9d89b/flamegraph.pl#L419 |
| NATIVE = 2, 'Native', 'yellow' |
| # Follow Brendan Gregg's Flamegraph convention: green for Java/JIT |
| # https://github.com/brendangregg/FlameGraph/blob/810687f180f3c4929b5d965f54817a5218c9d89b/flamegraph.pl#L411 |
| DEX = 3, 'DEX', 'green' |
| OAT = 4, 'OAT', 'green' |
| # Follow Brendan Gregg's Flamegraph convention: blue for off-CPU |
| # https://github.com/brendangregg/FlameGraph/blob/810687f180f3c4929b5d965f54817a5218c9d89b/flamegraph.pl#L470 |
| OFF_CPU = 5, 'Off-CPU', 'blue' |
| # Not used by this exporter yet, but some Firefox Profiler code assumes |
| # there is an 'Other' category by searching for a category with |
| # color=grey, so include this. |
| OTHER = 6, 'Other', 'grey' |
| # Follow Brendan Gregg's Flamegraph convention: green for Java/JIT |
| # https://github.com/brendangregg/FlameGraph/blob/810687f180f3c4929b5d965f54817a5218c9d89b/flamegraph.pl#L411 |
| JIT = 7, 'JIT', 'green' |
| |
| def __init__(self, value, label, color): |
| self._value_ = value |
| self.label = label |
| self.color = color |
| |
| @classmethod |
| def to_json(cls): |
| return [{ |
| "name": enum.label, |
| "color": enum.color, |
| # We don't use subcategories, but Firefox Profiler seems to require it. |
| "subcategories":['Other'] |
| } for enum in cls] |
| |
| |
| @dataclass |
| class StackFrame: |
| symbol_name: str |
| dso_name: str |
| |
| def to_gecko_frame_string(self): |
| return '%s (in %s)' % (self.symbol_name, self.dso_name) |
| |
| def category(self) -> Category: |
| # Heuristic: kernel code contains "kallsyms" as the library name. |
| if self.dso_name == "[kernel.kallsyms]" or self.dso_name.endswith(".ko"): |
| # Heuristic: empirically, off-CPU profiles mostly measure off-CPU |
| # time accounted to the linux kernel __schedule function, which |
| # handles blocking. This only works if we have kernel symbol |
| # (kallsyms) access though. __schedule defined here: |
| # https://cs.android.com/android/kernel/superproject/+/common-android-mainline:common/kernel/sched/core.c;l=6593;drc=0c99414a07ddaa18d8eb4be90b551d2687cbde2f |
| if self.symbol_name == "__schedule": |
| return Category.OFF_CPU |
| return Category.KERNEL |
| elif self.dso_name.endswith(".so"): |
| return Category.NATIVE |
| elif self.dso_name.endswith(".vdex"): |
| return Category.DEX |
| # APKs are full of dex code. |
| elif self.dso_name.endswith(".apk"): |
| return Category.DEX |
| # /system/framework/ has .jar files which seem to be full of .dex code. |
| elif self.dso_name.endswith(".jar"): |
| return Category.DEX |
| elif self.dso_name.endswith(".oat"): |
| return Category.OAT |
| # In ART, odex is just OAT code |
| elif self.dso_name.endswith(".odex"): |
| return Category.OAT |
| # "[JIT app cache]" is returned for JIT code here: |
| # https://cs.android.com/android/platform/superproject/+/master:system/extras/simpleperf/dso.cpp;l=551;drc=4d8137f55782cc1e8cc93e4694ba3a7159d9a2bc |
| elif self.dso_name == "[JIT app cache]": |
| return Category.JIT |
| return Category.USER |
| |
| |
| def is_complete_stack(stack: List[StackFrame]) -> bool: |
| """ Check if the callstack is complete. The stack starts from root. """ |
| for frame in stack: |
| if frame.symbol_name == '__libc_init' or frame.symbol_name == '__start_thread': |
| return True |
| return False |
| |
| |
| @dataclass |
| class Thread: |
| """A builder for a profile of a single thread. |
| |
| Attributes: |
| comm: Thread command-line (name). |
| pid: process ID of containing process. |
| tid: thread ID. |
| samples: Timeline of profile samples. |
| frameTable: interned stack frame ID -> stack frame. |
| stringTable: interned string ID -> string. |
| stringMap: interned string -> string ID. |
| stackTable: interned stack ID -> stack. |
| stackMap: (stack prefix ID, leaf stack frame ID) -> interned Stack ID. |
| frameMap: Stack Frame string -> interned Frame ID. |
| """ |
| comm: str |
| pid: int |
| tid: int |
| samples: List[GeckoSample] = field(default_factory=list) |
| frameTable: List[GeckoFrame] = field(default_factory=list) |
| stringTable: List[str] = field(default_factory=list) |
| # TODO: this is redundant with frameTable, could we remove this? |
| stringMap: Dict[str, int] = field(default_factory=dict) |
| stackTable: List[GeckoStack] = field(default_factory=list) |
| stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) |
| frameMap: Dict[str, int] = field(default_factory=dict) |
| |
| def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: |
| """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" |
| key = (prefix_id, frame_id) |
| stack_id = self.stackMap.get(key) |
| if stack_id is not None: |
| return stack_id |
| stack_id = len(self.stackTable) |
| self.stackTable.append(GeckoStack( |
| prefix_id=prefix_id, |
| frame_id=frame_id, |
| category_id=0, |
| )) |
| self.stackMap[key] = stack_id |
| return stack_id |
| |
| def _intern_string(self, string: str) -> int: |
| """Gets a matching string, or saves the new string. Returns a String ID.""" |
| string_id = self.stringMap.get(string) |
| if string_id is not None: |
| return string_id |
| string_id = len(self.stringTable) |
| self.stringTable.append(string) |
| self.stringMap[string] = string_id |
| return string_id |
| |
| def _intern_frame(self, frame: StackFrame) -> int: |
| """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" |
| frame_str = frame.to_gecko_frame_string() |
| frame_id = self.frameMap.get(frame_str) |
| if frame_id is not None: |
| return frame_id |
| frame_id = len(self.frameTable) |
| self.frameMap[frame_str] = frame_id |
| string_id = self._intern_string(frame_str) |
| |
| |
| self.frameTable.append(GeckoFrame( |
| string_id=string_id, |
| relevantForJS=False, |
| innerWindowID=0, |
| implementation=None, |
| optimizations=None, |
| line=None, |
| column=None, |
| category=frame.category().value, |
| subcategory=0, |
| )) |
| return frame_id |
| |
| def add_sample(self, comm: str, stack: List[StackFrame], time_ms: Milliseconds) -> None: |
| """Add a timestamped stack trace sample to the thread builder. |
| |
| Args: |
| comm: command-line (name) of the thread at this sample |
| stack: sampled stack frames. Root first, leaf last. |
| time_ms: timestamp of sample in milliseconds |
| """ |
| # Unix threads often don't set their name immediately upon creation. |
| # Use the last name |
| if self.comm != comm: |
| self.comm = comm |
| |
| prefix_stack_id = None |
| for frame in stack: |
| frame_id = self._intern_frame(frame) |
| prefix_stack_id = self._intern_stack(frame_id, prefix_stack_id) |
| |
| self.samples.append(GeckoSample( |
| stack_id=prefix_stack_id, |
| time_ms=time_ms, |
| responsiveness=0, |
| complete_stack=is_complete_stack(stack), |
| )) |
| |
| def sort_samples(self) -> None: |
| """ The samples aren't guaranteed to be in order. Sort them by time. """ |
| self.samples.sort(key=lambda s: s.time_ms) |
| |
| def remove_stack_gaps(self, max_remove_gap_length: int, gap_distr: Dict[int, int]) -> None: |
| """ Ideally all callstacks are complete. But some may be broken for different reasons. |
| To create a smooth view in "Stack Chart", remove small gaps of broken callstacks. |
| |
| Args: |
| max_remove_gap_length: the max length of continuous broken-stack samples to remove |
| """ |
| if max_remove_gap_length == 0: |
| return |
| i = 0 |
| remove_flags = [False] * len(self.samples) |
| while i < len(self.samples): |
| if self.samples[i].complete_stack: |
| i += 1 |
| continue |
| n = 1 |
| while (i + n < len(self.samples)) and (not self.samples[i + n].complete_stack): |
| n += 1 |
| gap_distr[n] += 1 |
| if n <= max_remove_gap_length: |
| for j in range(i, i + n): |
| remove_flags[j] = True |
| i += n |
| if True in remove_flags: |
| old_samples = self.samples |
| self.samples = [s for s, remove in zip(old_samples, remove_flags) if not remove] |
| |
| def to_json_dict(self) -> Dict: |
| """Converts this Thread to GeckoThread JSON format.""" |
| |
| # Gecko profile format is row-oriented data as List[List], |
| # And a schema for interpreting each index. |
| # Schema: |
| # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L230 |
| return { |
| "tid": self.tid, |
| "pid": self.pid, |
| "name": self.comm, |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L51 |
| "markers": { |
| "schema": { |
| "name": 0, |
| "startTime": 1, |
| "endTime": 2, |
| "phase": 3, |
| "category": 4, |
| "data": 5, |
| }, |
| "data": [], |
| }, |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 |
| "samples": { |
| "schema": { |
| "stack": 0, |
| "time": 1, |
| "responsiveness": 2, |
| }, |
| "data": [s.to_json() for s in self.samples], |
| }, |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 |
| "frameTable": { |
| "schema": { |
| "location": 0, |
| "relevantForJS": 1, |
| "innerWindowID": 2, |
| "implementation": 3, |
| "optimizations": 4, |
| "line": 5, |
| "column": 6, |
| "category": 7, |
| "subcategory": 8, |
| }, |
| "data": self.frameTable, |
| }, |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 |
| "stackTable": { |
| "schema": { |
| "prefix": 0, |
| "frame": 1, |
| "category": 2, |
| }, |
| "data": self.stackTable, |
| }, |
| "stringTable": self.stringTable, |
| "registerTime": 0, |
| "unregisterTime": None, |
| "processType": "default", |
| } |
| |
| |
| def remove_stack_gaps(max_remove_gap_length: int, thread_map: Dict[int, Thread]) -> None: |
| """ Remove stack gaps for each thread, and print status. """ |
| if max_remove_gap_length == 0: |
| return |
| total_sample_count = 0 |
| remove_sample_count = 0 |
| gap_distr = Counter() |
| for tid in list(thread_map.keys()): |
| thread = thread_map[tid] |
| old_n = len(thread.samples) |
| thread.remove_stack_gaps(max_remove_gap_length, gap_distr) |
| new_n = len(thread.samples) |
| total_sample_count += old_n |
| remove_sample_count += old_n - new_n |
| if new_n == 0: |
| del thread_map[tid] |
| if total_sample_count != 0: |
| logging.info('Remove stack gaps with length <= %d. %d (%.2f%%) samples are removed.', |
| max_remove_gap_length, remove_sample_count, |
| remove_sample_count / total_sample_count * 100 |
| ) |
| logging.debug('Stack gap length distribution among samples (gap_length: count): %s', |
| gap_distr) |
| |
| |
| def _gecko_profile( |
| record_file: str, |
| symfs_dir: Optional[str], |
| kallsyms_file: Optional[str], |
| report_lib_options: ReportLibOptions, |
| max_remove_gap_length: int, |
| percpu_samples: bool) -> GeckoProfile: |
| """convert a simpleperf profile to gecko format""" |
| lib = GetReportLib(record_file) |
| |
| lib.ShowIpForUnknownSymbol() |
| if symfs_dir is not None: |
| lib.SetSymfs(symfs_dir) |
| if kallsyms_file is not None: |
| lib.SetKallsymsFile(kallsyms_file) |
| if percpu_samples: |
| # Grouping samples by cpus doesn't support off cpu samples. |
| if lib.GetSupportedTraceOffCpuModes(): |
| report_lib_options.trace_offcpu = 'on-cpu' |
| lib.SetReportOptions(report_lib_options) |
| |
| arch = lib.GetArch() |
| meta_info = lib.MetaInfo() |
| record_cmd = lib.GetRecordCmd() |
| |
| # Map from tid to Thread |
| thread_map: Dict[int, Thread] = {} |
| # Map from pid to process name |
| process_names: Dict[int, str] = {} |
| |
| while True: |
| sample = lib.GetNextSample() |
| if sample is None: |
| lib.Close() |
| break |
| symbol = lib.GetSymbolOfCurrentSample() |
| callchain = lib.GetCallChainOfCurrentSample() |
| sample_time_ms = sample.time / 1000000 |
| stack : List[StackFrame] = [ |
| StackFrame(symbol.symbol_name, symbol.dso_name), |
| ] |
| |
| for i in range(callchain.nr): |
| entry = callchain.entries[i] |
| stack.append(StackFrame( |
| symbol_name = entry.symbol.symbol_name, |
| dso_name = entry.symbol.dso_name |
| )) |
| # We want root first, leaf last. |
| stack.reverse() |
| |
| if percpu_samples: |
| if sample.tid == sample.pid: |
| process_names[sample.pid] = sample.thread_comm |
| process_name = process_names.get(sample.pid) |
| stack = [ |
| # This is a synthetic stack frame, these aren't real symbols or |
| # DSOs, but they show up nicely in the UI. |
| StackFrame( |
| symbol_name = '%s tid %d' % (sample.thread_comm, sample.tid), |
| dso_name = '%s pid %d' % (process_name, sample.pid), |
| ) |
| ] + stack |
| thread = thread_map.get(sample.cpu) |
| if thread is None: |
| thread = Thread(comm=f'Cpu {sample.cpu}', pid=sample.cpu, tid=sample.cpu) |
| thread_map[sample.cpu] = thread |
| thread.add_sample( |
| comm=f'Cpu {sample.cpu}', |
| stack=stack, |
| time_ms=sample_time_ms) |
| else: |
| # add thread sample |
| thread = thread_map.get(sample.tid) |
| if thread is None: |
| thread = Thread(comm=sample.thread_comm, pid=sample.pid, tid=sample.tid) |
| thread_map[sample.tid] = thread |
| thread.add_sample( |
| comm=sample.thread_comm, |
| stack=stack, |
| # We are being a bit fast and loose here with time here. simpleperf |
| # uses CLOCK_MONOTONIC by default, which doesn't use the normal unix |
| # epoch, but rather some arbitrary time. In practice, this doesn't |
| # matter, the Firefox Profiler normalises all the timestamps to begin at |
| # the minimum time. Consider fixing this in future, if needed, by |
| # setting `simpleperf record --clockid realtime`. |
| time_ms=sample_time_ms) |
| |
| for thread in thread_map.values(): |
| thread.sort_samples() |
| |
| remove_stack_gaps(max_remove_gap_length, thread_map) |
| |
| threads = [thread.to_json_dict() for thread in thread_map.values()] |
| |
| profile_timestamp = meta_info.get('timestamp') |
| end_time_ms = (int(profile_timestamp) * 1000) if profile_timestamp else 0 |
| |
| # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 |
| gecko_profile_meta = { |
| "interval": 1, |
| "processType": 0, |
| "product": record_cmd, |
| "device": meta_info.get("product_props"), |
| "platform": meta_info.get("android_build_fingerprint"), |
| "stackwalk": 1, |
| "debug": 0, |
| "gcpoison": 0, |
| "asyncstack": 1, |
| # The profile timestamp is actually the end time, not the start time. |
| # This is close enough for our purposes; I mostly just want to know which |
| # day the profile was taken! Consider fixing this in future, if needed, |
| # by setting `simpleperf record --clockid realtime` and taking the minimum |
| # sample time. |
| "startTime": end_time_ms, |
| "shutdownTime": None, |
| "version": 24, |
| "presymbolicated": True, |
| "categories": Category.to_json(), |
| "markerSchema": [], |
| "abi": arch, |
| "oscpu": meta_info.get("android_build_fingerprint"), |
| "appBuildID": meta_info.get("app_versioncode"), |
| } |
| |
| # Schema: |
| # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L377 |
| # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md |
| return { |
| "meta": gecko_profile_meta, |
| "libs": [], |
| "threads": threads, |
| "processes": [], |
| "pausedRanges": [], |
| } |
| |
| |
| def main() -> None: |
| parser = BaseArgumentParser(description=__doc__) |
| parser.add_argument('--symfs', |
| help='Set the path to find binaries with symbols and debug info.') |
| parser.add_argument('--kallsyms', help='Set the path to find kernel symbols.') |
| parser.add_argument('-i', '--record_file', nargs='?', default='perf.data', |
| help='Default is perf.data.') |
| parser.add_argument('--remove-gaps', metavar='MAX_GAP_LENGTH', dest='max_remove_gap_length', |
| type=int, default=3, help=""" |
| Ideally all callstacks are complete. But some may be broken for different |
| reasons. To create a smooth view in "Stack Chart", remove small gaps of |
| broken callstacks. MAX_GAP_LENGTH is the max length of continuous |
| broken-stack samples we want to remove. |
| """ |
| ) |
| parser.add_argument( |
| '--percpu-samples', action='store_true', |
| help='show samples based on cpus instead of threads') |
| parser.add_report_lib_options() |
| args = parser.parse_args() |
| profile = _gecko_profile( |
| record_file=args.record_file, |
| symfs_dir=args.symfs, |
| kallsyms_file=args.kallsyms, |
| report_lib_options=args.report_lib_options, |
| max_remove_gap_length=args.max_remove_gap_length, |
| percpu_samples=args.percpu_samples, |
| ) |
| |
| json.dump(profile, sys.stdout, sort_keys=True) |
| |
| |
| if __name__ == '__main__': |
| main() |
