diff options
author | Yabin Cui <yabinc@google.com> | 2022-01-14 00:01:16 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2022-01-14 00:01:16 +0000 |
commit | 6934c6e767c0179af456419cc9022358bff6a279 (patch) | |
tree | 4a0f085560adcd97e834fa2ac5881f71adc4154d | |
parent | 40eaa5b7e509b1144e9e7f5be2dfb2f50c0dc210 (diff) | |
parent | e4e22ed331d685268e6f1c96b237ed3911d68643 (diff) | |
download | extras-6934c6e767c0179af456419cc9022358bff6a279.tar.gz |
Merge changes I74c20512,I65f755a5,I71fc59c7
* changes:
simpleperf: add sample filter options in report scripts.
simpleperf: adjust format of gecko_profile_generator.py.
simpleperf: expose SetSampleFilter in python report lib.
-rwxr-xr-x | simpleperf/scripts/annotate.py | 38 | ||||
-rwxr-xr-x | simpleperf/scripts/gecko_profile_generator.py | 604 | ||||
-rwxr-xr-x | simpleperf/scripts/inferno/inferno.py | 3 | ||||
-rwxr-xr-x | simpleperf/scripts/pprof_proto_generator.py | 33 | ||||
-rwxr-xr-x | simpleperf/scripts/report_html.py | 12 | ||||
-rwxr-xr-x | simpleperf/scripts/report_sample.py | 9 | ||||
-rw-r--r-- | simpleperf/scripts/simpleperf_report_lib.py | 24 | ||||
-rw-r--r-- | simpleperf/scripts/simpleperf_utils.py | 77 | ||||
-rwxr-xr-x | simpleperf/scripts/stackcollapse.py | 21 | ||||
-rw-r--r-- | simpleperf/scripts/test/annotate_test.py | 22 | ||||
-rw-r--r-- | simpleperf/scripts/test/gecko_profile_generator_test.py | 64 | ||||
-rw-r--r-- | simpleperf/scripts/test/inferno_test.py | 36 | ||||
-rw-r--r-- | simpleperf/scripts/test/pprof_proto_generator_test.py | 36 | ||||
-rw-r--r-- | simpleperf/scripts/test/report_html_test.py | 37 | ||||
-rw-r--r-- | simpleperf/scripts/test/report_lib_test.py | 36 | ||||
-rw-r--r-- | simpleperf/scripts/test/report_sample_test.py | 37 | ||||
-rw-r--r-- | simpleperf/scripts/test/stackcollapse_test.py | 35 |
17 files changed, 745 insertions, 379 deletions
diff --git a/simpleperf/scripts/annotate.py b/simpleperf/scripts/annotate.py index 92978a4a..16af623e 100755 --- a/simpleperf/scripts/annotate.py +++ b/simpleperf/scripts/annotate.py @@ -146,8 +146,7 @@ class SourceFileAnnotator(object): def __init__(self, config): # check config variables - config_names = ['perf_data_list', 'source_dirs', 'comm_filters', - 'pid_filters', 'tid_filters', 'dso_filters', 'ndk_path'] + config_names = ['perf_data_list', 'source_dirs', 'comm_filters', 'dso_filters', 'ndk_path'] for name in config_names: if name not in config: log_exit('config [%s] is missing' % name) @@ -163,14 +162,6 @@ class SourceFileAnnotator(object): self.symfs_dir = symfs_dir self.kallsyms = kallsyms self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None - if config.get('pid_filters'): - self.pid_filter = {int(x) for x in config['pid_filters']} - else: - self.pid_filter = None - if config.get('tid_filters'): - self.tid_filter = {int(x) for x in config['tid_filters']} - else: - self.tid_filter = None self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None config['annotate_dest_dir'] = 'annotated_files' @@ -202,6 +193,8 @@ class SourceFileAnnotator(object): lib.SetSymfs(self.symfs_dir) if self.kallsyms: lib.SetKallsymsFile(self.kallsyms) + if self.config.get('sample_filter'): + lib.SetSampleFilter(self.config.get('sample_filter')) while True: sample = lib.GetNextSample() if sample is None: @@ -227,12 +220,6 @@ class SourceFileAnnotator(object): if self.comm_filter: if sample.thread_comm not in self.comm_filter: return False - if self.pid_filter: - if sample.pid not in self.pid_filter: - return False - if self.tid_filter: - if sample.tid not in self.tid_filter: - return False return True def _filter_symbol(self, symbol): @@ -254,6 +241,8 @@ class SourceFileAnnotator(object): lib.SetSymfs(self.symfs_dir) if self.kallsyms: lib.SetKallsymsFile(self.kallsyms) + if self.config.get('sample_filter'): + lib.SetSampleFilter(self.config.get('sample_filter')) while True: sample = lib.GetNextSample() if sample is None: @@ -483,18 +472,16 @@ def main(): The paths of profiling data. Default is perf.data.""") parser.add_argument('-s', '--source_dirs', type=extant_dir, nargs='+', action='append', help=""" Directories to find source files.""") - parser.add_argument('--comm', nargs='+', action='append', help=""" - Use samples only in threads with selected names.""") - parser.add_argument('--pid', nargs='+', action='append', help=""" - Use samples only in processes with selected process ids.""") - parser.add_argument('--tid', nargs='+', action='append', help=""" - Use samples only in threads with selected thread ids.""") - parser.add_argument('--dso', nargs='+', action='append', help=""" - Use samples only in selected binaries.""") parser.add_argument('--ndk_path', type=extant_dir, help='Set the path of a ndk release.') parser.add_argument('--raw-period', action='store_true', help='show raw period instead of percentage') parser.add_argument('--summary-width', type=int, default=80, help='max width of summary file') + sample_filter_group = parser.add_argument_group('Sample filter options') + parser.add_sample_filter_options(sample_filter_group) + sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" + Use samples only in threads with selected names.""") + sample_filter_group.add_argument('--dso', nargs='+', action='append', help=""" + Use samples only in selected binaries.""") args = parser.parse_args() config = {} @@ -503,12 +490,11 @@ def main(): config['perf_data_list'].append('perf.data') config['source_dirs'] = flatten_arg_list(args.source_dirs) config['comm_filters'] = flatten_arg_list(args.comm) - config['pid_filters'] = flatten_arg_list(args.pid) - config['tid_filters'] = flatten_arg_list(args.tid) config['dso_filters'] = flatten_arg_list(args.dso) config['ndk_path'] = args.ndk_path config['raw_period'] = args.raw_period config['summary_width'] = args.summary_width + config['sample_filter'] = args.sample_filter annotator = SourceFileAnnotator(config) annotator.annotate() diff --git a/simpleperf/scripts/gecko_profile_generator.py b/simpleperf/scripts/gecko_profile_generator.py index 886d89c5..e7108ee6 100755 --- a/simpleperf/scripts/gecko_profile_generator.py +++ b/simpleperf/scripts/gecko_profile_generator.py @@ -44,29 +44,29 @@ GeckoProfile = Dict # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 class Frame(NamedTuple): - string_id: StringID - relevantForJS: bool - innerWindowID: int - implementation: None - optimizations: None - line: None - column: None - category: CategoryID - subcategory: int + string_id: StringID + relevantForJS: bool + innerWindowID: int + implementation: None + optimizations: None + line: None + column: None + category: CategoryID + subcategory: int # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 class Stack(NamedTuple): - prefix_id: Optional[StackID] - frame_id: FrameID - category_id: CategoryID + prefix_id: Optional[StackID] + frame_id: FrameID + category_id: CategoryID # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 class Sample(NamedTuple): - stack_id: Optional[StackID] - time_ms: Milliseconds - responsiveness: int + stack_id: Optional[StackID] + time_ms: Milliseconds + responsiveness: int # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/profile.js#L425 @@ -121,302 +121,308 @@ CATEGORIES = [ @dataclass class Thread: - """A builder for a profile of a single thread. - - Attributes: - comm: Thread command-line (name). - pid: process ID of containing process. - tid: thread ID. - samples: Timeline of profile samples. - frameTable: interned stack frame ID -> stack frame. - stringTable: interned string ID -> string. - stringMap: interned string -> string ID. - stackTable: interned stack ID -> stack. - stackMap: (stack prefix ID, leaf stack frame ID) -> interned Stack ID. - frameMap: Stack Frame string -> interned Frame ID. - """ - comm: str - pid: int - tid: int - samples: List[Sample] = field(default_factory=list) - frameTable: List[Frame] = field(default_factory=list) - stringTable: List[str] = field(default_factory=list) - # TODO: this is redundant with frameTable, could we remove this? - stringMap: Dict[str, int] = field(default_factory=dict) - stackTable: List[Stack] = field(default_factory=list) - stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) - frameMap: Dict[str, int] = field(default_factory=dict) - - def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: - """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" - key = (prefix_id, frame_id) - stack_id = self.stackMap.get(key) - if stack_id is not None: - return stack_id - stack_id = len(self.stackTable) - self.stackTable.append(Stack(prefix_id=prefix_id, - frame_id=frame_id, - category_id=0)) - self.stackMap[key] = stack_id - return stack_id - - def _intern_string(self, string: str) -> int: - """Gets a matching string, or saves the new string. Returns a String ID.""" - string_id = self.stringMap.get(string) - if string_id is not None: - return string_id - string_id = len(self.stringTable) - self.stringTable.append(string) - self.stringMap[string] = string_id - return string_id - - def _intern_frame(self, frame_str: str) -> int: - """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" - frame_id = self.frameMap.get(frame_str) - if frame_id is not None: - return frame_id - frame_id = len(self.frameTable) - self.frameMap[frame_str] = frame_id - string_id = self._intern_string(frame_str) - - category = 0 - # Heuristic: kernel code contains "kallsyms" as the library name. - if "kallsyms" in frame_str or ".ko" in frame_str: - category = 1 - elif ".so" in frame_str: - category = 2 - elif ".vdex" in frame_str: - category = 3 - elif ".oat" in frame_str: - category = 4 - - self.frameTable.append(Frame( - string_id=string_id, - relevantForJS=False, - innerWindowID=0, - implementation=None, - optimizations=None, - line=None, - column=None, - category=category, - subcategory=0, - )) - return frame_id - - def _add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None: - """Add a timestamped stack trace sample to the thread builder. - - Args: - comm: command-line (name) of the thread at this sample - stack: sampled stack frames. Root first, leaf last. - time_ms: timestamp of sample in milliseconds + """A builder for a profile of a single thread. + + Attributes: + comm: Thread command-line (name). + pid: process ID of containing process. + tid: thread ID. + samples: Timeline of profile samples. + frameTable: interned stack frame ID -> stack frame. + stringTable: interned string ID -> string. + stringMap: interned string -> string ID. + stackTable: interned stack ID -> stack. + stackMap: (stack prefix ID, leaf stack frame ID) -> interned Stack ID. + frameMap: Stack Frame string -> interned Frame ID. """ - # Unix threads often don't set their name immediately upon creation. - # Use the last name - if self.comm != comm: - self.comm = comm - - prefix_stack_id = None - for frame in stack: - frame_id = self._intern_frame(frame) - prefix_stack_id = self._intern_stack(frame_id, prefix_stack_id) - - self.samples.append(Sample(stack_id=prefix_stack_id, - time_ms=time_ms, - responsiveness=0)) - - def _to_json_dict(self) -> Dict: - """Converts this Thread to GeckoThread JSON format.""" - # The samples aren't guaranteed to be in order. Sort them by time. - self.samples.sort(key=lambda s: s.time_ms) - - # Gecko profile format is row-oriented data as List[List], - # And a schema for interpreting each index. - # Schema: - # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md - # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L230 - return { - "tid": self.tid, - "pid": self.pid, - "name": self.comm, - # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L51 - "markers": { - "schema": { - "name": 0, - "startTime": 1, - "endTime": 2, - "phase": 3, - "category": 4, - "data": 5, + comm: str + pid: int + tid: int + samples: List[Sample] = field(default_factory=list) + frameTable: List[Frame] = field(default_factory=list) + stringTable: List[str] = field(default_factory=list) + # TODO: this is redundant with frameTable, could we remove this? + stringMap: Dict[str, int] = field(default_factory=dict) + stackTable: List[Stack] = field(default_factory=list) + stackMap: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) + frameMap: Dict[str, int] = field(default_factory=dict) + + def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: + """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" + key = (prefix_id, frame_id) + stack_id = self.stackMap.get(key) + if stack_id is not None: + return stack_id + stack_id = len(self.stackTable) + self.stackTable.append(Stack(prefix_id=prefix_id, + frame_id=frame_id, + category_id=0)) + self.stackMap[key] = stack_id + return stack_id + + def _intern_string(self, string: str) -> int: + """Gets a matching string, or saves the new string. Returns a String ID.""" + string_id = self.stringMap.get(string) + if string_id is not None: + return string_id + string_id = len(self.stringTable) + self.stringTable.append(string) + self.stringMap[string] = string_id + return string_id + + def _intern_frame(self, frame_str: str) -> int: + """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" + frame_id = self.frameMap.get(frame_str) + if frame_id is not None: + return frame_id + frame_id = len(self.frameTable) + self.frameMap[frame_str] = frame_id + string_id = self._intern_string(frame_str) + + category = 0 + # Heuristic: kernel code contains "kallsyms" as the library name. + if "kallsyms" in frame_str or ".ko" in frame_str: + category = 1 + elif ".so" in frame_str: + category = 2 + elif ".vdex" in frame_str: + category = 3 + elif ".oat" in frame_str: + category = 4 + + self.frameTable.append(Frame( + string_id=string_id, + relevantForJS=False, + innerWindowID=0, + implementation=None, + optimizations=None, + line=None, + column=None, + category=category, + subcategory=0, + )) + return frame_id + + def _add_sample(self, comm: str, stack: List[str], time_ms: Milliseconds) -> None: + """Add a timestamped stack trace sample to the thread builder. + + Args: + comm: command-line (name) of the thread at this sample + stack: sampled stack frames. Root first, leaf last. + time_ms: timestamp of sample in milliseconds + """ + # Unix threads often don't set their name immediately upon creation. + # Use the last name + if self.comm != comm: + self.comm = comm + + prefix_stack_id = None + for frame in stack: + frame_id = self._intern_frame(frame) + prefix_stack_id = self._intern_stack(frame_id, prefix_stack_id) + + self.samples.append(Sample(stack_id=prefix_stack_id, + time_ms=time_ms, + responsiveness=0)) + + def _to_json_dict(self) -> Dict: + """Converts this Thread to GeckoThread JSON format.""" + # The samples aren't guaranteed to be in order. Sort them by time. + self.samples.sort(key=lambda s: s.time_ms) + + # Gecko profile format is row-oriented data as List[List], + # And a schema for interpreting each index. + # Schema: + # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L230 + return { + "tid": self.tid, + "pid": self.pid, + "name": self.comm, + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L51 + "markers": { + "schema": { + "name": 0, + "startTime": 1, + "endTime": 2, + "phase": 3, + "category": 4, + "data": 5, + }, + "data": [], }, - "data": [], - }, - # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 - "samples": { - "schema": { - "stack": 0, - "time": 1, - "responsiveness": 2, + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 + "samples": { + "schema": { + "stack": 0, + "time": 1, + "responsiveness": 2, + }, + "data": self.samples }, - "data": self.samples - }, - # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 - "frameTable": { - "schema": { - "location": 0, - "relevantForJS": 1, - "innerWindowID": 2, - "implementation": 3, - "optimizations": 4, - "line": 5, - "column": 6, - "category": 7, - "subcategory": 8, + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 + "frameTable": { + "schema": { + "location": 0, + "relevantForJS": 1, + "innerWindowID": 2, + "implementation": 3, + "optimizations": 4, + "line": 5, + "column": 6, + "category": 7, + "subcategory": 8, + }, + "data": self.frameTable, }, - "data": self.frameTable, - }, - # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 - "stackTable": { - "schema": { - "prefix": 0, - "frame": 1, - "category": 2, + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 + "stackTable": { + "schema": { + "prefix": 0, + "frame": 1, + "category": 2, + }, + "data": self.stackTable, }, - "data": self.stackTable, - }, - "stringTable": self.stringTable, - "registerTime": 0, - "unregisterTime": None, - "processType": "default", - } + "stringTable": self.stringTable, + "registerTime": 0, + "unregisterTime": None, + "processType": "default", + } def _gecko_profile( - record_file: str, - symfs_dir: Optional[str], - kallsyms_file: Optional[str], - proguard_mapping_file: List[str], - comm_filter: Set[str]) -> GeckoProfile: - """convert a simpleperf profile to gecko format""" - lib = ReportLib() - - lib.ShowIpForUnknownSymbol() - for file_path in proguard_mapping_file: - lib.AddProguardMappingFile(file_path) - if symfs_dir is not None: - lib.SetSymfs(symfs_dir) - lib.SetRecordFile(record_file) - if kallsyms_file is not None: - lib.SetKallsymsFile(kallsyms_file) - - arch = lib.GetArch() - meta_info = lib.MetaInfo() - record_cmd = lib.GetRecordCmd() - - # Map from tid to Thread - threadMap: Dict[int, Thread] = {} - - while True: - sample = lib.GetNextSample() - if sample is None: - lib.Close() - break - if comm_filter: - if sample.thread_comm not in comm_filter: - continue - event = lib.GetEventOfCurrentSample() - symbol = lib.GetSymbolOfCurrentSample() - callchain = lib.GetCallChainOfCurrentSample() - sample_time_ms = sample.time / 1000000 - - stack = ['%s (in %s)' % (symbol.symbol_name, symbol.dso_name)] - for i in range(callchain.nr): - entry = callchain.entries[i] - stack.append('%s (in %s)' % (entry.symbol.symbol_name, entry.symbol.dso_name)) - # We want root first, leaf last. - stack.reverse() - - # add thread sample - thread = threadMap.get(sample.tid) - if thread is None: - thread = Thread(comm=sample.thread_comm, pid=sample.pid, tid=sample.tid) - threadMap[sample.tid] = thread - thread._add_sample( - comm=sample.thread_comm, - stack=stack, - # We are being a bit fast and loose here with time here. simpleperf - # uses CLOCK_MONOTONIC by default, which doesn't use the normal unix - # epoch, but rather some arbitrary time. In practice, this doesn't - # matter, the Firefox Profiler normalises all the timestamps to begin at - # the minimum time. Consider fixing this in future, if needed, by - # setting `simpleperf record --clockid realtime`. - time_ms=sample_time_ms) - - threads = [thread._to_json_dict() for thread in threadMap.values()] - - profile_timestamp = meta_info.get('timestamp') - end_time_ms = (int(profile_timestamp) * 1000) if profile_timestamp else 0 - - # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 - gecko_profile_meta = { - "interval": 1, - "processType": 0, - "product": record_cmd, - "device": meta_info.get("product_props"), - "platform": meta_info.get("android_build_fingerprint"), - "stackwalk": 1, - "debug": 0, - "gcpoison": 0, - "asyncstack": 1, - # The profile timestamp is actually the end time, not the start time. - # This is close enough for our purposes; I mostly just want to know which - # day the profile was taken! Consider fixing this in future, if needed, - # by setting `simpleperf record --clockid realtime` and taking the minimum - # sample time. - "startTime": end_time_ms, - "shutdownTime": None, - "version": 24, - "presymbolicated": True, - "categories": CATEGORIES, - "markerSchema": [], - "abi": arch, - "oscpu": meta_info.get("android_build_fingerprint"), - } - - # Schema: - # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L377 - # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md - return { - "meta": gecko_profile_meta, - "libs": [], - "threads": threads, - "processes": [], - "pausedRanges": [], - } + record_file: str, + symfs_dir: Optional[str], + kallsyms_file: Optional[str], + proguard_mapping_file: List[str], + comm_filter: Set[str], + sample_filter: Optional[str]) -> GeckoProfile: + """convert a simpleperf profile to gecko format""" + lib = ReportLib() + + lib.ShowIpForUnknownSymbol() + for file_path in proguard_mapping_file: + lib.AddProguardMappingFile(file_path) + if symfs_dir is not None: + lib.SetSymfs(symfs_dir) + lib.SetRecordFile(record_file) + if kallsyms_file is not None: + lib.SetKallsymsFile(kallsyms_file) + if sample_filter: + lib.SetSampleFilter(sample_filter) + + arch = lib.GetArch() + meta_info = lib.MetaInfo() + record_cmd = lib.GetRecordCmd() + + # Map from tid to Thread + threadMap: Dict[int, Thread] = {} + + while True: + sample = lib.GetNextSample() + if sample is None: + lib.Close() + break + if comm_filter: + if sample.thread_comm not in comm_filter: + continue + event = lib.GetEventOfCurrentSample() + symbol = lib.GetSymbolOfCurrentSample() + callchain = lib.GetCallChainOfCurrentSample() + sample_time_ms = sample.time / 1000000 + + stack = ['%s (in %s)' % (symbol.symbol_name, symbol.dso_name)] + for i in range(callchain.nr): + entry = callchain.entries[i] + stack.append('%s (in %s)' % (entry.symbol.symbol_name, entry.symbol.dso_name)) + # We want root first, leaf last. + stack.reverse() + + # add thread sample + thread = threadMap.get(sample.tid) + if thread is None: + thread = Thread(comm=sample.thread_comm, pid=sample.pid, tid=sample.tid) + threadMap[sample.tid] = thread + thread._add_sample( + comm=sample.thread_comm, + stack=stack, + # We are being a bit fast and loose here with time here. simpleperf + # uses CLOCK_MONOTONIC by default, which doesn't use the normal unix + # epoch, but rather some arbitrary time. In practice, this doesn't + # matter, the Firefox Profiler normalises all the timestamps to begin at + # the minimum time. Consider fixing this in future, if needed, by + # setting `simpleperf record --clockid realtime`. + time_ms=sample_time_ms) + + threads = [thread._to_json_dict() for thread in threadMap.values()] + + profile_timestamp = meta_info.get('timestamp') + end_time_ms = (int(profile_timestamp) * 1000) if profile_timestamp else 0 + + # Schema: https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L305 + gecko_profile_meta = { + "interval": 1, + "processType": 0, + "product": record_cmd, + "device": meta_info.get("product_props"), + "platform": meta_info.get("android_build_fingerprint"), + "stackwalk": 1, + "debug": 0, + "gcpoison": 0, + "asyncstack": 1, + # The profile timestamp is actually the end time, not the start time. + # This is close enough for our purposes; I mostly just want to know which + # day the profile was taken! Consider fixing this in future, if needed, + # by setting `simpleperf record --clockid realtime` and taking the minimum + # sample time. + "startTime": end_time_ms, + "shutdownTime": None, + "version": 24, + "presymbolicated": True, + "categories": CATEGORIES, + "markerSchema": [], + "abi": arch, + "oscpu": meta_info.get("android_build_fingerprint"), + } + + # Schema: + # https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L377 + # https://github.com/firefox-devtools/profiler/blob/main/docs-developer/gecko-profile-format.md + return { + "meta": gecko_profile_meta, + "libs": [], + "threads": threads, + "processes": [], + "pausedRanges": [], + } def main() -> None: - parser = BaseArgumentParser(description=__doc__) - parser.add_argument('--symfs', - help='Set the path to find binaries with symbols and debug info.') - parser.add_argument('--kallsyms', help='Set the path to find kernel symbols.') - parser.add_argument('-i', '--record_file', nargs='?', default='perf.data', - help='Default is perf.data.') - parser.add_argument( - '--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols', - default = []) - parser.add_argument('--comm', nargs='+', action='append', help=""" + parser = BaseArgumentParser(description=__doc__) + parser.add_argument('--symfs', + help='Set the path to find binaries with symbols and debug info.') + parser.add_argument('--kallsyms', help='Set the path to find kernel symbols.') + parser.add_argument('-i', '--record_file', nargs='?', default='perf.data', + help='Default is perf.data.') + parser.add_argument( + '--proguard-mapping-file', nargs='+', + help='Add proguard mapping file to de-obfuscate symbols', + default=[]) + sample_filter_group = parser.add_argument_group('Sample filter options') + parser.add_sample_filter_options(sample_filter_group) + sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" Use samples only in threads with selected names.""") - args = parser.parse_args() - profile = _gecko_profile( - record_file=args.record_file, - symfs_dir=args.symfs, - kallsyms_file=args.kallsyms, - proguard_mapping_file=args.proguard_mapping_file, - comm_filter=set(flatten_arg_list(args.comm))) - - json.dump(profile, sys.stdout, sort_keys=True) + args = parser.parse_args() + profile = _gecko_profile( + record_file=args.record_file, + symfs_dir=args.symfs, + kallsyms_file=args.kallsyms, + proguard_mapping_file=args.proguard_mapping_file, + comm_filter=set(flatten_arg_list(args.comm)), + sample_filter=args.sample_filter) + + json.dump(profile, sys.stdout, sort_keys=True) if __name__ == '__main__': diff --git a/simpleperf/scripts/inferno/inferno.py b/simpleperf/scripts/inferno/inferno.py index ecb56b21..512b1dc9 100755 --- a/simpleperf/scripts/inferno/inferno.py +++ b/simpleperf/scripts/inferno/inferno.py @@ -117,6 +117,8 @@ def parse_samples(process, args, sample_filter_fn): lib.AddProguardMappingFile(file_path) if args.trace_offcpu: lib.SetTraceOffCpuMode(args.trace_offcpu) + if args.sample_filter: + lib.SetSampleFilter(args.sample_filter) process.cmd = lib.GetRecordCmd() product_props = lib.MetaInfo().get("product_props") if product_props: @@ -316,6 +318,7 @@ def main(): report_group.add_argument('--proguard-mapping-file', nargs='+', help='Add proguard mapping file to de-obfuscate symbols') parser.add_trace_offcpu_option(report_group) + parser.add_sample_filter_options(report_group, False) debug_group = parser.add_argument_group('Debug options') debug_group.add_argument('--disable_adb_root', action='store_true', help="""Force adb to run diff --git a/simpleperf/scripts/pprof_proto_generator.py b/simpleperf/scripts/pprof_proto_generator.py index b45eaf4d..2e807d95 100755 --- a/simpleperf/scripts/pprof_proto_generator.py +++ b/simpleperf/scripts/pprof_proto_generator.py @@ -271,14 +271,6 @@ class PprofProfileGenerator(object): if not os.path.isdir(config['binary_cache_dir']): config['binary_cache_dir'] = None self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None - if config.get('pid_filters'): - self.pid_filter = {int(x) for x in config['pid_filters']} - else: - self.pid_filter = None - if config.get('tid_filters'): - self.tid_filter = {int(x) for x in config['tid_filters']} - else: - self.tid_filter = None self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None self.max_chain_length = config['max_chain_length'] self.profile = profile_pb2.Profile() @@ -313,6 +305,8 @@ class PprofProfileGenerator(object): self.lib.ShowArtFrames() for file_path in self.config['proguard_mapping_file'] or []: self.lib.AddProguardMappingFile(file_path) + if self.config.get('sample_filter'): + self.lib.SetSampleFilter(self.config['sample_filter']) comments = [ "Simpleperf Record Command:\n" + self.lib.GetRecordCmd(), @@ -390,12 +384,6 @@ class PprofProfileGenerator(object): if self.comm_filter: if sample.thread_comm not in self.comm_filter: return False - if self.pid_filter: - if sample.pid not in self.pid_filter: - return False - if self.tid_filter: - if sample.tid not in self.tid_filter: - return False return True def _filter_symbol(self, symbol): @@ -642,14 +630,6 @@ def main(): Set profiling data file to report. Default is perf.data""") parser.add_argument('-o', '--output_file', default='pprof.profile', help=""" The path of generated pprof profile data.""") - parser.add_argument('--comm', nargs='+', action='append', help=""" - Use samples only in threads with selected names.""") - parser.add_argument('--pid', nargs='+', action='append', help=""" - Use samples only in processes with selected process ids.""") - parser.add_argument('--tid', nargs='+', action='append', help=""" - Use samples only in threads with selected thread ids.""") - parser.add_argument('--dso', nargs='+', action='append', help=""" - Use samples only in selected binaries.""") parser.add_argument('--max_chain_length', type=int, default=1000000000, help=""" Maximum depth of samples to be converted.""") # Large value as infinity standin. parser.add_argument('--ndk_path', type=extant_dir, help='Set the path of a ndk release.') @@ -661,6 +641,12 @@ def main(): parser.add_argument( '-j', '--jobs', type=int, default=os.cpu_count(), help='Use multithreading to speed up source code annotation.') + sample_filter_group = parser.add_argument_group('Sample filter options') + parser.add_sample_filter_options(sample_filter_group) + sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" + Use samples only in threads with selected names.""") + sample_filter_group.add_argument('--dso', nargs='+', action='append', help=""" + Use samples only in selected binaries.""") args = parser.parse_args() if args.show: @@ -673,13 +659,12 @@ def main(): config = {} config['output_file'] = args.output_file config['comm_filters'] = flatten_arg_list(args.comm) - config['pid_filters'] = flatten_arg_list(args.pid) - config['tid_filters'] = flatten_arg_list(args.tid) config['dso_filters'] = flatten_arg_list(args.dso) config['ndk_path'] = args.ndk_path config['show_art_frames'] = args.show_art_frames config['max_chain_length'] = args.max_chain_length config['proguard_mapping_file'] = args.proguard_mapping_file + config['sample_filter'] = args.sample_filter generator = PprofProfileGenerator(config) for record_file in args.record_file: generator.load_record_file(record_file) diff --git a/simpleperf/scripts/report_html.py b/simpleperf/scripts/report_html.py index b8e6ad9a..9de2f98b 100755 --- a/simpleperf/scripts/report_html.py +++ b/simpleperf/scripts/report_html.py @@ -623,7 +623,8 @@ class RecordData(object): self.gen_addr_hit_map_in_record_info = False self.binary_finder = BinaryFinder(binary_cache_path, ReadElf(ndk_path)) - def load_record_file(self, record_file: str, show_art_frames: bool): + def load_record_file( + self, record_file: str, show_art_frames: bool, sample_filter: Optional[str]): lib = ReportLib() lib.SetRecordFile(record_file) # If not showing ip for unknown symbols, the percent of the unknown symbol may be @@ -637,6 +638,8 @@ class RecordData(object): lib.AddProguardMappingFile(file_path) if self.trace_offcpu: lib.SetTraceOffCpuMode(self.trace_offcpu) + if sample_filter: + lib.SetSampleFilter(sample_filter) self.meta_info = lib.MetaInfo() self.cmdline = lib.GetRecordCmd() self.arch = lib.GetArch() @@ -993,6 +996,7 @@ def get_args() -> argparse.Namespace: '--proguard-mapping-file', nargs='+', help='Add proguard mapping file to de-obfuscate symbols') parser.add_trace_offcpu_option() + parser.add_sample_filter_options() return parser.parse_args() @@ -1018,10 +1022,10 @@ def main(): log_exit('Invalid --jobs option.') # 2. Produce record data. - record_data = RecordData(binary_cache_path, ndk_path, - build_addr_hit_map, args.proguard_mapping_file, args.trace_offcpu) + record_data = RecordData(binary_cache_path, ndk_path, build_addr_hit_map, + args.proguard_mapping_file, args.trace_offcpu) for record_file in args.record_file: - record_data.load_record_file(record_file, args.show_art_frames) + record_data.load_record_file(record_file, args.show_art_frames, args.sample_filter) if args.aggregate_by_thread_name: record_data.aggregate_by_thread_name() record_data.limit_percents(args.min_func_percent, args.min_callchain_percent) diff --git a/simpleperf/scripts/report_sample.py b/simpleperf/scripts/report_sample.py index bc56b453..7388442a 100755 --- a/simpleperf/scripts/report_sample.py +++ b/simpleperf/scripts/report_sample.py @@ -31,7 +31,8 @@ def report_sample( proguard_mapping_file: List[str], header: bool, comm_filter: Set[str], - trace_offcpu: Optional[str]): + trace_offcpu: Optional[str], + sample_filter: Optional[str]): """ read record_file, and print each sample""" lib = ReportLib() @@ -46,6 +47,8 @@ def report_sample( lib.SetKallsymsFile(kallsyms_file) if trace_offcpu: lib.SetTraceOffCpuMode(trace_offcpu) + if sample_filter: + lib.SetSampleFilter(sample_filter) if header: print("# ========") @@ -103,6 +106,7 @@ def main(): parser.add_argument('--comm', nargs='+', action='append', help=""" Use samples only in threads with selected names.""") parser.add_trace_offcpu_option() + parser.add_sample_filter_options() args = parser.parse_args() report_sample( record_file=args.record_file, @@ -112,7 +116,8 @@ def main(): proguard_mapping_file=args.proguard_mapping_file, header=args.header, comm_filter=set(flatten_arg_list(args.comm)), - trace_offcpu=args.trace_offcpu) + trace_offcpu=args.trace_offcpu, + sample_filter=args.sample_filter) if __name__ == '__main__': diff --git a/simpleperf/scripts/simpleperf_report_lib.py b/simpleperf/scripts/simpleperf_report_lib.py index e26a03ff..5c2893c1 100644 --- a/simpleperf/scripts/simpleperf_report_lib.py +++ b/simpleperf/scripts/simpleperf_report_lib.py @@ -261,6 +261,8 @@ class ReportLib(object): self._GetSupportedTraceOffCpuModesFunc.restype = ct.c_char_p self._SetTraceOffCpuModeFunc = self._lib.SetTraceOffCpuMode self._SetTraceOffCpuModeFunc.restype = ct.c_bool + self._SetSampleFilterFunc = self._lib.SetSampleFilter + self._SetSampleFilterFunc.restype = ct.c_bool self._GetNextSampleFunc = self._lib.GetNextSample self._GetNextSampleFunc.restype = ct.POINTER(SampleStruct) self._GetEventOfCurrentSampleFunc = self._lib.GetEventOfCurrentSample @@ -360,6 +362,28 @@ class ReportLib(object): res: bool = self._SetTraceOffCpuModeFunc(self.getInstance(), _char_pt(mode)) _check(res, f'Failed to call SetTraceOffCpuMode({mode})') + def SetSampleFilter(self, filter: str): + """ Set options used to filter samples. Available options are: + --exclude-pid pid1,pid2,... Exclude samples for selected processes. + --exclude-tid tid1,tid2,... Exclude samples for selected threads. + --exclude-process-name process_name_regex Exclude samples for processes with name + containing the regular expression. + --exclude-thread-name thread_name_regex Exclude samples for threads with name + containing the regular expression. + --include-pid pid1,pid2,... Include samples for selected processes. + --include-tid tid1,tid2,... Include samples for selected threads. + --include-process-name process_name_regex Include samples for processes with name + containing the regular expression. + --include-thread-name thread_name_regex Include samples for threads with name + containing the regular expression. + --filter-file <file> Use filter file to filter samples based on timestamps. The + file format is in doc/sampler_filter.md. + + The filter argument should be a concatenation of options. + """ + res: bool = self._SetSampleFilterFunc(self.getInstance(), _char_pt(filter)) + _check(res, f'Failed to call SetSampleFilter({filter})') + def GetNextSample(self) -> Optional[SampleStruct]: """ Return the next sample. If no more samples, return None. """ psample = self._GetNextSampleFunc(self.getInstance()) diff --git a/simpleperf/scripts/simpleperf_utils.py b/simpleperf/scripts/simpleperf_utils.py index a7b4c6f9..755919c0 100644 --- a/simpleperf/scripts/simpleperf_utils.py +++ b/simpleperf/scripts/simpleperf_utils.py @@ -1002,6 +1002,8 @@ class ArgParseFormatter( class BaseArgumentParser(argparse.ArgumentParser): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs, formatter_class=ArgParseFormatter) + self.has_sample_filter_options = False + self.sample_filter_with_pid_shortcut = False def add_trace_offcpu_option(self, subparser: Optional[Any] = None): parser = subparser if subparser else self @@ -1014,12 +1016,87 @@ class BaseArgumentParser(argparse.ArgumentParser): If not set, mixed-on-off-cpu mode is used. """) + def add_sample_filter_options( + self, group: Optional[Any] = None, with_pid_shortcut: bool = True): + if not group: + group = self.add_argument_group('Sample filter options') + group.add_argument('--exclude-pid', metavar='pid', nargs='+', type=int, + help='exclude samples for selected processes') + group.add_argument('--exclude-tid', metavar='tid', nargs='+', type=int, + help='exclude samples for selected threads') + group.add_argument( + '--exclude-process-name', metavar='process_name_regex', nargs='+', + help='exclude samples for processes with name containing the regular expression') + group.add_argument( + '--exclude-thread-name', metavar='thread_name_regex', nargs='+', + help='exclude samples for threads with name containing the regular expression') + + if with_pid_shortcut: + group.add_argument('--pid', metavar='pid', nargs='+', type=int, + help='only include samples for selected processes') + group.add_argument('--tid', metavar='tid', nargs='+', type=int, + help='only include samples for selected threads') + group.add_argument('--include-pid', metavar='pid', nargs='+', type=int, + help='only include samples for selected processes') + group.add_argument('--include-tid', metavar='tid', nargs='+', type=int, + help='only include samples for selected threads') + group.add_argument( + '--include-process-name', metavar='process_name_regex', nargs='+', + help='only include samples for processes with name containing the regular expression') + group.add_argument( + '--include-thread-name', metavar='thread_name_regex', nargs='+', + help='only include samples for threads with name containing the regular expression') + group.add_argument( + '--filter-file', metavar='file', + help='use filter file to filter samples based on timestamps. ' + + 'The file format is in doc/sampler_filter.md.') + self.has_sample_filter_options = True + self.sample_filter_with_pid_shortcut = with_pid_shortcut + + def _build_sample_filter(self, args: argparse.Namespace) -> Optional[str]: + """ Convert sample filter options into a sample filter string, which can be passed to + ReportLib.SetSampleFilter(). + """ + filters = [] + if args.exclude_pid: + filters.append('--exclude-pid ' + ','.join(str(pid) for pid in args.exclude_pid)) + if args.exclude_tid: + filters.append('--exclude-tid ' + ','.join(str(tid) for tid in args.exclude_tid)) + if args.exclude_process_name: + for name in args.exclude_process_name: + filters.append('--exclude-process-name ' + name) + if args.exclude_thread_name: + for name in args.exclude_thread_name: + filters.append('--exclude-thread-name ' + name) + + if args.include_pid: + filters.append('--include-pid ' + ','.join(str(pid) for pid in args.include_pid)) + if args.include_tid: + filters.append('--include-tid ' + ','.join(str(tid) for tid in args.include_tid)) + if self.sample_filter_with_pid_shortcut: + if args.pid: + filters.append('--include-pid ' + ','.join(str(pid) for pid in args.pid)) + if args.tid: + filters.append('--include-tid ' + ','.join(str(pid) for pid in args.tid)) + if args.include_process_name: + for name in args.include_process_name: + filters.append('--include-process-name ' + name) + if args.include_thread_name: + for name in args.include_thread_name: + filters.append('--include-thread-name ' + name) + if args.filter_file: + filters.append('--filter-file ' + args.filter_file) + return ' '.join(filters) + def parse_known_args(self, *args, **kwargs): self.add_argument( '--log', choices=['debug', 'info', 'warning'], default='info', help='set log level') namespace, left_args = super().parse_known_args(*args, **kwargs) + if self.has_sample_filter_options: + setattr(namespace, 'sample_filter', self._build_sample_filter(namespace)) + if not Log.initialized: Log.init(namespace.log) return namespace, left_args diff --git a/simpleperf/scripts/stackcollapse.py b/simpleperf/scripts/stackcollapse.py index 52351bc0..2574843a 100755 --- a/simpleperf/scripts/stackcollapse.py +++ b/simpleperf/scripts/stackcollapse.py @@ -27,7 +27,7 @@ from collections import defaultdict from simpleperf_report_lib import ReportLib from simpleperf_utils import BaseArgumentParser, flatten_arg_list -from typing import DefaultDict, List, Set +from typing import DefaultDict, List, Optional, Set import logging import sys @@ -44,7 +44,8 @@ def collapse_stacks( annotate_kernel: bool, annotate_jit: bool, include_addrs: bool, - comm_filter: Set[str]): + comm_filter: Set[str], + sample_filter: Optional[str]): """read record_file, aggregate per-stack and print totals per-stack""" lib = ReportLib() @@ -58,6 +59,8 @@ def collapse_stacks( lib.SetRecordFile(record_file) if kallsyms_file is not None: lib.SetKallsymsFile(kallsyms_file) + if sample_filter: + lib.SetSampleFilter(sample_filter) stacks: DefaultDict[str, int] = defaultdict(int) event_defaulted = False @@ -79,7 +82,8 @@ def collapse_stacks( elif event.name != event_filter: if event_defaulted and not event_warning_shown: logging.warning( - 'Input has multiple event types. Filtering for the first event type seen: %s' % event_filter) + 'Input has multiple event types. Filtering for the first event type seen: %s' % + event_filter) event_warning_shown = True continue @@ -112,8 +116,6 @@ def main(): parser.add_argument('--kallsyms', help='Set the path to find kernel symbols.') parser.add_argument('-i', '--record_file', nargs='?', default='perf.data', help='Default is perf.data.') - parser.add_argument('--event-filter', nargs='?', default='', - help='Event type filter e.g. "cpu-cycles" or "instructions"') parser.add_argument('--pid', action='store_true', help='Include PID with process names') parser.add_argument('--tid', action='store_true', help='Include TID and PID with process names') parser.add_argument('--kernel', action='store_true', @@ -125,7 +127,11 @@ def main(): '--proguard-mapping-file', nargs='+', help='Add proguard mapping file to de-obfuscate symbols', default=[]) - parser.add_argument('--comm', nargs='+', action='append', help=""" + sample_filter_group = parser.add_argument_group('Sample filter options') + parser.add_sample_filter_options(sample_filter_group, False) + sample_filter_group.add_argument('--event-filter', nargs='?', default='', + help='Event type filter e.g. "cpu-cycles" or "instructions"') + sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" Use samples only in threads with selected names.""") args = parser.parse_args() collapse_stacks( @@ -139,7 +145,8 @@ def main(): annotate_kernel=args.kernel, annotate_jit=args.jit, include_addrs=args.addrs, - comm_filter=set(flatten_arg_list(args.comm))) + comm_filter=set(flatten_arg_list(args.comm)), + sample_filter=args.sample_filter) if __name__ == '__main__': diff --git a/simpleperf/scripts/test/annotate_test.py b/simpleperf/scripts/test/annotate_test.py index 87e9b853..b7e06b47 100644 --- a/simpleperf/scripts/test/annotate_test.py +++ b/simpleperf/scripts/test/annotate_test.py @@ -16,6 +16,7 @@ from pathlib import Path import re +import tempfile from binary_cache_builder import BinaryCacheBuilder from . test_utils import TestBase, TestHelper @@ -50,3 +51,24 @@ class TestAnnotate(TestBase): self.assertEqual(source_file.name, 'two_functions.cpp') check_items = ['/* Total 50.06%, Self 50.06% */ *p = i;'] self.check_strings_in_file(source_file, check_items) + + def test_sample_filters(self): + def get_report(filter: str): + self.run_cmd(['annotate.py', '-i', TestHelper.testdata_path( + 'perf_display_bitmaps.data')] + filter.split()) + + get_report('--exclude-pid 31850') + get_report('--include-pid 31850') + get_report('--pid 31850') + get_report('--exclude-tid 31881') + get_report('--include-tid 31881') + get_report('--tid 31881') + get_report('--exclude-process-name com.example.android.displayingbitmaps') + get_report('--include-process-name com.example.android.displayingbitmaps') + get_report('--exclude-thread-name com.example.android.displayingbitmaps') + get_report('--include-thread-name com.example.android.displayingbitmaps') + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + get_report('--filter-file ' + filter_file.name) diff --git a/simpleperf/scripts/test/gecko_profile_generator_test.py b/simpleperf/scripts/test/gecko_profile_generator_test.py index 8f300767..e0a00b3b 100644 --- a/simpleperf/scripts/test/gecko_profile_generator_test.py +++ b/simpleperf/scripts/test/gecko_profile_generator_test.py @@ -15,21 +15,57 @@ # limitations under the License. import json +import re +import tempfile +from typing import Set from . test_utils import TestBase, TestHelper + class TestGeckoProfileGenerator(TestBase): - def run_generator(self, testdata_file): - testdata_path = TestHelper.testdata_path(testdata_file) - gecko_profile_json = self.run_cmd( - ['gecko_profile_generator.py', '-i', testdata_path], return_output=True) - return json.loads(gecko_profile_json) - - def test_golden(self): - got = self.run_generator('perf_with_interpreter_frames.data') - golden_path = TestHelper.testdata_path('perf_with_interpreter_frames.gecko.json') - with open(golden_path) as f: - want = json.load(f) - self.assertEqual( - json.dumps(got, sort_keys=True, indent=2), - json.dumps(want, sort_keys=True, indent=2)) + def run_generator(self, testdata_file): + testdata_path = TestHelper.testdata_path(testdata_file) + gecko_profile_json = self.run_cmd( + ['gecko_profile_generator.py', '-i', testdata_path], return_output=True) + return json.loads(gecko_profile_json) + + def test_golden(self): + got = self.run_generator('perf_with_interpreter_frames.data') + golden_path = TestHelper.testdata_path('perf_with_interpreter_frames.gecko.json') + with open(golden_path) as f: + want = json.load(f) + self.assertEqual( + json.dumps(got, sort_keys=True, indent=2), + json.dumps(want, sort_keys=True, indent=2)) + + def test_sample_filters(self): + def get_threads_for_filter(filter: str) -> Set[int]: + report = self.run_cmd(['gecko_profile_generator.py', '-i', TestHelper.testdata_path( + 'perf_display_bitmaps.data')] + filter.split(), return_output=True) + pattern = re.compile(r'"tid":\s+(\d+),') + threads = set() + for m in re.finditer(pattern, report): + threads.add(int(m.group(1))) + return threads + + self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--pid 31850')) + self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--tid 31881')) + self.assertNotIn(31881, get_threads_for_filter( + '--exclude-process-name com.example.android.displayingbitmaps')) + self.assertIn(31881, get_threads_for_filter( + '--include-process-name com.example.android.displayingbitmaps')) + self.assertNotIn(31850, get_threads_for_filter( + '--exclude-thread-name com.example.android.displayingbitmaps')) + self.assertIn(31850, get_threads_for_filter( + '--include-thread-name com.example.android.displayingbitmaps')) + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + threads = get_threads_for_filter('--filter-file ' + filter_file.name) + self.assertIn(31881, threads) + self.assertNotIn(31850, threads) diff --git a/simpleperf/scripts/test/inferno_test.py b/simpleperf/scripts/test/inferno_test.py index a31a0fec..00ec8c50 100644 --- a/simpleperf/scripts/test/inferno_test.py +++ b/simpleperf/scripts/test/inferno_test.py @@ -16,7 +16,9 @@ import collections import json -from typing import Any, Dict, List +import re +import tempfile +from typing import Any, Dict, List, Set from . test_utils import INFERNO_SCRIPT, TestBase, TestHelper @@ -45,3 +47,35 @@ class TestInferno(TestBase): report = self.get_report(['--record_file', testdata_file, '-sc', '--trace-offcpu', 'off-cpu']) self.assertIn('Thread 6525 (com.google.samples.apps.sunflower) (42 samples)', report) + + def test_sample_filters(self): + def get_threads_for_filter(filter: str) -> Set[int]: + report = self.get_report( + ['--record_file', TestHelper.testdata_path('perf_display_bitmaps.data'), + '-sc'] + filter.split()) + threads = set() + pattern = re.compile(r'Thread\s+(\d+)\s+') + threads = set() + for m in re.finditer(pattern, report): + threads.add(int(m.group(1))) + return threads + + self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) + self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertNotIn(31881, get_threads_for_filter( + '--exclude-process-name com.example.android.displayingbitmaps')) + self.assertIn(31881, get_threads_for_filter( + '--include-process-name com.example.android.displayingbitmaps')) + self.assertNotIn(31850, get_threads_for_filter( + '--exclude-thread-name com.example.android.displayingbitmaps')) + self.assertIn(31850, get_threads_for_filter( + '--include-thread-name com.example.android.displayingbitmaps')) + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + threads = get_threads_for_filter('--filter-file ' + filter_file.name) + self.assertIn(31881, threads) + self.assertNotIn(31850, threads) diff --git a/simpleperf/scripts/test/pprof_proto_generator_test.py b/simpleperf/scripts/test/pprof_proto_generator_test.py index 6d483451..ddee18ce 100644 --- a/simpleperf/scripts/test/pprof_proto_generator_test.py +++ b/simpleperf/scripts/test/pprof_proto_generator_test.py @@ -16,7 +16,9 @@ from collections import namedtuple import google.protobuf -from typing import List, Optional +import re +import tempfile +from typing import List, Optional, Set from binary_cache_builder import BinaryCacheBuilder from pprof_proto_generator import load_pprof_profile, PprofProfileGenerator @@ -252,3 +254,35 @@ class TestPprofProtoGenerator(TestBase): # path. self.assertIn('testdata/perf_with_interpreter_frames.data', comments) self.assertIn('Architecture:\naarch64', comments) + + def test_sample_filters(self): + def get_threads_for_filter(filter: str) -> Set[int]: + report = self.run_generator(filter.split(), testdata_file='perf_display_bitmaps.data') + threads = set() + pattern = re.compile(r'\s+tid:(\d+)') + threads = set() + for m in re.finditer(pattern, report): + threads.add(int(m.group(1))) + return threads + + self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--pid 31850')) + self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--tid 31881')) + self.assertNotIn(31881, get_threads_for_filter( + '--exclude-process-name com.example.android.displayingbitmaps')) + self.assertIn(31881, get_threads_for_filter( + '--include-process-name com.example.android.displayingbitmaps')) + self.assertNotIn(31850, get_threads_for_filter( + '--exclude-thread-name com.example.android.displayingbitmaps')) + self.assertIn(31850, get_threads_for_filter( + '--include-thread-name com.example.android.displayingbitmaps')) + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + threads = get_threads_for_filter('--filter-file ' + filter_file.name) + self.assertIn(31881, threads) + self.assertNotIn(31850, threads) diff --git a/simpleperf/scripts/test/report_html_test.py b/simpleperf/scripts/test/report_html_test.py index 39543f8c..73dc8624 100644 --- a/simpleperf/scripts/test/report_html_test.py +++ b/simpleperf/scripts/test/report_html_test.py @@ -16,7 +16,8 @@ import collections import json -from typing import Any, Dict, List +import tempfile +from typing import Any, Dict, List, Set from binary_cache_builder import BinaryCacheBuilder from . test_utils import TestBase, TestHelper @@ -206,3 +207,37 @@ class TestReportHtml(TestBase): self.assertEqual(len(record_data['sampleInfo']), 1) self.assertEqual(record_data['sampleInfo'][0]['eventName'], 'cpu-clock:u') self.assertEqual(record_data['sampleInfo'][0]['eventCount'], 396124304) + + def test_sample_filters(self): + def get_threads_for_filter(filter: str) -> Set[int]: + record_data = self.get_record_data( + ['-i', TestHelper.testdata_path('perf_display_bitmaps.data')] + filter.split()) + threads = set() + try: + for thread in record_data['sampleInfo'][0]['processes'][0]['threads']: + threads.add(thread['tid']) + except IndexError: + pass + return threads + + self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--pid 31850')) + self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--tid 31881')) + self.assertNotIn(31881, get_threads_for_filter( + '--exclude-process-name com.example.android.displayingbitmaps')) + self.assertIn(31881, get_threads_for_filter( + '--include-process-name com.example.android.displayingbitmaps')) + self.assertNotIn(31850, get_threads_for_filter( + '--exclude-thread-name com.example.android.displayingbitmaps')) + self.assertIn(31850, get_threads_for_filter( + '--include-thread-name com.example.android.displayingbitmaps')) + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + threads = get_threads_for_filter('--filter-file ' + filter_file.name) + self.assertIn(31881, threads) + self.assertNotIn(31850, threads) diff --git a/simpleperf/scripts/test/report_lib_test.py b/simpleperf/scripts/test/report_lib_test.py index d4c3f091..61899459 100644 --- a/simpleperf/scripts/test/report_lib_test.py +++ b/simpleperf/scripts/test/report_lib_test.py @@ -14,6 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import tempfile +from typing import Set + from simpleperf_report_lib import ReportLib from . test_utils import TestBase, TestHelper @@ -268,3 +271,36 @@ class TestReportLib(TestBase): self.assertEqual(self.report_lib.GetSupportedTraceOffCpuModes(), []) with self.assertRaises(RuntimeError): self.report_lib.SetTraceOffCpuMode('on-cpu') + + def test_set_sample_filter(self): + """ Test using ReportLib.SetSampleFilter(). """ + def get_threads_for_filter(filter: str) -> Set[int]: + self.report_lib.Close() + self.report_lib = ReportLib() + self.report_lib.SetRecordFile(TestHelper.testdata_path('perf_display_bitmaps.data')) + self.report_lib.SetSampleFilter(filter) + threads = set() + while self.report_lib.GetNextSample(): + sample = self.report_lib.GetCurrentSample() + threads.add(sample.tid) + return threads + + self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) + self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertNotIn(31881, get_threads_for_filter( + '--exclude-process-name com.example.android.displayingbitmaps')) + self.assertIn(31881, get_threads_for_filter( + '--include-process-name com.example.android.displayingbitmaps')) + self.assertNotIn(31850, get_threads_for_filter( + '--exclude-thread-name com.example.android.displayingbitmaps')) + self.assertIn(31850, get_threads_for_filter( + '--include-thread-name com.example.android.displayingbitmaps')) + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + threads = get_threads_for_filter('--filter-file ' + filter_file.name) + self.assertIn(31881, threads) + self.assertNotIn(31850, threads) diff --git a/simpleperf/scripts/test/report_sample_test.py b/simpleperf/scripts/test/report_sample_test.py index c4cf6049..5f1e0aa3 100644 --- a/simpleperf/scripts/test/report_sample_test.py +++ b/simpleperf/scripts/test/report_sample_test.py @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +import re +import tempfile +from typing import Set + from . test_utils import TestBase, TestHelper @@ -73,3 +77,36 @@ class TestReportSample(TestBase): '--trace-offcpu', 'on-cpu'], return_output=True) self.assertIn('cpu-clock:u', got) self.assertNotIn('sched:sched_switch', got) + + def test_sample_filters(self): + def get_threads_for_filter(filter: str) -> Set[int]: + report = self.run_cmd( + ['report_sample.py', '-i', TestHelper.testdata_path('perf_display_bitmaps.data')] + + filter.split(), return_output=True) + pattern = re.compile(r'\s+31850/(\d+)\s+') + threads = set() + for m in re.finditer(pattern, report): + threads.add(int(m.group(1))) + return threads + + self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--pid 31850')) + self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--tid 31881')) + self.assertNotIn(31881, get_threads_for_filter( + '--exclude-process-name com.example.android.displayingbitmaps')) + self.assertIn(31881, get_threads_for_filter( + '--include-process-name com.example.android.displayingbitmaps')) + self.assertNotIn(31850, get_threads_for_filter( + '--exclude-thread-name com.example.android.displayingbitmaps')) + self.assertIn(31850, get_threads_for_filter( + '--include-thread-name com.example.android.displayingbitmaps')) + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + threads = get_threads_for_filter('--filter-file ' + filter_file.name) + self.assertIn(31881, threads) + self.assertNotIn(31850, threads) diff --git a/simpleperf/scripts/test/stackcollapse_test.py b/simpleperf/scripts/test/stackcollapse_test.py index 16260dd0..91c9b6a2 100644 --- a/simpleperf/scripts/test/stackcollapse_test.py +++ b/simpleperf/scripts/test/stackcollapse_test.py @@ -16,6 +16,9 @@ import json from pathlib import Path +import re +import tempfile +from typing import Set from . test_utils import TestBase, TestHelper @@ -85,3 +88,35 @@ class TestStackCollapse(TestBase): ], return_output=True) golden_path = TestHelper.testdata_path('perf_with_jit_symbol.foldedstack_addrs') self.assertEqual(got, Path(golden_path).read_text()) + + def test_sample_filters(self): + def get_threads_for_filter(filter: str) -> Set[int]: + report = self.run_cmd( + ['stackcollapse.py', '-i', TestHelper.testdata_path('perf_display_bitmaps.data'), + '--tid'] + filter.split(), + return_output=True) + pattern = re.compile(r'-31850/(\d+);') + threads = set() + for m in re.finditer(pattern, report): + threads.add(int(m.group(1))) + return threads + + self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) + self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) + self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) + self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertNotIn(31881, get_threads_for_filter( + '--exclude-process-name com.example.android.displayingbitmaps')) + self.assertIn(31881, get_threads_for_filter( + '--include-process-name com.example.android.displayingbitmaps')) + self.assertNotIn(31850, get_threads_for_filter( + '--exclude-thread-name com.example.android.displayingbitmaps')) + self.assertIn(31850, get_threads_for_filter( + '--include-thread-name com.example.android.displayingbitmaps')) + + with tempfile.NamedTemporaryFile('w') as filter_file: + filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') + filter_file.flush() + threads = get_threads_for_filter('--filter-file ' + filter_file.name) + self.assertIn(31881, threads) + self.assertNotIn(31850, threads) |