diff options
69 files changed, 762 insertions, 598 deletions
diff --git a/alloc-stress/Android.bp b/alloc-stress/Android.bp index 4d74841a..e64c822b 100644 --- a/alloc-stress/Android.bp +++ b/alloc-stress/Android.bp @@ -24,6 +24,7 @@ cc_binary { shared_libs: [ "libhardware", "libcutils", + "libprocessgroup", ], cppflags: [ "-g", diff --git a/alloc-stress/alloc-stress.cpp b/alloc-stress/alloc-stress.cpp index 33c12635..ad2353cc 100644 --- a/alloc-stress/alloc-stress.cpp +++ b/alloc-stress/alloc-stress.cpp @@ -14,6 +14,8 @@ #include <tuple> #include <vector> +#include <processgroup/processgroup.h> + //#define TRACE_CHILD_LIFETIME #ifdef TRACE_CHILD_LIFETIME @@ -157,36 +159,6 @@ static void write_oomadj_to_lmkd(int oomadj) { cout << "Wrote " << written << " bytes to lmkd control socket." << endl; } -static void create_memcg() { - char buf[256]; - uid_t uid = getuid(); - pid_t pid = getpid(); - - snprintf(buf, sizeof(buf), "/dev/memcg/apps/uid_%u", uid); - int tasks = mkdir(buf, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); - if (tasks < 0 && errno != EEXIST) { - cerr << "Failed to create memory cgroup under " << buf << endl; - return; - } - - snprintf(buf, sizeof(buf), "/dev/memcg/apps/uid_%u/pid_%u", uid, pid); - tasks = mkdir(buf, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); - if (tasks < 0) { - cerr << "Failed to create memory cgroup under " << buf << endl; - return; - } - - snprintf(buf, sizeof(buf), "/dev/memcg/apps/uid_%u/pid_%u/tasks", uid, pid); - tasks = open(buf, O_WRONLY); - if (tasks < 0) { - cerr << "Unable to add process to memory cgroup" << endl; - return; - } - snprintf(buf, sizeof(buf), "%u", pid); - write(tasks, buf, strlen(buf)); - close(tasks); -} - void usage() { cout << "Application allocates memory until it's killed." << endl << "It starts at max oom_score_adj and gradually " @@ -202,7 +174,7 @@ int main(int argc, char* argv[]) { if ((argc > 1) && (std::string(argv[1]) == "--worker")) { if (std::string(argv[5]) == "1") { - create_memcg(); + createProcessGroup(getuid(), getpid(), true); } write_oomadj_to_lmkd(atoi(argv[2])); diff --git a/boottime_tools/bootanalyze/bootanalyze.py b/boottime_tools/bootanalyze/bootanalyze.py index 7eea05b9..5bed9f29 100755 --- a/boottime_tools/bootanalyze/bootanalyze.py +++ b/boottime_tools/bootanalyze/bootanalyze.py @@ -35,7 +35,6 @@ import yaml from datetime import datetime, date - TIME_DMESG = r"\[\s*(\d+\.\d+)\]" TIME_LOGCAT = r"[0-9]+\.?[0-9]*" KERNEL_TIME_KEY = "kernel" @@ -46,6 +45,7 @@ LAUNCHER_START = "LauncherStart" BOOT_TIME_TOO_BIG = 200.0 MAX_RETRIES = 5 DEBUG = False +DEBUG_PATTERN = False ADB_CMD = "adb" TIMING_THRESHOLD = 5.0 BOOT_PROP = r"\[ro\.boottime\.([^\]]+)\]:\s+\[(\d+)\]" @@ -99,6 +99,8 @@ def main(): for key, pattern in cfg['timings'].items()} shutdown_events_pattern = {key: re.compile(pattern) for key, pattern in cfg['shutdown_events'].items()} + if DEBUG_PATTERN: + print("search event:{} timing event:{}".format(search_events_pattern, timing_events_pattern)) data_points = {} kernel_timing_points = collections.OrderedDict() @@ -301,6 +303,13 @@ def handle_reboot_log(capture_log_on_error, shutdown_events_pattern, components_ print('{0:30}: {1:<7.5}'.format(item[0], item[1])) return shutdown_events, shutdown_timing_events +def collect_dmesg_events(search_events_pattern, timings_pattern, results): + dmesg_events, kernel_timing_events = collect_events(search_events_pattern, ADB_CMD +\ + ' shell su root dmesg -w', timings_pattern,\ + [KERNEL_BOOT_COMPLETE], True) + results.append(dmesg_events) + results.append(kernel_timing_events) + def iterate(args, search_events_pattern, timings_pattern, shutdown_events_pattern, cfg, error_time,\ components_to_monitor): shutdown_events = None @@ -315,16 +324,22 @@ def iterate(args, search_events_pattern, timings_pattern, shutdown_events_patter components_to_monitor) t.join() - dmesg_events, kernel_timing_events = collect_events(search_events_pattern, ADB_CMD +\ - ' shell su root dmesg -w', timings_pattern,\ - [KERNEL_BOOT_COMPLETE], True) + results = [] + t = threading.Thread(target=collect_dmesg_events, args=(search_events_pattern,\ + timings_pattern, results)) + t.start() - logcat_stop_events = [LOGCAT_BOOT_COMPLETE, KERNEL_BOOT_COMPLETE, LAUNCHER_START] + logcat_stop_events = [LOGCAT_BOOT_COMPLETE, LAUNCHER_START] if args.fs_check: logcat_stop_events.append("FsStat") logcat_events, logcat_timing_events = collect_events( search_events_pattern, ADB_CMD + ' logcat -b all -v epoch', timings_pattern,\ logcat_stop_events, False) + + t.join() + dmesg_events = results[0] + kernel_timing_events = results[1] + logcat_event_time = extract_time(logcat_events, TIME_LOGCAT, float) logcat_original_time = extract_time(logcat_events, TIME_LOGCAT, str); dmesg_event_time = extract_time(dmesg_events, TIME_DMESG, float); @@ -352,11 +367,12 @@ def iterate(args, search_events_pattern, timings_pattern, shutdown_events_patter v = v + time_correction_delta debug("correcting event to event[{0}, {1}]".format(k, v)) - if logcat_event_time.get(KERNEL_TIME_KEY) is None: - print("kernel time not captured in logcat, cannot get time diff") - return None, None, None, None, None, None diffs = [] - diffs.append((logcat_event_time[KERNEL_TIME_KEY], logcat_event_time[KERNEL_TIME_KEY])) + if logcat_event_time.get(KERNEL_TIME_KEY) is None: + print("kernel time not captured in logcat") + else: + diffs.append((logcat_event_time[KERNEL_TIME_KEY], logcat_event_time[KERNEL_TIME_KEY])) + if logcat_event_time.get(BOOT_ANIM_END_TIME_KEY) and dmesg_event_time.get(BOOT_ANIM_END_TIME_KEY): diffs.append((logcat_event_time[BOOT_ANIM_END_TIME_KEY],\ logcat_event_time[BOOT_ANIM_END_TIME_KEY] -\ @@ -364,6 +380,7 @@ def iterate(args, search_events_pattern, timings_pattern, shutdown_events_patter if not dmesg_event_time.get(KERNEL_BOOT_COMPLETE): print("BootAnimEnd time or BootComplete-kernel not captured in both log" +\ ", cannot get time diff") + print("dmesg {} logcat {}".format(dmesg_event_time, logcat_event_time)) return None, None, None, None, None, None diffs.append((logcat_event_time[LOGCAT_BOOT_COMPLETE],\ logcat_event_time[LOGCAT_BOOT_COMPLETE] - dmesg_event_time[KERNEL_BOOT_COMPLETE])) @@ -631,56 +648,79 @@ def collect_logcat_for_shutdown(capture_log_on_error, shutdown_events_pattern,\ f.write('\n'.join(lines)) return events, timing_events +def log_timeout(time_left, stop_events, events, timing_events): + print("timeout waiting for event, continue", time_left) + print(" remaininig events {}, event {} timing events {}".\ + format(stop_events, events, timing_events)) def collect_events(search_events, command, timings, stop_events, disable_timing_after_zygote): events = collections.OrderedDict() timing_events = {} - process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + data_available = stop_events is None zygote_pids = [] start_time = time.time() zygote_found = False - line = None - read_poll = select.poll() - read_poll.register(process.stdout, select.POLLIN) + print("remaining stop_events:", stop_events) + init = True while True: + if init: + process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE) + read_poll = select.poll() + read_poll.register(process.stdout, select.POLLIN) + init = False + if len(stop_events) == 0: + break time_left = start_time + max_wait_time - time.time() if time_left <= 0: - print("timeout waiting for event, continue", time_left) + log_timeout(time_left, stop_events, events, timing_events) break - read_r = read_poll.poll(time_left * 1000.0) - if len(read_r) > 0 and read_r[0][1] == select.POLLIN: - line = process.stdout.readline().decode('utf-8', 'ignore') - else: - print("poll timeout waiting for event, continue", time_left) + polled_events = read_poll.poll(time_left * 1000.0) + if len(polled_events) == 0: + log_timeout(time_left, stop_events, events, timing_events) break - if not data_available: - print("Collecting data samples from '%s'. Please wait...\n" % command) - data_available = True - event = get_boot_event(line, search_events) - if event: - debug("event[{0}] captured: {1}".format(event, line)) - if event == "starting_zygote": - events[event] = line - zygote_found = True - elif event.startswith("zygote"): - handle_zygote_event(zygote_pids, events, event, line) + for polled_event in polled_events: + if polled_event[1] == select.POLLIN: + line = process.stdout.readline().decode('utf-8', 'ignore') else: - new_event = update_name_if_already_exist(events, event) - events[new_event] = line - if event in stop_events: - stop_events.remove(event) - print("remaining stop_events:", stop_events) - if len(stop_events) == 0: + if polled_event[1] == select.POLLHUP: + if len(stop_events) == 0: + break; + # adb connection lost + print("poll error waiting for event, adb lost?") + if time_left > 0: + print("retry adb") + run_adb_cmd('wait-for-device') + print(" reconnected") + init = True + continue + else: break - - timing_event = get_boot_event(line, timings) - if timing_event and (not disable_timing_after_zygote or not zygote_found): - if timing_event not in timing_events: - timing_events[timing_event] = [] - timing_events[timing_event].append(line) - debug("timing_event[{0}] captured: {1}".format(timing_event, line)) + if not data_available: + print("Collecting data samples from '%s'. Please wait...\n" % command) + data_available = True + event = get_boot_event(line, search_events) + if event: + debug("event[{0}] captured: {1}".format(event, line)) + if event == "starting_zygote": + events[event] = line + zygote_found = True + elif event.startswith("zygote"): + handle_zygote_event(zygote_pids, events, event, line) + else: + new_event = update_name_if_already_exist(events, event) + events[new_event] = line + if event in stop_events: + stop_events.remove(event) + print("remaining stop_events:", stop_events) + + timing_event = get_boot_event(line, timings) + if timing_event and (not disable_timing_after_zygote or not zygote_found): + if timing_event not in timing_events: + timing_events[timing_event] = [] + timing_events[timing_event].append(line) + debug("timing_event[{0}] captured: {1}".format(timing_event, line)) process.terminate() return events, timing_events @@ -743,7 +783,10 @@ def extract_time(events, pattern, date_transform_function): def do_reboot(serial, use_adb_reboot): - original_devices = subprocess.check_output("adb devices", shell=True) + # do not update time + run_adb_cmd('shell settings put global auto_time 0') + run_adb_cmd('shell settings put global auto_time_zone 0') + original_devices = subprocess.check_output("adb devices", shell=True).decode('utf-8', 'ignore') if use_adb_reboot: print('Rebooting the device using adb reboot') run_adb_cmd('reboot') @@ -780,6 +823,7 @@ def reboot(serial, use_stressfs, permissive, use_adb_reboot, adb_buffersize=None print('Waiting the device') run_adb_cmd('wait-for-device') + print(' found a device') if adb_buffersize is not None: # increase the buffer size diff --git a/boottime_tools/bootanalyze/config.yaml b/boottime_tools/bootanalyze/config.yaml index d6590fdb..83c1bcd8 100644 --- a/boottime_tools/bootanalyze/config.yaml +++ b/boottime_tools/bootanalyze/config.yaml @@ -61,7 +61,7 @@ events: KeyguardShown: KeyguardServiceDelegate.*\*\*\*\* SHOWN CALLED \*\*\*\* BootComplete: Starting phase 1000 BootComplete_kernel: processing action \(sys\.boot_completed=1\) - LauncherStart: START.*HOME.*(NexusLauncherActivity|GEL|LensPickerTrampolineActivity|SetupWizard|CarLauncher) + LauncherStart: START.*HOME.*(NexusLauncherActivity|GEL|LensPickerTrampolineActivity|SetupWizard|CarLauncher|launcher.*Launcher) FsStat: fs_stat, partition:userdata stat:(0x\S+) shutdown_events: ShutdownStart: ShutdownThread:\sNotifying thread to start shutdown diff --git a/cppreopts/fstab.postinstall b/cppreopts/fstab.postinstall index 81272ebb..c77c9083 100644 --- a/cppreopts/fstab.postinstall +++ b/cppreopts/fstab.postinstall @@ -18,4 +18,6 @@ #<src> <mnt_point> <type> <mnt_flags and options> <fs_mgr_flags> system /postinstall ext4 ro,nosuid,nodev,noexec slotselect_other,logical +system /postinstall erofs ro,nosuid,nodev,noexec slotselect_other,logical /dev/block/by-name/system /postinstall ext4 ro,nosuid,nodev,noexec slotselect_other + diff --git a/ext4_utils/mkuserimg_mke2fs.py b/ext4_utils/mkuserimg_mke2fs.py index 599e40ee..41ae8cd7 100644 --- a/ext4_utils/mkuserimg_mke2fs.py +++ b/ext4_utils/mkuserimg_mke2fs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2018 The Android Open Source Project # @@ -165,7 +165,7 @@ def ConstructE2fsCommands(args): if args.flash_erase_block_size: mke2fs_extended_opts.append("stripe_width={}".format( - int(args.flash_erase_block_size) / BLOCKSIZE)) + int(args.flash_erase_block_size) // BLOCKSIZE)) if args.flash_logical_block_size: # stride should be the max of 8kb and the logical block size stride = max(int(args.flash_logical_block_size), 8192) diff --git a/ioshark/compile_ioshark.c b/ioshark/compile_ioshark.c index 13755299..fe9085cf 100644 --- a/ioshark/compile_ioshark.c +++ b/ioshark/compile_ioshark.c @@ -22,7 +22,7 @@ #include <signal.h> #include <string.h> #include <sys/stat.h> -#include <sys/errno.h> +#include <errno.h> #include <fcntl.h> #include <ctype.h> #include "ioshark.h" diff --git a/ioshark/compile_ioshark_subr.c b/ioshark/compile_ioshark_subr.c index 43fb2544..6fc301cc 100644 --- a/ioshark/compile_ioshark_subr.c +++ b/ioshark/compile_ioshark_subr.c @@ -21,7 +21,7 @@ #include <stdlib.h> #include <string.h> #include <sys/stat.h> -#include <sys/errno.h> +#include <errno.h> #include "ioshark.h" #include "compile_ioshark.h" #include <endian.h> diff --git a/ioshark/convert_format.c b/ioshark/convert_format.c index 3436ca77..989bfcb8 100644 --- a/ioshark/convert_format.c +++ b/ioshark/convert_format.c @@ -22,7 +22,7 @@ #include <signal.h> #include <string.h> #include <sys/stat.h> -#include <sys/errno.h> +#include <errno.h> #include <fcntl.h> #include <ctype.h> #include <endian.h> diff --git a/ioshark/dump_ioshark_filenames.c b/ioshark/dump_ioshark_filenames.c index c082c274..a28ab790 100644 --- a/ioshark/dump_ioshark_filenames.c +++ b/ioshark/dump_ioshark_filenames.c @@ -21,7 +21,7 @@ #include <signal.h> #include <string.h> #include <sys/stat.h> -#include <sys/errno.h> +#include <errno.h> #include <fcntl.h> #include <ctype.h> #include "ioshark.h" diff --git a/ioshark/ioshark_bench.c b/ioshark/ioshark_bench.c index e44a2a53..9540b741 100644 --- a/ioshark/ioshark_bench.c +++ b/ioshark/ioshark_bench.c @@ -22,7 +22,7 @@ #include <stdlib.h> #include <string.h> #include <sys/stat.h> -#include <sys/errno.h> +#include <errno.h> #include <fcntl.h> #include <string.h> #include <assert.h> diff --git a/ioshark/ioshark_bench_mmap.c b/ioshark/ioshark_bench_mmap.c index e8b3acce..3b37fbf6 100644 --- a/ioshark/ioshark_bench_mmap.c +++ b/ioshark/ioshark_bench_mmap.c @@ -23,7 +23,7 @@ #include <string.h> #include <pthread.h> #include <sys/stat.h> -#include <sys/errno.h> +#include <errno.h> #include <fcntl.h> #include <string.h> #include <assert.h> diff --git a/libfec/Android.bp b/libfec/Android.bp index c19f6a90..302d8e40 100644 --- a/libfec/Android.bp +++ b/libfec/Android.bp @@ -58,7 +58,7 @@ cc_defaults { "-DFEC_NO_KLOG", ], }, - linux_glibc: { + host_linux: { sanitize: { misc_undefined: ["integer"], }, diff --git a/mtectrl/mtectrl.cc b/mtectrl/mtectrl.cc index 0738f9ec..4e722407 100644 --- a/mtectrl/mtectrl.cc +++ b/mtectrl/mtectrl.cc @@ -24,7 +24,7 @@ int main(int argc, char** argv) { if (argc != 2) { std::cerr << "Usage: " << argv[0] - << " [none|memtag|memtag_once|memtag_kernel|memtag_kernel_once]\n"; + << " none|memtag|memtag-once|memtag-kernel|memtag-kernel-once[,...]\n"; return 1; } std::string value = argv[1]; diff --git a/simpleperf/Android.bp b/simpleperf/Android.bp index b5edda80..58943931 100644 --- a/simpleperf/Android.bp +++ b/simpleperf/Android.bp @@ -131,6 +131,9 @@ cc_defaults { linux_glibc_x86_64: { stl: "libc++_static", }, + linux_musl_x86_64: { + stl: "libc++_static", + }, windows: { enabled: true, stl: "libc++_static", @@ -434,6 +437,16 @@ cc_binary { dir: "simpleperf/linux/x86_64", }, }, + linux_musl_x86: { + dist: { + dir: "simpleperf/linux_musl/x86", + }, + }, + linux_musl_x86_64: { + dist: { + dir: "simpleperf/linux_musl/x86_64", + }, + }, windows_x86: { dist: { dir: "simpleperf/windows/x86", @@ -518,6 +531,16 @@ cc_library_shared { dir: "simpleperf/linux/x86_64", }, }, + linux_musl_x86: { + dist: { + dir: "simpleperf/linux_musl/x86", + }, + }, + linux_musl_x86_64: { + dist: { + dir: "simpleperf/linux_musl/x86_64", + }, + }, windows_x86: { dist: { dir: "simpleperf/windows/x86", diff --git a/simpleperf/Android.mk b/simpleperf/Android.mk index df0009f4..954c28c0 100644 --- a/simpleperf/Android.mk +++ b/simpleperf/Android.mk @@ -33,4 +33,6 @@ SIMPLEPERF_SCRIPT_PATH := \ $(SIMPLEPERF_SCRIPT_PATH) : $(SOONG_ZIP) $(hide) $(SOONG_ZIP) -d -o $@ -C system/extras/simpleperf $(SIMPLEPERF_SCRIPT_LIST) +$(call declare-1p-target,$(SIMPLEPERF_SCRIPT_PATH),system/extras) + $(call dist-for-goals,simpleperf,$(SIMPLEPERF_SCRIPT_PATH):simpleperf/simpleperf_script.zip) diff --git a/simpleperf/ETMDecoder.cpp b/simpleperf/ETMDecoder.cpp index 7c1cca05..62fe997c 100644 --- a/simpleperf/ETMDecoder.cpp +++ b/simpleperf/ETMDecoder.cpp @@ -730,12 +730,19 @@ class ETMDecoderImpl : public ETMDecoder { return false; } size_t left_size = size; + const size_t MAX_RESET_RETRY_COUNT = 3; + size_t reset_retry_count = 0; while (left_size > 0) { uint32_t processed; auto resp = decoder.TraceDataIn(OCSD_OP_DATA, data_index_, left_size, data, &processed); if (IsRespError(resp)) { // A decoding error shouldn't ruin all data. Reset decoders to recover from it. - LOG(DEBUG) << "reset etm decoders for seeing a decode failure, resp " << resp; + // But some errors may not be recoverable by resetting decoders. So use a max retry limit. + if (++reset_retry_count > MAX_RESET_RETRY_COUNT) { + break; + } + LOG(DEBUG) << "reset etm decoders for seeing a decode failure, resp " << resp + << ", reset_retry_count is " << reset_retry_count; decoder.TraceDataIn(OCSD_OP_RESET, data_index_ + processed, 0, nullptr, nullptr); } data += processed; diff --git a/simpleperf/IOEventLoop.cpp b/simpleperf/IOEventLoop.cpp index 06bdd713..239fff97 100644 --- a/simpleperf/IOEventLoop.cpp +++ b/simpleperf/IOEventLoop.cpp @@ -84,6 +84,10 @@ bool IOEventLoop::EnsureInit() { return false; } event_config_free(cfg); + if (event_base_priority_init(ebase_, 2) != 0) { + LOG(ERROR) << "event_base_priority_init failed"; + return false; + } } if (ebase_ == nullptr) { LOG(ERROR) << "failed to create event_base"; @@ -110,39 +114,44 @@ static bool MakeFdNonBlocking(int fd) { return true; } -IOEventRef IOEventLoop::AddReadEvent(int fd, const std::function<bool()>& callback) { +IOEventRef IOEventLoop::AddReadEvent(int fd, const std::function<bool()>& callback, + IOEventPriority priority) { if (!MakeFdNonBlocking(fd)) { return nullptr; } - return AddEvent(fd, EV_READ | EV_PERSIST, nullptr, callback); + return AddEvent(fd, EV_READ | EV_PERSIST, nullptr, callback, priority); } -IOEventRef IOEventLoop::AddWriteEvent(int fd, const std::function<bool()>& callback) { +IOEventRef IOEventLoop::AddWriteEvent(int fd, const std::function<bool()>& callback, + IOEventPriority priority) { if (!MakeFdNonBlocking(fd)) { return nullptr; } - return AddEvent(fd, EV_WRITE | EV_PERSIST, nullptr, callback); + return AddEvent(fd, EV_WRITE | EV_PERSIST, nullptr, callback, priority); } -bool IOEventLoop::AddSignalEvent(int sig, const std::function<bool()>& callback) { - return AddEvent(sig, EV_SIGNAL | EV_PERSIST, nullptr, callback) != nullptr; +bool IOEventLoop::AddSignalEvent(int sig, const std::function<bool()>& callback, + IOEventPriority priority) { + return AddEvent(sig, EV_SIGNAL | EV_PERSIST, nullptr, callback, priority) != nullptr; } -bool IOEventLoop::AddSignalEvents(std::vector<int> sigs, const std::function<bool()>& callback) { +bool IOEventLoop::AddSignalEvents(std::vector<int> sigs, const std::function<bool()>& callback, + IOEventPriority priority) { for (auto sig : sigs) { - if (!AddSignalEvent(sig, callback)) { + if (!AddSignalEvent(sig, callback, priority)) { return false; } } return true; } -IOEventRef IOEventLoop::AddPeriodicEvent(timeval duration, const std::function<bool()>& callback) { - return AddEvent(-1, EV_PERSIST, &duration, callback); +IOEventRef IOEventLoop::AddPeriodicEvent(timeval duration, const std::function<bool()>& callback, + IOEventPriority priority) { + return AddEvent(-1, EV_PERSIST, &duration, callback, priority); } IOEventRef IOEventLoop::AddEvent(int fd_or_sig, int16_t events, timeval* timeout, - const std::function<bool()>& callback) { + const std::function<bool()>& callback, IOEventPriority priority) { if (!EnsureInit()) { return nullptr; } @@ -152,6 +161,7 @@ IOEventRef IOEventLoop::AddEvent(int fd_or_sig, int16_t events, timeval* timeout LOG(ERROR) << "event_new() failed"; return nullptr; } + event_priority_set(e->e, priority); if (event_add(e->e, timeout) != 0) { LOG(ERROR) << "event_add() failed"; return nullptr; diff --git a/simpleperf/IOEventLoop.h b/simpleperf/IOEventLoop.h index f0192969..1578a4d0 100644 --- a/simpleperf/IOEventLoop.h +++ b/simpleperf/IOEventLoop.h @@ -18,6 +18,7 @@ #define SIMPLE_PERF_IOEVENT_LOOP_H_ #include <stdint.h> +#include <sys/time.h> #include <time.h> #include <functional> @@ -31,6 +32,12 @@ namespace simpleperf { struct IOEvent; typedef IOEvent* IOEventRef; +enum IOEventPriority { + // Lower value means higher priority. + IOEventHighPriority = 0, + IOEventLowPriority = 1, +}; + // IOEventLoop is a class wrapper of libevent, it monitors events happened, // and calls the corresponding callbacks. Possible events are: file ready to // read, file ready to write, signal happens, periodic timer timeout. @@ -45,22 +52,27 @@ class IOEventLoop { // Register a read Event, so [callback] is called when [fd] can be read // without blocking. If registered successfully, return the reference // to control the Event, otherwise return nullptr. - IOEventRef AddReadEvent(int fd, const std::function<bool()>& callback); + IOEventRef AddReadEvent(int fd, const std::function<bool()>& callback, + IOEventPriority priority = IOEventLowPriority); // Register a write Event, so [callback] is called when [fd] can be written // without blocking. - IOEventRef AddWriteEvent(int fd, const std::function<bool()>& callback); + IOEventRef AddWriteEvent(int fd, const std::function<bool()>& callback, + IOEventPriority priority = IOEventLowPriority); // Register a signal Event, so [callback] is called each time signal [sig] // happens. - bool AddSignalEvent(int sig, const std::function<bool()>& callback); + bool AddSignalEvent(int sig, const std::function<bool()>& callback, + IOEventPriority priority = IOEventLowPriority); // Register a vector of signal Events. - bool AddSignalEvents(std::vector<int> sigs, const std::function<bool()>& callback); + bool AddSignalEvents(std::vector<int> sigs, const std::function<bool()>& callback, + IOEventPriority priority = IOEventLowPriority); // Register a periodic Event, so [callback] is called periodically every // [duration]. - IOEventRef AddPeriodicEvent(timeval duration, const std::function<bool()>& callback); + IOEventRef AddPeriodicEvent(timeval duration, const std::function<bool()>& callback, + IOEventPriority priority = IOEventLowPriority); // Run a loop polling for Events. It only exits when ExitLoop() is called // in a callback function of registered Events. @@ -80,7 +92,8 @@ class IOEventLoop { private: bool EnsureInit(); IOEventRef AddEvent(int fd_or_sig, int16_t events, timeval* timeout, - const std::function<bool()>& callback); + const std::function<bool()>& callback, + IOEventPriority priority = IOEventLowPriority); static void EventCallbackFn(int, int16_t, void*); event_base* ebase_; diff --git a/simpleperf/IOEventLoop_test.cpp b/simpleperf/IOEventLoop_test.cpp index 09f64522..658fe820 100644 --- a/simpleperf/IOEventLoop_test.cpp +++ b/simpleperf/IOEventLoop_test.cpp @@ -250,3 +250,45 @@ TEST(IOEventLoop, exit_before_loop) { IOEventLoop loop; ASSERT_TRUE(loop.ExitLoop()); } + +TEST(IOEventLoop, priority) { + int low_priority_fd[2]; + ASSERT_EQ(0, pipe(low_priority_fd)); + int high_priority_fd[2]; + ASSERT_EQ(0, pipe(high_priority_fd)); + + IOEventLoop loop; + int count = 0; + + ASSERT_NE(nullptr, loop.AddReadEvent( + low_priority_fd[0], + [&]() { + char c; + read(low_priority_fd[0], &c, 1); + CHECK_EQ(count, 1); + count++; + return loop.ExitLoop(); + }, + IOEventLowPriority)); + + ASSERT_NE(nullptr, loop.AddReadEvent( + high_priority_fd[0], + [&]() { + char c; + read(high_priority_fd[0], &c, 1); + CHECK_EQ(count, 0); + count++; + return true; + }, + IOEventHighPriority)); + + char c; + CHECK_EQ(write(low_priority_fd[1], &c, 1), 1); + CHECK_EQ(write(high_priority_fd[1], &c, 1), 1); + ASSERT_TRUE(loop.RunLoop()); + ASSERT_EQ(2, count); + for (int i = 0; i < 2; i++) { + close(low_priority_fd[i]); + close(high_priority_fd[i]); + } +} diff --git a/simpleperf/JITDebugReader.cpp b/simpleperf/JITDebugReader.cpp index 52db2959..28058940 100644 --- a/simpleperf/JITDebugReader.cpp +++ b/simpleperf/JITDebugReader.cpp @@ -411,17 +411,15 @@ bool JITDebugReader::InitializeProcess(Process& process) { if (art_lib_path.empty()) { return false; } - process.is_64bit = art_lib_path.find("lib64") != std::string::npos; // 2. Read libart.so to find the addresses of __jit_debug_descriptor and __dex_debug_descriptor. - const DescriptorsLocation* location = GetDescriptorsLocation(art_lib_path, process.is_64bit); + const DescriptorsLocation* location = GetDescriptorsLocation(art_lib_path); if (location == nullptr) { return false; } - process.descriptors_addr = location->relative_addr + min_vaddr_in_memory; - process.descriptors_size = location->size; - process.jit_descriptor_offset = location->jit_descriptor_offset; - process.dex_descriptor_offset = location->dex_descriptor_offset; + process.is_64bit = location->is_64bit; + process.jit_descriptor_addr = location->jit_descriptor_addr + min_vaddr_in_memory; + process.dex_descriptor_addr = location->dex_descriptor_addr + min_vaddr_in_memory; for (auto& map : thread_mmaps) { if (StartsWith(map.name, kJITZygoteCacheMmapPrefix)) { @@ -434,10 +432,10 @@ bool JITDebugReader::InitializeProcess(Process& process) { } const JITDebugReader::DescriptorsLocation* JITDebugReader::GetDescriptorsLocation( - const std::string& art_lib_path, bool is_64bit) { + const std::string& art_lib_path) { auto it = descriptors_location_cache_.find(art_lib_path); if (it != descriptors_location_cache_.end()) { - return it->second.relative_addr == 0u ? nullptr : &it->second; + return it->second.jit_descriptor_addr == 0u ? nullptr : &it->second; } DescriptorsLocation& location = descriptors_location_cache_[art_lib_path]; @@ -469,18 +467,9 @@ const JITDebugReader::DescriptorsLocation* JITDebugReader::GetDescriptorsLocatio if (jit_addr == 0u || dex_addr == 0u) { return nullptr; } - location.relative_addr = std::min(jit_addr, dex_addr); - location.size = std::max(jit_addr, dex_addr) + - (is_64bit ? sizeof(JITDescriptor64) : sizeof(JITDescriptor32)) - - location.relative_addr; - if (location.size >= 4096u) { - PLOG(WARNING) << "The descriptors_size is unexpected large: " << location.size; - } - if (descriptors_buf_.size() < location.size) { - descriptors_buf_.resize(location.size); - } - location.jit_descriptor_offset = jit_addr - location.relative_addr; - location.dex_descriptor_offset = dex_addr - location.relative_addr; + location.is_64bit = elf->Is64Bit(); + location.jit_descriptor_addr = jit_addr; + location.dex_descriptor_addr = dex_addr; return &location; } @@ -505,14 +494,40 @@ bool JITDebugReader::ReadRemoteMem(Process& process, uint64_t remote_addr, uint6 bool JITDebugReader::ReadDescriptors(Process& process, Descriptor* jit_descriptor, Descriptor* dex_descriptor) { - if (!ReadRemoteMem(process, process.descriptors_addr, process.descriptors_size, - descriptors_buf_.data())) { + if (process.is_64bit) { + return ReadDescriptorsImpl<JITDescriptor64>(process, jit_descriptor, dex_descriptor); + } + return ReadDescriptorsImpl<JITDescriptor32>(process, jit_descriptor, dex_descriptor); +} + +template <typename DescriptorT> +bool JITDebugReader::ReadDescriptorsImpl(Process& process, Descriptor* jit_descriptor, + Descriptor* dex_descriptor) { + DescriptorT raw_jit_descriptor; + DescriptorT raw_dex_descriptor; + iovec local_iovs[2]; + local_iovs[0].iov_base = &raw_jit_descriptor; + local_iovs[0].iov_len = sizeof(DescriptorT); + local_iovs[1].iov_base = &raw_dex_descriptor; + local_iovs[1].iov_len = sizeof(DescriptorT); + iovec remote_iovs[2]; + remote_iovs[0].iov_base = + reinterpret_cast<void*>(static_cast<uintptr_t>(process.jit_descriptor_addr)); + remote_iovs[0].iov_len = sizeof(DescriptorT); + remote_iovs[1].iov_base = + reinterpret_cast<void*>(static_cast<uintptr_t>(process.dex_descriptor_addr)); + remote_iovs[1].iov_len = sizeof(DescriptorT); + ssize_t result = process_vm_readv(process.pid, local_iovs, 2, remote_iovs, 2, 0); + if (static_cast<size_t>(result) != sizeof(DescriptorT) * 2) { + PLOG(DEBUG) << "ReadDescriptor(pid " << process.pid << ", jit_addr " << std::hex + << process.jit_descriptor_addr << ", dex_addr " << process.dex_descriptor_addr + << ") failed"; + process.died = true; return false; } - if (!LoadDescriptor(process.is_64bit, &descriptors_buf_[process.jit_descriptor_offset], - jit_descriptor) || - !LoadDescriptor(process.is_64bit, &descriptors_buf_[process.dex_descriptor_offset], - dex_descriptor)) { + + if (!ParseDescriptor(raw_jit_descriptor, jit_descriptor) || + !ParseDescriptor(raw_dex_descriptor, dex_descriptor)) { return false; } jit_descriptor->type = DescriptorType::kJIT; @@ -520,17 +535,8 @@ bool JITDebugReader::ReadDescriptors(Process& process, Descriptor* jit_descripto return true; } -bool JITDebugReader::LoadDescriptor(bool is_64bit, const char* data, Descriptor* descriptor) { - if (is_64bit) { - return LoadDescriptorImpl<JITDescriptor64>(data, descriptor); - } - return LoadDescriptorImpl<JITDescriptor32>(data, descriptor); -} - template <typename DescriptorT> -bool JITDebugReader::LoadDescriptorImpl(const char* data, Descriptor* descriptor) { - DescriptorT raw_descriptor; - MoveFromBinaryFormat(raw_descriptor, data); +bool JITDebugReader::ParseDescriptor(const DescriptorT& raw_descriptor, Descriptor* descriptor) { if (!raw_descriptor.Valid()) { return false; } diff --git a/simpleperf/JITDebugReader.h b/simpleperf/JITDebugReader.h index 72f3790b..b9e984d5 100644 --- a/simpleperf/JITDebugReader.h +++ b/simpleperf/JITDebugReader.h @@ -162,13 +162,10 @@ class JITDebugReader { bool initialized = false; bool died = false; bool is_64bit = false; - // The jit descriptor and dex descriptor can be read in one process_vm_readv() call. - uint64_t descriptors_addr = 0; - uint64_t descriptors_size = 0; - // offset relative to descriptors_addr - uint64_t jit_descriptor_offset = 0; - // offset relative to descriptors_addr - uint64_t dex_descriptor_offset = 0; + // remote addr of jit descriptor + uint64_t jit_descriptor_addr = 0; + // remote addr of dex descriptor + uint64_t dex_descriptor_addr = 0; // The state we know about the remote jit debug descriptor. Descriptor last_jit_descriptor; @@ -181,10 +178,9 @@ class JITDebugReader { // The location of descriptors in libart.so. struct DescriptorsLocation { - uint64_t relative_addr = 0; - uint64_t size = 0; - uint64_t jit_descriptor_offset = 0; - uint64_t dex_descriptor_offset = 0; + bool is_64bit = false; + uint64_t jit_descriptor_addr = 0; + uint64_t dex_descriptor_addr = 0; }; bool ReadProcess(Process& process, std::vector<JITDebugInfo>* debug_info); @@ -192,12 +188,14 @@ class JITDebugReader { std::vector<JITDebugInfo>* debug_info); bool IsDescriptorChanged(Process& process, Descriptor& old_descriptor); bool InitializeProcess(Process& process); - const DescriptorsLocation* GetDescriptorsLocation(const std::string& art_lib_path, bool is_64bit); + const DescriptorsLocation* GetDescriptorsLocation(const std::string& art_lib_path); bool ReadRemoteMem(Process& process, uint64_t remote_addr, uint64_t size, void* data); bool ReadDescriptors(Process& process, Descriptor* jit_descriptor, Descriptor* dex_descriptor); - bool LoadDescriptor(bool is_64bit, const char* data, Descriptor* descriptor); template <typename DescriptorT> - bool LoadDescriptorImpl(const char* data, Descriptor* descriptor); + bool ReadDescriptorsImpl(Process& process, Descriptor* jit_descriptor, + Descriptor* dex_descriptor); + template <typename DescriptorT> + bool ParseDescriptor(const DescriptorT& raw_descriptor, Descriptor* descriptor); bool ReadNewCodeEntries(Process& process, const Descriptor& descriptor, uint64_t last_action_timestamp, uint32_t read_entry_limit, @@ -226,7 +224,6 @@ class JITDebugReader { // All monitored processes std::unordered_map<pid_t, Process> processes_; std::unordered_map<std::string, DescriptorsLocation> descriptors_location_cache_; - std::vector<char> descriptors_buf_; std::priority_queue<JITDebugInfo, std::vector<JITDebugInfo>, std::greater<JITDebugInfo>> debug_info_q_; diff --git a/simpleperf/OfflineUnwinder.cpp b/simpleperf/OfflineUnwinder.cpp index f611ef4d..bbc488b4 100644 --- a/simpleperf/OfflineUnwinder.cpp +++ b/simpleperf/OfflineUnwinder.cpp @@ -66,6 +66,9 @@ CHECK_ERROR_CODE(ERROR_INVALID_ELF); CHECK_ERROR_CODE(ERROR_THREAD_DOES_NOT_EXIST); CHECK_ERROR_CODE(ERROR_THREAD_TIMEOUT); CHECK_ERROR_CODE(ERROR_SYSTEM_CALL); +CHECK_ERROR_CODE(ERROR_BAD_ARCH); +CHECK_ERROR_CODE(ERROR_MAPS_PARSE); +CHECK_ERROR_CODE(ERROR_INVALID_PARAMETER); CHECK_ERROR_CODE(ERROR_MAX); // Max frames seen so far is 463, in http://b/110923759. diff --git a/simpleperf/OfflineUnwinder.h b/simpleperf/OfflineUnwinder.h index b6445577..9eb9e8ff 100644 --- a/simpleperf/OfflineUnwinder.h +++ b/simpleperf/OfflineUnwinder.h @@ -39,7 +39,10 @@ enum UnwindStackErrorCode : uint8_t { // not exist. ERROR_THREAD_TIMEOUT, // Timeout trying to unwind a local thread. ERROR_SYSTEM_CALL, // System call failed while unwinding. - ERROR_MAX = ERROR_SYSTEM_CALL, + ERROR_BAD_ARCH, // Arch invalid (none, or mismatched). + ERROR_MAPS_PARSE, // Failed to parse maps data. + ERROR_INVALID_PARAMETER, // Invalid parameter passed to function. + ERROR_MAX = ERROR_INVALID_PARAMETER, }; struct UnwindingResult { diff --git a/simpleperf/cmd_monitor.cpp b/simpleperf/cmd_monitor.cpp index ed9edfac..d81ccfea 100644 --- a/simpleperf/cmd_monitor.cpp +++ b/simpleperf/cmd_monitor.cpp @@ -266,21 +266,22 @@ bool MonitorCommand::PrepareMonitoring() { // 5. Add read/signal/periodic Events. IOEventLoop* loop = event_selection_set_.GetIOEventLoop(); auto exit_loop_callback = [loop]() { return loop->ExitLoop(); }; - if (!loop->AddSignalEvents({SIGCHLD, SIGINT, SIGTERM}, exit_loop_callback)) { + if (!loop->AddSignalEvents({SIGCHLD, SIGINT, SIGTERM}, exit_loop_callback, IOEventHighPriority)) { return false; } // Only add an event for SIGHUP if we didn't inherit SIG_IGN (e.g. from // nohup). if (!SignalIsIgnored(SIGHUP)) { - if (!loop->AddSignalEvent(SIGHUP, exit_loop_callback)) { + if (!loop->AddSignalEvent(SIGHUP, exit_loop_callback, IOEventHighPriority)) { return false; } } if (duration_in_sec_ != 0) { - if (!loop->AddPeriodicEvent(SecondToTimeval(duration_in_sec_), - [loop]() { return loop->ExitLoop(); })) { + if (!loop->AddPeriodicEvent( + SecondToTimeval(duration_in_sec_), [loop]() { return loop->ExitLoop(); }, + IOEventHighPriority)) { return false; } } @@ -291,6 +292,10 @@ bool MonitorCommand::DoMonitoring() { if (!event_selection_set_.GetIOEventLoop()->RunLoop()) { return false; } + if (!event_selection_set_.SyncKernelBuffer()) { + return false; + } + event_selection_set_.CloseEventFiles(); if (!event_selection_set_.FinishReadMmapEventData()) { return false; } diff --git a/simpleperf/cmd_record.cpp b/simpleperf/cmd_record.cpp index f206b808..4b79eedc 100644 --- a/simpleperf/cmd_record.cpp +++ b/simpleperf/cmd_record.cpp @@ -114,9 +114,7 @@ constexpr size_t DEFAULT_CALL_CHAIN_JOINER_CACHE_SIZE = 8 * 1024 * 1024; static constexpr size_t kRecordBufferSize = 64 * 1024 * 1024; static constexpr size_t kSystemWideRecordBufferSize = 256 * 1024 * 1024; -// If the kernel needs to allocate continuous DMA memory for ETR (like when IOMMU for ETR isn't -// available), requesting 4M ETR buffer may fail and cause warning. So use 1M buffer here. -static constexpr size_t kDefaultAuxBufferSize = 1 * 1024 * 1024; +static constexpr size_t kDefaultAuxBufferSize = 4 * 1024 * 1024; // On Pixel 3, it takes about 1ms to enable ETM, and 16-40ms to disable ETM and copy 4M ETM data. // So make default period to 100ms. @@ -223,7 +221,7 @@ class RecordCommand : public Command { "--aux-buffer-size <buffer_size> Set aux buffer size, only used in cs-etm event type.\n" " Need to be power of 2 and page size aligned.\n" " Used memory size is (buffer_size * (cpu_count + 1).\n" -" Default is 1M.\n" +" Default is 4M.\n" "--no-inherit Don't record created child threads/processes.\n" "--cpu-percent <percent> Set the max percent of cpu time used for recording.\n" " percent is in range [1-100], default is 25.\n" @@ -631,25 +629,26 @@ bool RecordCommand::PrepareRecording(Workload* workload) { } IOEventLoop* loop = event_selection_set_.GetIOEventLoop(); auto exit_loop_callback = [loop]() { return loop->ExitLoop(); }; - if (!loop->AddSignalEvents({SIGCHLD, SIGINT, SIGTERM}, exit_loop_callback)) { + if (!loop->AddSignalEvents({SIGCHLD, SIGINT, SIGTERM}, exit_loop_callback, IOEventHighPriority)) { return false; } // Only add an event for SIGHUP if we didn't inherit SIG_IGN (e.g. from nohup). if (!SignalIsIgnored(SIGHUP)) { - if (!loop->AddSignalEvent(SIGHUP, exit_loop_callback)) { + if (!loop->AddSignalEvent(SIGHUP, exit_loop_callback, IOEventHighPriority)) { return false; } } if (stop_signal_fd_ != -1) { - if (!loop->AddReadEvent(stop_signal_fd_, exit_loop_callback)) { + if (!loop->AddReadEvent(stop_signal_fd_, exit_loop_callback, IOEventHighPriority)) { return false; } } if (duration_in_sec_ != 0) { - if (!loop->AddPeriodicEvent(SecondToTimeval(duration_in_sec_), - [loop]() { return loop->ExitLoop(); })) { + if (!loop->AddPeriodicEvent( + SecondToTimeval(duration_in_sec_), [loop]() { return loop->ExitLoop(); }, + IOEventHighPriority)) { return false; } } @@ -720,9 +719,10 @@ bool RecordCommand::DoRecording(Workload* workload) { return false; } time_stat_.stop_recording_time = GetSystemClock(); - if (!event_selection_set_.FinishReadMmapEventData()) { + if (!event_selection_set_.SyncKernelBuffer()) { return false; } + event_selection_set_.CloseEventFiles(); time_stat_.finish_recording_time = GetSystemClock(); uint64_t recording_time = time_stat_.finish_recording_time - time_stat_.start_recording_time; LOG(INFO) << "Recorded for " << recording_time / 1e9 << " seconds. Start post processing."; @@ -756,26 +756,31 @@ static bool WriteRecordDataToOutFd(const std::string& in_filename, } bool RecordCommand::PostProcessRecording(const std::vector<std::string>& args) { - // 1. Merge map records dumped while recording by map record thread. + // 1. Read records left in the buffer. + if (!event_selection_set_.FinishReadMmapEventData()) { + return false; + } + + // 2. Merge map records dumped while recording by map record thread. if (map_record_thread_) { if (!map_record_thread_->Join() || !MergeMapRecords()) { return false; } } - // 2. Post unwind dwarf callchain. + // 3. Post unwind dwarf callchain. if (unwind_dwarf_callchain_ && post_unwind_) { if (!PostUnwindRecords()) { return false; } } - // 3. Optionally join Callchains. + // 4. Optionally join Callchains. if (callchain_joiner_) { JoinCallChains(); } - // 4. Dump additional features, and close record file. + // 5. Dump additional features, and close record file. if (!DumpAdditionalFeatures(args)) { return false; } @@ -787,7 +792,7 @@ bool RecordCommand::PostProcessRecording(const std::vector<std::string>& args) { } time_stat_.post_process_time = GetSystemClock(); - // 4. Show brief record result. + // 6. Show brief record result. auto record_stat = event_selection_set_.GetRecordStat(); if (event_selection_set_.HasAuxTrace()) { LOG(INFO) << "Aux data traced: " << record_stat.aux_data_size; @@ -1280,8 +1285,9 @@ bool RecordCommand::CreateAndInitRecordFile() { return false; } // Use first perf_event_attr and first event id to dump mmap and comm records. - EventAttrWithId dumping_attr_id = event_selection_set_.GetEventAttrWithId()[0]; - map_record_reader_.emplace(*dumping_attr_id.attr, dumping_attr_id.ids[0], + dumping_attr_id_ = event_selection_set_.GetEventAttrWithId()[0]; + CHECK(!dumping_attr_id_.ids.empty()); + map_record_reader_.emplace(*dumping_attr_id_.attr, dumping_attr_id_.ids[0], event_selection_set_.RecordNotExecutableMaps()); map_record_reader_->SetCallback([this](Record* r) { return ProcessRecord(r); }); @@ -1505,14 +1511,13 @@ bool RecordCommand::SaveRecordWithoutUnwinding(Record* record) { bool RecordCommand::ProcessJITDebugInfo(const std::vector<JITDebugInfo>& debug_info, bool sync_kernel_records) { - EventAttrWithId attr_id = event_selection_set_.GetEventAttrWithId()[0]; for (auto& info : debug_info) { if (info.type == JITDebugInfo::JIT_DEBUG_JIT_CODE) { uint64_t timestamp = jit_debug_reader_->SyncWithRecords() ? info.timestamp : last_record_timestamp_; - Mmap2Record record(*attr_id.attr, false, info.pid, info.pid, info.jit_code_addr, + Mmap2Record record(*dumping_attr_id_.attr, false, info.pid, info.pid, info.jit_code_addr, info.jit_code_len, info.file_offset, map_flags::PROT_JIT_SYMFILE_MAP, - info.file_path, attr_id.ids[0], timestamp); + info.file_path, dumping_attr_id_.ids[0], timestamp); if (!ProcessRecord(&record)) { return false; } @@ -1521,8 +1526,9 @@ bool RecordCommand::ProcessJITDebugInfo(const std::vector<JITDebugInfo>& debug_i ThreadMmap& map = *info.extracted_dex_file_map; uint64_t timestamp = jit_debug_reader_->SyncWithRecords() ? info.timestamp : last_record_timestamp_; - Mmap2Record record(*attr_id.attr, false, info.pid, info.pid, map.start_addr, map.len, - map.pgoff, map.prot, map.name, attr_id.ids[0], timestamp); + Mmap2Record record(*dumping_attr_id_.attr, false, info.pid, info.pid, map.start_addr, + map.len, map.pgoff, map.prot, map.name, dumping_attr_id_.ids[0], + timestamp); if (!ProcessRecord(&record)) { return false; } diff --git a/simpleperf/cmd_record_test.cpp b/simpleperf/cmd_record_test.cpp index ef2c1b68..01861c60 100644 --- a/simpleperf/cmd_record_test.cpp +++ b/simpleperf/cmd_record_test.cpp @@ -565,10 +565,12 @@ TEST(record_cmd, exit_with_parent_option) { TEST(record_cmd, use_cmd_exit_code_option) { TemporaryFile tmpfile; int exit_code; - RecordCmd()->Run({"--use-cmd-exit-code", "-o", tmpfile.path, "ls", "."}, &exit_code); - ASSERT_EQ(exit_code, 0); - RecordCmd()->Run({"--use-cmd-exit-code", "-o", tmpfile.path, "ls", "/not_exist_path"}, + RecordCmd()->Run({"-e", GetDefaultEvent(), "--use-cmd-exit-code", "-o", tmpfile.path, "ls", "."}, &exit_code); + ASSERT_EQ(exit_code, 0); + RecordCmd()->Run( + {"-e", GetDefaultEvent(), "--use-cmd-exit-code", "-o", tmpfile.path, "ls", "/not_exist_path"}, + &exit_code); ASSERT_NE(exit_code, 0); } diff --git a/simpleperf/doc/collect_etm_data_for_autofdo.md b/simpleperf/doc/collect_etm_data_for_autofdo.md index f9f5a158..145c0adf 100644 --- a/simpleperf/doc/collect_etm_data_for_autofdo.md +++ b/simpleperf/doc/collect_etm_data_for_autofdo.md @@ -94,7 +94,7 @@ Then we can use a.prof for PGO during compilation, via `-fprofile-sample-use=a.p ## Collect ETM data with a daemon Android also has a daemon collecting ETM data periodically. It only runs on userdebug and eng -devices. The source code is in `<aosp-top>/system/extras/profcollectd`. +devices. The source code is in https://android.googlesource.com/platform/system/extras/+/master/profcollectd/. ## Support ETM in the kernel @@ -107,21 +107,37 @@ The Coresight driver can be enabled by below kernel configs: CONFIG_CORESIGHT=y CONFIG_CORESIGHT_LINK_AND_SINK_TMC=y CONFIG_CORESIGHT_SOURCE_ETM4X=y - CONFIG_CORESIGHT_DYNAMIC_REPLICATOR=y ``` -On Kernel 5.10+, we can build Coresight driver as kernel modules instead. +On Kernel 5.10+, we recommend building Coresight driver as kernel modules. Because it works with +GKI kernel. -Android common kernel 5.10+ should have all the Coresight patches needed. And we have backported -necessary Coresight patches to Android common kernel 4.14 and 4.19. Android common kernel 5.4 -misses a few patches. Please create an [ndk issue](https://github.com/android/ndk/issues) if you -need ETM function on 5.4 kernel. +```config + CONFIG_CORESIGHT=m + CONFIG_CORESIGHT_LINK_AND_SINK_TMC=m + CONFIG_CORESIGHT_SOURCE_ETM4X=m +``` + +Android common kernel 5.10+ should have all the Coresight patches needed to collect ETM data. +Android common kernel 5.4 misses two patches. But by adding patches in +https://android-review.googlesource.com/q/topic:test_etm_on_hikey960_5.4, we can collect ETM data +on hikey960 with 5.4 kernel. +For Android common kernel 4.14 and 4.19, we have backported all necessary Coresight patches. Besides Coresight driver, we also need to add Coresight devices in device tree. An example is in https://github.com/torvalds/linux/blob/master/arch/arm64/boot/dts/arm/juno-base.dtsi. There should be a path flowing ETM data from ETM device through funnels, ETF and replicators, all the way to ETR, which writes ETM data to system memory. +One optional flag in ETM device tree is "arm,coresight-loses-context-with-cpu". It saves ETM +registers when a CPU enters low power state. It may be needed to avoid +"coresight_disclaim_device_unlocked" warning when doing system wide collection. + +One optional flag in ETR device tree is "arm,scatter-gather". Simpleperf requests 4M system memory +for ETR to store ETM data. Without IOMMU, the memory needs to be contiguous. If the kernel can't +fulfill the request, simpleperf will report out of memory error. Fortunately, we can use +"arm,scatter-gather" flag to let ETR run in scatter gather mode, which uses non-contiguous memory. + ## Enable ETM in the bootloader Unless ARMv8.4 Self-hosted Trace extension is implemented, ETM is considered as an external debug diff --git a/simpleperf/dso.cpp b/simpleperf/dso.cpp index bd565b1c..6d559770 100644 --- a/simpleperf/dso.cpp +++ b/simpleperf/dso.cpp @@ -326,14 +326,13 @@ BuildId Dso::FindExpectedBuildIdForPath(const std::string& path) { return BuildId(); } -BuildId Dso::GetExpectedBuildId() { +BuildId Dso::GetExpectedBuildId() const { return FindExpectedBuildIdForPath(path_); } -Dso::Dso(DsoType type, const std::string& path, const std::string& debug_file_path) +Dso::Dso(DsoType type, const std::string& path) : type_(type), path_(path), - debug_file_path_(debug_file_path), is_loaded_(false), dump_id_(UINT_MAX), symbol_dump_id_(0), @@ -465,8 +464,7 @@ static void SortAndFixSymbols(std::vector<Symbol>& symbols) { class DexFileDso : public Dso { public: - DexFileDso(const std::string& path, const std::string& debug_file_path) - : Dso(DSO_DEX_FILE, path, debug_file_path) {} + DexFileDso(const std::string& path) : Dso(DSO_DEX_FILE, path) {} void AddDexFileOffset(uint64_t dex_file_offset) override { auto it = std::lower_bound(dex_file_offsets_.begin(), dex_file_offsets_.end(), dex_file_offset); @@ -484,11 +482,12 @@ class DexFileDso : public Dso { std::vector<Symbol> LoadSymbolsImpl() override { std::vector<Symbol> symbols; - auto tuple = SplitUrlInApk(debug_file_path_); + const std::string& debug_file_path = GetDebugFilePath(); + auto tuple = SplitUrlInApk(debug_file_path); // Symbols of dex files are collected on device. If the dex file doesn't exist, probably // we are reporting on host, and there is no need to report warning of missing dex files. - if (!IsRegularFile(std::get<0>(tuple) ? std::get<1>(tuple) : debug_file_path_)) { - LOG(DEBUG) << "skip reading symbols from non-exist dex_file " << debug_file_path_; + if (!IsRegularFile(std::get<0>(tuple) ? std::get<1>(tuple) : debug_file_path)) { + LOG(DEBUG) << "skip reading symbols from non-exist dex_file " << debug_file_path; return symbols; } bool status = false; @@ -501,19 +500,19 @@ class DexFileDso : public Dso { std::vector<uint8_t> data; if (ahelper && ahelper->FindEntry(std::get<2>(tuple), &entry) && ahelper->GetEntryData(entry, &data)) { - status = ReadSymbolsFromDexFileInMemory(data.data(), data.size(), debug_file_path_, + status = ReadSymbolsFromDexFileInMemory(data.data(), data.size(), debug_file_path, dex_file_offsets_, symbol_callback); } } else { - status = ReadSymbolsFromDexFile(debug_file_path_, dex_file_offsets_, symbol_callback); + status = ReadSymbolsFromDexFile(debug_file_path, dex_file_offsets_, symbol_callback); } if (!status) { android::base::LogSeverity level = symbols_.empty() ? android::base::WARNING : android::base::DEBUG; - LOG(level) << "Failed to read symbols from dex_file " << debug_file_path_; + LOG(level) << "Failed to read symbols from dex_file " << debug_file_path; return symbols; } - LOG(VERBOSE) << "Read symbols from dex_file " << debug_file_path_ << " successfully"; + LOG(VERBOSE) << "Read symbols from dex_file " << debug_file_path << " successfully"; SortAndFixSymbols(symbols); return symbols; } @@ -524,8 +523,8 @@ class DexFileDso : public Dso { class ElfDso : public Dso { public: - ElfDso(const std::string& path, const std::string& debug_file_path) - : Dso(DSO_ELF_FILE, path, debug_file_path) {} + ElfDso(const std::string& path, bool force_64bit) + : Dso(DSO_ELF_FILE, path), force_64bit_(force_64bit) {} std::string_view GetReportPath() const override { if (JITDebugReader::IsPathInJITSymFile(path_)) { @@ -551,11 +550,11 @@ class ElfDso : public Dso { BuildId build_id = GetExpectedBuildId(); ElfStatus status; - auto elf = ElfFile::Open(debug_file_path_, &build_id, &status); + auto elf = ElfFile::Open(GetDebugFilePath(), &build_id, &status); if (elf) { min_vaddr_ = elf->ReadMinExecutableVaddr(&file_offset_of_min_vaddr_); } else { - LOG(WARNING) << "failed to read min virtual address of " << debug_file_path_ << ": " + LOG(WARNING) << "failed to read min virtual address of " << GetDebugFilePath() << ": " << status; } } @@ -587,7 +586,7 @@ class ElfDso : public Dso { // ELF_FILE Dso objects should actually be DEX_FILE, because they have dex file offsets. // So here converts ELF_FILE Dso into DEX_FILE Dso. type_ = DSO_DEX_FILE; - dex_file_dso_.reset(new DexFileDso(path_, path_)); + dex_file_dso_.reset(new DexFileDso(path_)); } dex_file_dso_->AddDexFileOffset(dex_file_offset); } @@ -597,6 +596,11 @@ class ElfDso : public Dso { } protected: + std::string FindDebugFilePath() const override { + BuildId build_id = GetExpectedBuildId(); + return debug_elf_file_finder_.FindDebugFile(path_, force_64bit_, build_id); + } + std::vector<Symbol> LoadSymbolsImpl() override { if (dex_file_dso_) { return dex_file_dso_->LoadSymbolsImpl(); @@ -609,11 +613,11 @@ class ElfDso : public Dso { } }; ElfStatus status; - auto elf = ElfFile::Open(debug_file_path_, &build_id, &status); + auto elf = ElfFile::Open(GetDebugFilePath(), &build_id, &status); if (elf) { status = elf->ParseSymbols(symbol_callback); } - ReportReadElfSymbolResult(status, path_, debug_file_path_, + ReportReadElfSymbolResult(status, path_, GetDebugFilePath(), symbols_.empty() ? android::base::WARNING : android::base::DEBUG); SortAndFixSymbols(symbols); return symbols; @@ -622,6 +626,7 @@ class ElfDso : public Dso { private: static constexpr uint64_t uninitialized_value = std::numeric_limits<uint64_t>::max(); + bool force_64bit_; uint64_t min_vaddr_ = uninitialized_value; uint64_t file_offset_of_min_vaddr_ = uninitialized_value; std::unique_ptr<DexFileDso> dex_file_dso_; @@ -629,8 +634,8 @@ class ElfDso : public Dso { class KernelDso : public Dso { public: - KernelDso(const std::string& path, const std::string& debug_file_path) - : Dso(DSO_KERNEL, path, debug_file_path) { + KernelDso(const std::string& path) : Dso(DSO_KERNEL, path) { + debug_file_path_ = FindDebugFilePath(); if (!vmlinux_.empty()) { // Use vmlinux as the kernel debug file. BuildId build_id = GetExpectedBuildId(); @@ -639,7 +644,7 @@ class KernelDso : public Dso { debug_file_path_ = vmlinux_; has_debug_file_ = true; } - } else if (IsRegularFile(debug_file_path_)) { + } else if (IsRegularFile(GetDebugFilePath())) { has_debug_file_ = true; } } @@ -663,6 +668,11 @@ class KernelDso : public Dso { } protected: + std::string FindDebugFilePath() const override { + BuildId build_id = GetExpectedBuildId(); + return debug_elf_file_finder_.FindDebugFile(path_, false, build_id); + } + std::vector<Symbol> LoadSymbolsImpl() override { std::vector<Symbol> symbols; if (has_debug_file_) { @@ -689,7 +699,7 @@ class KernelDso : public Dso { if (!fix_kernel_address_randomization_) { LOG(WARNING) << "Don't know how to fix addresses changed by kernel address randomization. So " "symbols in " - << debug_file_path_ << " are not used"; + << GetDebugFilePath() << " are not used"; return; } // symbols_ are kernel symbols got from /proc/kallsyms while recording. Those symbols are @@ -703,10 +713,10 @@ class KernelDso : public Dso { } }; ElfStatus status; - if (auto elf = ElfFile::Open(debug_file_path_, &status); elf) { + if (auto elf = ElfFile::Open(GetDebugFilePath(), &status); elf) { status = elf->ParseSymbols(symbol_callback); } - ReportReadElfSymbolResult(status, path_, debug_file_path_); + ReportReadElfSymbolResult(status, path_, GetDebugFilePath()); } void ReadSymbolsFromKallsyms(std::string& kallsyms, std::vector<Symbol>* symbols) { @@ -771,7 +781,7 @@ class KernelDso : public Dso { kernel_start_file_offset_ = 0; if (has_debug_file_) { ElfStatus status; - if (auto elf = ElfFile::Open(debug_file_path_, &status); elf) { + if (auto elf = ElfFile::Open(GetDebugFilePath(), &status); elf) { for (const auto& section : elf->GetSectionHeader()) { if (section.name == ".text") { kernel_start_addr_ = section.vaddr; @@ -791,9 +801,9 @@ class KernelDso : public Dso { class KernelModuleDso : public Dso { public: - KernelModuleDso(const std::string& path, const std::string& debug_file_path, - uint64_t memory_start, uint64_t memory_end, Dso* kernel_dso) - : Dso(DSO_KERNEL_MODULE, path, debug_file_path), + KernelModuleDso(const std::string& path, uint64_t memory_start, uint64_t memory_end, + Dso* kernel_dso) + : Dso(DSO_KERNEL_MODULE, path), memory_start_(memory_start), memory_end_(memory_end), kernel_dso_(kernel_dso) {} @@ -819,6 +829,11 @@ class KernelModuleDso : public Dso { } protected: + std::string FindDebugFilePath() const override { + BuildId build_id = GetExpectedBuildId(); + return debug_elf_file_finder_.FindDebugFile(path_, false, build_id); + } + std::vector<Symbol> LoadSymbolsImpl() override { std::vector<Symbol> symbols; BuildId build_id = GetExpectedBuildId(); @@ -829,11 +844,11 @@ class KernelModuleDso : public Dso { } }; ElfStatus status; - auto elf = ElfFile::Open(debug_file_path_, &build_id, &status); + auto elf = ElfFile::Open(GetDebugFilePath(), &build_id, &status); if (elf) { status = elf->ParseSymbols(symbol_callback); } - ReportReadElfSymbolResult(status, path_, debug_file_path_, + ReportReadElfSymbolResult(status, path_, GetDebugFilePath(), symbols_.empty() ? android::base::WARNING : android::base::DEBUG); SortAndFixSymbols(symbols); return symbols; @@ -897,7 +912,7 @@ class KernelModuleDso : public Dso { class SymbolMapFileDso : public Dso { public: - SymbolMapFileDso(const std::string& path) : Dso(DSO_SYMBOL_MAP_FILE, path, path) {} + SymbolMapFileDso(const std::string& path) : Dso(DSO_SYMBOL_MAP_FILE, path) {} uint64_t IpToVaddrInFile(uint64_t ip, uint64_t, uint64_t) override { return ip; } @@ -907,7 +922,7 @@ class SymbolMapFileDso : public Dso { class UnknownDso : public Dso { public: - UnknownDso(const std::string& path) : Dso(DSO_UNKNOWN_FILE, path, path) {} + UnknownDso(const std::string& path) : Dso(DSO_UNKNOWN_FILE, path) {} uint64_t IpToVaddrInFile(uint64_t ip, uint64_t, uint64_t) override { return ip; } @@ -917,15 +932,13 @@ class UnknownDso : public Dso { std::unique_ptr<Dso> Dso::CreateDso(DsoType dso_type, const std::string& dso_path, bool force_64bit) { - BuildId build_id = FindExpectedBuildIdForPath(dso_path); - std::string debug_path = debug_elf_file_finder_.FindDebugFile(dso_path, force_64bit, build_id); switch (dso_type) { case DSO_ELF_FILE: - return std::unique_ptr<Dso>(new ElfDso(dso_path, debug_path)); + return std::unique_ptr<Dso>(new ElfDso(dso_path, force_64bit)); case DSO_KERNEL: - return std::unique_ptr<Dso>(new KernelDso(dso_path, debug_path)); + return std::unique_ptr<Dso>(new KernelDso(dso_path)); case DSO_DEX_FILE: - return std::unique_ptr<Dso>(new DexFileDso(dso_path, dso_path)); + return std::unique_ptr<Dso>(new DexFileDso(dso_path)); case DSO_SYMBOL_MAP_FILE: return std::unique_ptr<Dso>(new SymbolMapFileDso(dso_path)); case DSO_UNKNOWN_FILE: @@ -938,26 +951,28 @@ std::unique_ptr<Dso> Dso::CreateDso(DsoType dso_type, const std::string& dso_pat std::unique_ptr<Dso> Dso::CreateDsoWithBuildId(DsoType dso_type, const std::string& dso_path, BuildId& build_id) { - std::string debug_path = debug_elf_file_finder_.FindDebugFile(dso_path, false, build_id); + std::unique_ptr<Dso> dso; switch (dso_type) { case DSO_ELF_FILE: - return std::unique_ptr<Dso>(new ElfDso(dso_path, debug_path)); + dso.reset(new ElfDso(dso_path, false)); + break; case DSO_KERNEL: - return std::unique_ptr<Dso>(new KernelDso(dso_path, debug_path)); + dso.reset(new KernelDso(dso_path)); + break; case DSO_KERNEL_MODULE: - return std::unique_ptr<Dso>(new KernelModuleDso(dso_path, debug_path, 0, 0, nullptr)); + dso.reset(new KernelModuleDso(dso_path, 0, 0, nullptr)); + break; default: LOG(FATAL) << "Unexpected dso_type " << static_cast<int>(dso_type); + return nullptr; } - return nullptr; + dso->debug_file_path_ = debug_elf_file_finder_.FindDebugFile(dso_path, false, build_id); + return dso; } std::unique_ptr<Dso> Dso::CreateKernelModuleDso(const std::string& dso_path, uint64_t memory_start, uint64_t memory_end, Dso* kernel_dso) { - BuildId build_id = FindExpectedBuildIdForPath(dso_path); - std::string debug_path = debug_elf_file_finder_.FindDebugFile(dso_path, false, build_id); - return std::unique_ptr<Dso>( - new KernelModuleDso(dso_path, debug_path, memory_start, memory_end, kernel_dso)); + return std::unique_ptr<Dso>(new KernelModuleDso(dso_path, memory_start, memory_end, kernel_dso)); } const char* DsoTypeToString(DsoType dso_type) { diff --git a/simpleperf/dso.h b/simpleperf/dso.h index 866aadfd..30427766 100644 --- a/simpleperf/dso.h +++ b/simpleperf/dso.h @@ -145,7 +145,13 @@ class Dso { // Return the path recorded in perf.data. const std::string& Path() const { return path_; } // Return the path containing symbol table and debug information. - const std::string& GetDebugFilePath() const { return debug_file_path_; } + const std::string& GetDebugFilePath() const { + if (!debug_file_path_.has_value()) { + debug_file_path_ = FindDebugFilePath(); + } + return debug_file_path_.value(); + } + // Return the path beautified for reporting. virtual std::string_view GetReportPath() const { return Path(); } // Return the file name without directory info. @@ -195,9 +201,10 @@ class Dso { static uint32_t g_dump_id_; static simpleperf_dso_impl::DebugElfFileFinder debug_elf_file_finder_; - Dso(DsoType type, const std::string& path, const std::string& debug_file_path); - BuildId GetExpectedBuildId(); + Dso(DsoType type, const std::string& path); + BuildId GetExpectedBuildId() const; + virtual std::string FindDebugFilePath() const { return path_; } virtual std::vector<Symbol> LoadSymbolsImpl() = 0; DsoType type_; @@ -205,7 +212,7 @@ class Dso { const std::string path_; // path of the shared library having symbol table and debug information // It is the same as path_, or has the same build id as path_. - std::string debug_file_path_; + mutable std::optional<std::string> debug_file_path_; // File name of the shared library, got by removing directories in path_. std::string file_name_; std::vector<Symbol> symbols_; diff --git a/simpleperf/dso_test.cpp b/simpleperf/dso_test.cpp index 1338c1c0..53f62a57 100644 --- a/simpleperf/dso_test.cpp +++ b/simpleperf/dso_test.cpp @@ -310,3 +310,15 @@ TEST(dso, FunctionName) { symbol = Symbol("ctep.v", 0x0, 0x1); ASSERT_EQ(symbol.FunctionName(), "ctep.v"); } + +TEST(dso, search_debug_file_only_when_needed) { + Dso::SetBuildIds({std::make_pair("/elf", BuildId("1b12a384a9f4a3f3659b7171ca615dbec3a81f71"))}); + Dso::SetSymFsDir(GetTestDataDir()); + CapturedStderr capture; + capture.Start(); + auto dso = Dso::CreateDso(DSO_ELF_FILE, "/elf"); + ASSERT_EQ(capture.str().find("build id mismatch"), std::string::npos); + ASSERT_EQ(dso->GetDebugFilePath(), "/elf"); + ASSERT_NE(capture.str().find("build id mismatch"), std::string::npos); + capture.Stop(); +} diff --git a/simpleperf/environment.h b/simpleperf/environment.h index 1dd6952e..34ce7fa5 100644 --- a/simpleperf/environment.h +++ b/simpleperf/environment.h @@ -144,7 +144,8 @@ static inline uint64_t GetSystemClock() { #if defined(__ANDROID__) bool IsInAppUid(); -#else +#endif +#if !defined(__ANDROID__) && !defined(ANDROID_HOST_MUSL) static inline int gettid() { return syscall(__NR_gettid); } diff --git a/simpleperf/event_selection_set.cpp b/simpleperf/event_selection_set.cpp index d5bca318..58b4b956 100644 --- a/simpleperf/event_selection_set.cpp +++ b/simpleperf/event_selection_set.cpp @@ -840,14 +840,7 @@ bool EventSelectionSet::ReadMmapEventData(bool with_time_limit) { } bool EventSelectionSet::FinishReadMmapEventData() { - // Stop the read thread, so we don't get more records beyond current time. - if (!SyncKernelBuffer() || !record_read_thread_->StopReadThread()) { - return false; - } - if (!ReadMmapEventData(false)) { - return false; - } - return true; + return ReadMmapEventData(false); } void EventSelectionSet::CloseEventFiles() { diff --git a/simpleperf/report_lib_interface.cpp b/simpleperf/report_lib_interface.cpp index c55209ab..1025c325 100644 --- a/simpleperf/report_lib_interface.cpp +++ b/simpleperf/report_lib_interface.cpp @@ -199,7 +199,7 @@ class ReportLib { } const char* GetSupportedTraceOffCpuModes(); bool SetTraceOffCpuMode(const char* mode); - bool SetSampleFilter(const char* filter); + bool SetSampleFilter(const char** filters, int filters_len); Sample* GetNextSample(); Event* GetEventOfCurrentSample() { return ¤t_event_; } @@ -296,8 +296,11 @@ bool ReportLib::SetTraceOffCpuMode(const char* mode) { return true; } -bool ReportLib::SetSampleFilter(const char* filter) { - std::vector<std::string> args = android::base::Split(filter, " "); +bool ReportLib::SetSampleFilter(const char** filters, int filters_len) { + std::vector<std::string> args; + for (int i = 0; i < filters_len; i++) { + args.emplace_back(filters[i]); + } OptionFormatMap option_formats = GetRecordFilterOptionFormats(false); OptionValueMap options; std::vector<std::pair<OptionName, OptionValue>> ordered_options; @@ -595,7 +598,7 @@ void MergeJavaMethods(ReportLib* report_lib, bool merge) EXPORT; bool AddProguardMappingFile(ReportLib* report_lib, const char* mapping_file) EXPORT; const char* GetSupportedTraceOffCpuModes(ReportLib* report_lib) EXPORT; bool SetTraceOffCpuMode(ReportLib* report_lib, const char* mode) EXPORT; -bool SetSampleFilter(ReportLib* report_lib, const char* filter) EXPORT; +bool SetSampleFilter(ReportLib* report_lib, const char** filters, int filters_len) EXPORT; Sample* GetNextSample(ReportLib* report_lib) EXPORT; Event* GetEventOfCurrentSample(ReportLib* report_lib) EXPORT; @@ -656,8 +659,8 @@ bool SetTraceOffCpuMode(ReportLib* report_lib, const char* mode) { return report_lib->SetTraceOffCpuMode(mode); } -bool SetSampleFilter(ReportLib* report_lib, const char* filter) { - return report_lib->SetSampleFilter(filter); +bool SetSampleFilter(ReportLib* report_lib, const char** filters, int filters_len) { + return report_lib->SetSampleFilter(filters, filters_len); } Sample* GetNextSample(ReportLib* report_lib) { diff --git a/simpleperf/scripts/annotate.py b/simpleperf/scripts/annotate.py index 16af623e..7bf8fe92 100755 --- a/simpleperf/scripts/annotate.py +++ b/simpleperf/scripts/annotate.py @@ -146,7 +146,7 @@ class SourceFileAnnotator(object): def __init__(self, config): # check config variables - config_names = ['perf_data_list', 'source_dirs', 'comm_filters', 'dso_filters', 'ndk_path'] + config_names = ['perf_data_list', 'source_dirs', 'dso_filters', 'ndk_path'] for name in config_names: if name not in config: log_exit('config [%s] is missing' % name) @@ -161,7 +161,6 @@ class SourceFileAnnotator(object): self.config = config self.symfs_dir = symfs_dir self.kallsyms = kallsyms - self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None config['annotate_dest_dir'] = 'annotated_files' @@ -193,15 +192,12 @@ class SourceFileAnnotator(object): lib.SetSymfs(self.symfs_dir) if self.kallsyms: lib.SetKallsymsFile(self.kallsyms) - if self.config.get('sample_filter'): - lib.SetSampleFilter(self.config.get('sample_filter')) + lib.SetReportOptions(self.config['report_lib_options']) while True: sample = lib.GetNextSample() if sample is None: lib.Close() break - if not self._filter_sample(sample): - continue symbols = [] symbols.append(lib.GetSymbolOfCurrentSample()) callchain = lib.GetCallChainOfCurrentSample() @@ -215,13 +211,6 @@ class SourceFileAnnotator(object): self.addr2line.add_addr(symbol.dso_name, build_id, symbol.symbol_addr, symbol.symbol_addr) - def _filter_sample(self, sample): - """Return true if the sample can be used.""" - if self.comm_filter: - if sample.thread_comm not in self.comm_filter: - return False - return True - def _filter_symbol(self, symbol): if not self.dso_filter or symbol.dso_name in self.dso_filter: return True @@ -241,15 +230,12 @@ class SourceFileAnnotator(object): lib.SetSymfs(self.symfs_dir) if self.kallsyms: lib.SetKallsymsFile(self.kallsyms) - if self.config.get('sample_filter'): - lib.SetSampleFilter(self.config.get('sample_filter')) + lib.SetReportOptions(self.config['report_lib_options']) while True: sample = lib.GetNextSample() if sample is None: lib.Close() break - if not self._filter_sample(sample): - continue self._generate_periods_for_sample(lib, sample) def _generate_periods_for_sample(self, lib, sample): @@ -477,11 +463,9 @@ def main(): help='show raw period instead of percentage') parser.add_argument('--summary-width', type=int, default=80, help='max width of summary file') sample_filter_group = parser.add_argument_group('Sample filter options') - parser.add_sample_filter_options(sample_filter_group) - sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" - Use samples only in threads with selected names.""") sample_filter_group.add_argument('--dso', nargs='+', action='append', help=""" Use samples only in selected binaries.""") + parser.add_report_lib_options(sample_filter_group=sample_filter_group) args = parser.parse_args() config = {} @@ -489,12 +473,11 @@ def main(): if not config['perf_data_list']: config['perf_data_list'].append('perf.data') config['source_dirs'] = flatten_arg_list(args.source_dirs) - config['comm_filters'] = flatten_arg_list(args.comm) config['dso_filters'] = flatten_arg_list(args.dso) config['ndk_path'] = args.ndk_path config['raw_period'] = args.raw_period config['summary_width'] = args.summary_width - config['sample_filter'] = args.sample_filter + config['report_lib_options'] = args.report_lib_options annotator = SourceFileAnnotator(config) annotator.annotate() diff --git a/simpleperf/scripts/bin/android/arm/simpleperf b/simpleperf/scripts/bin/android/arm/simpleperf Binary files differindex 5ccbd4f1..db2fde7f 100755 --- a/simpleperf/scripts/bin/android/arm/simpleperf +++ b/simpleperf/scripts/bin/android/arm/simpleperf diff --git a/simpleperf/scripts/bin/android/arm64/simpleperf b/simpleperf/scripts/bin/android/arm64/simpleperf Binary files differindex 862e4d93..785b6e33 100755 --- a/simpleperf/scripts/bin/android/arm64/simpleperf +++ b/simpleperf/scripts/bin/android/arm64/simpleperf diff --git a/simpleperf/scripts/bin/android/x86/simpleperf b/simpleperf/scripts/bin/android/x86/simpleperf Binary files differindex a78e0514..97a168d5 100755 --- a/simpleperf/scripts/bin/android/x86/simpleperf +++ b/simpleperf/scripts/bin/android/x86/simpleperf diff --git a/simpleperf/scripts/bin/android/x86_64/simpleperf b/simpleperf/scripts/bin/android/x86_64/simpleperf Binary files differindex 8872cf4a..c250c450 100755 --- a/simpleperf/scripts/bin/android/x86_64/simpleperf +++ b/simpleperf/scripts/bin/android/x86_64/simpleperf diff --git a/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib b/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib Binary files differindex 32631f67..191e51f8 100755 --- a/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib +++ b/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib diff --git a/simpleperf/scripts/bin/darwin/x86_64/simpleperf b/simpleperf/scripts/bin/darwin/x86_64/simpleperf Binary files differindex c4daaf2b..1a6a8875 100755 --- a/simpleperf/scripts/bin/darwin/x86_64/simpleperf +++ b/simpleperf/scripts/bin/darwin/x86_64/simpleperf diff --git a/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so b/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so Binary files differindex 6f148a8e..ca444ec9 100755 --- a/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so +++ b/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so diff --git a/simpleperf/scripts/bin/linux/x86_64/simpleperf b/simpleperf/scripts/bin/linux/x86_64/simpleperf Binary files differindex f2968884..dca7b30d 100755 --- a/simpleperf/scripts/bin/linux/x86_64/simpleperf +++ b/simpleperf/scripts/bin/linux/x86_64/simpleperf diff --git a/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll b/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll Binary files differindex 074bff34..b49e8f8f 100755 --- a/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll +++ b/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll diff --git a/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe b/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe Binary files differindex fc98293a..9838e771 100755 --- a/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe +++ b/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe diff --git a/simpleperf/scripts/gecko_profile_generator.py b/simpleperf/scripts/gecko_profile_generator.py index e7108ee6..396c9a58 100755 --- a/simpleperf/scripts/gecko_profile_generator.py +++ b/simpleperf/scripts/gecko_profile_generator.py @@ -30,7 +30,7 @@ import sys from dataclasses import dataclass, field from simpleperf_report_lib import ReportLib -from simpleperf_utils import BaseArgumentParser, flatten_arg_list +from simpleperf_utils import BaseArgumentParser, flatten_arg_list, ReportLibOptions from typing import List, Dict, Optional, NamedTuple, Set, Tuple @@ -295,22 +295,17 @@ def _gecko_profile( record_file: str, symfs_dir: Optional[str], kallsyms_file: Optional[str], - proguard_mapping_file: List[str], - comm_filter: Set[str], - sample_filter: Optional[str]) -> GeckoProfile: + report_lib_options: ReportLibOptions) -> GeckoProfile: """convert a simpleperf profile to gecko format""" lib = ReportLib() lib.ShowIpForUnknownSymbol() - for file_path in proguard_mapping_file: - lib.AddProguardMappingFile(file_path) if symfs_dir is not None: lib.SetSymfs(symfs_dir) lib.SetRecordFile(record_file) if kallsyms_file is not None: lib.SetKallsymsFile(kallsyms_file) - if sample_filter: - lib.SetSampleFilter(sample_filter) + lib.SetReportOptions(report_lib_options) arch = lib.GetArch() meta_info = lib.MetaInfo() @@ -324,9 +319,6 @@ def _gecko_profile( if sample is None: lib.Close() break - if comm_filter: - if sample.thread_comm not in comm_filter: - continue event = lib.GetEventOfCurrentSample() symbol = lib.GetSymbolOfCurrentSample() callchain = lib.GetCallChainOfCurrentSample() @@ -405,22 +397,13 @@ def main() -> None: parser.add_argument('--kallsyms', help='Set the path to find kernel symbols.') parser.add_argument('-i', '--record_file', nargs='?', default='perf.data', help='Default is perf.data.') - parser.add_argument( - '--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols', - default=[]) - sample_filter_group = parser.add_argument_group('Sample filter options') - parser.add_sample_filter_options(sample_filter_group) - sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" - Use samples only in threads with selected names.""") + parser.add_report_lib_options() args = parser.parse_args() profile = _gecko_profile( record_file=args.record_file, symfs_dir=args.symfs, kallsyms_file=args.kallsyms, - proguard_mapping_file=args.proguard_mapping_file, - comm_filter=set(flatten_arg_list(args.comm)), - sample_filter=args.sample_filter) + report_lib_options=args.report_lib_options) json.dump(profile, sys.stdout, sort_keys=True) diff --git a/simpleperf/scripts/inferno/inferno.py b/simpleperf/scripts/inferno/inferno.py index 512b1dc9..ccf1b6b6 100755 --- a/simpleperf/scripts/inferno/inferno.py +++ b/simpleperf/scripts/inferno/inferno.py @@ -111,14 +111,7 @@ def parse_samples(process, args, sample_filter_fn): lib.SetRecordFile(record_file) if kallsyms_file: lib.SetKallsymsFile(kallsyms_file) - if args.show_art_frames: - lib.ShowArtFrames(True) - for file_path in args.proguard_mapping_file or []: - lib.AddProguardMappingFile(file_path) - if args.trace_offcpu: - lib.SetTraceOffCpuMode(args.trace_offcpu) - if args.sample_filter: - lib.SetSampleFilter(args.sample_filter) + lib.SetReportOptions(args.report_lib_options) process.cmd = lib.GetRecordCmd() product_props = lib.MetaInfo().get("product_props") if product_props: @@ -313,12 +306,8 @@ def main(): report_group.add_argument('--symfs', help="""Set the path to find binaries with symbols and debug info.""") report_group.add_argument('--title', help='Show a title in the report.') - report_group.add_argument('--show_art_frames', action='store_true', - help='Show frames of internal methods in the ART Java interpreter.') - report_group.add_argument('--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols') - parser.add_trace_offcpu_option(report_group) - parser.add_sample_filter_options(report_group, False) + parser.add_report_lib_options( + report_group, sample_filter_group=report_group, sample_filter_with_pid_shortcut=False) debug_group = parser.add_argument_group('Debug options') debug_group.add_argument('--disable_adb_root', action='store_true', help="""Force adb to run diff --git a/simpleperf/scripts/pprof_proto_generator.py b/simpleperf/scripts/pprof_proto_generator.py index 2e807d95..57c988b9 100755 --- a/simpleperf/scripts/pprof_proto_generator.py +++ b/simpleperf/scripts/pprof_proto_generator.py @@ -270,7 +270,6 @@ class PprofProfileGenerator(object): config['binary_cache_dir'] = 'binary_cache' if not os.path.isdir(config['binary_cache_dir']): config['binary_cache_dir'] = None - self.comm_filter = set(config['comm_filters']) if config.get('comm_filters') else None self.dso_filter = set(config['dso_filters']) if config.get('dso_filters') else None self.max_chain_length = config['max_chain_length'] self.profile = profile_pb2.Profile() @@ -303,10 +302,7 @@ class PprofProfileGenerator(object): if self.config.get('show_art_frames'): self.lib.ShowArtFrames() - for file_path in self.config['proguard_mapping_file'] or []: - self.lib.AddProguardMappingFile(file_path) - if self.config.get('sample_filter'): - self.lib.SetSampleFilter(self.config['sample_filter']) + self.lib.SetReportOptions(self.config['report_lib_options']) comments = [ "Simpleperf Record Command:\n" + self.lib.GetRecordCmd(), @@ -329,9 +325,6 @@ class PprofProfileGenerator(object): symbol = self.lib.GetSymbolOfCurrentSample() callchain = self.lib.GetCallChainOfCurrentSample() - if not self._filter_report_sample(report_sample): - continue - sample_type_id = self.get_sample_type_id(event.name) sample = Sample() sample.add_value(sample_type_id, 1) @@ -379,13 +372,6 @@ class PprofProfileGenerator(object): return self.profile - def _filter_report_sample(self, sample): - """Return true if the sample can be used.""" - if self.comm_filter: - if sample.thread_comm not in self.comm_filter: - return False - return True - def _filter_symbol(self, symbol): if not self.dso_filter or symbol.dso_name in self.dso_filter: return True @@ -633,20 +619,13 @@ def main(): parser.add_argument('--max_chain_length', type=int, default=1000000000, help=""" Maximum depth of samples to be converted.""") # Large value as infinity standin. parser.add_argument('--ndk_path', type=extant_dir, help='Set the path of a ndk release.') - parser.add_argument('--show_art_frames', action='store_true', - help='Show frames of internal methods in the ART Java interpreter.') - parser.add_argument( - '--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols') parser.add_argument( '-j', '--jobs', type=int, default=os.cpu_count(), help='Use multithreading to speed up source code annotation.') sample_filter_group = parser.add_argument_group('Sample filter options') - parser.add_sample_filter_options(sample_filter_group) - sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" - Use samples only in threads with selected names.""") sample_filter_group.add_argument('--dso', nargs='+', action='append', help=""" Use samples only in selected binaries.""") + parser.add_report_lib_options(sample_filter_group=sample_filter_group) args = parser.parse_args() if args.show: @@ -658,18 +637,16 @@ def main(): config = {} config['output_file'] = args.output_file - config['comm_filters'] = flatten_arg_list(args.comm) config['dso_filters'] = flatten_arg_list(args.dso) config['ndk_path'] = args.ndk_path - config['show_art_frames'] = args.show_art_frames config['max_chain_length'] = args.max_chain_length - config['proguard_mapping_file'] = args.proguard_mapping_file - config['sample_filter'] = args.sample_filter + config['report_lib_options'] = args.report_lib_options generator = PprofProfileGenerator(config) for record_file in args.record_file: generator.load_record_file(record_file) profile = generator.gen(args.jobs) store_pprof_profile(config['output_file'], profile) + logging.info("Report is generated at '%s' successfully." % config['output_file']) if __name__ == '__main__': diff --git a/simpleperf/scripts/purgatorio/purgatorio.py b/simpleperf/scripts/purgatorio/purgatorio.py index 0f072547..4e2560a2 100755 --- a/simpleperf/scripts/purgatorio/purgatorio.py +++ b/simpleperf/scripts/purgatorio/purgatorio.py @@ -43,6 +43,7 @@ from functools import cmp_to_key simpleperf_path = Path(__file__).absolute().parents[1] sys.path.insert(0, str(simpleperf_path)) import simpleperf_report_lib as sp +from simpleperf_utils import BaseArgumentParser # fmt: on @@ -169,11 +170,7 @@ def generate_datasource(args): if args.ksyms: lib.SetKallsymsFile(args.ksyms) - if not args.not_art: - lib.ShowArtFrames(True) - - for file_path in args.proguard_mapping_file or []: - lib.AddProguardMappingFile(file_path) + lib.SetReportOptions(args.report_lib_options) product = lib.MetaInfo().get('product_props') @@ -268,21 +265,19 @@ def generate_datasource(args): def main(): - parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = BaseArgumentParser() parser.add_argument('-i', '--input_file', type=str, required=True, help='input file') parser.add_argument('--title', '-t', type=str, help='document title') parser.add_argument('--ksyms', '-k', type=str, help='path to kernel symbols (kallsyms)') parser.add_argument('--usyms', '-u', type=str, help='path to tree with user space symbols') - parser.add_argument('--not_art', '-a', action='store_true', help='Don\'t show ART symbols') parser.add_argument('--output', '-o', type=str, help='output file') parser.add_argument('--dont_open', '-d', action='store_true', help='Don\'t open output file') parser.add_argument('--include_dso_names', '-n', action='store_true', help='Include dso names in backtraces') parser.add_argument('--include_symbols_addr', '-s', action='store_true', help='Include addresses of symbols in backtraces') - parser.add_argument( - '--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols') + parser.add_report_lib_options(default_show_art_frames=True) + args = parser.parse_args() # TODO test hierarchical ranges too diff --git a/simpleperf/scripts/report_html.py b/simpleperf/scripts/report_html.py index 9de2f98b..56f8dae5 100755 --- a/simpleperf/scripts/report_html.py +++ b/simpleperf/scripts/report_html.py @@ -31,7 +31,7 @@ from typing import Any, Callable, Dict, Iterator, List, Optional, Set, Tuple, Un from simpleperf_report_lib import ReportLib, SymbolStruct from simpleperf_utils import ( Addr2Nearestline, BaseArgumentParser, BinaryFinder, get_script_dir, log_exit, Objdump, - open_report_in_browser, ReadElf, SourceFileSearcher) + open_report_in_browser, ReadElf, ReportLibOptions, SourceFileSearcher) MAX_CALLSTACK_LENGTH = 750 @@ -605,13 +605,10 @@ class RecordData(object): def __init__( self, binary_cache_path: Optional[str], ndk_path: Optional[str], - build_addr_hit_map: bool, proguard_mapping_files: Optional[List[str]], - trace_offcpu: Optional[str]): + build_addr_hit_map: bool): self.binary_cache_path = binary_cache_path self.ndk_path = ndk_path self.build_addr_hit_map = build_addr_hit_map - self.proguard_mapping_files = proguard_mapping_files - self.trace_offcpu = trace_offcpu self.meta_info: Optional[Dict[str, str]] = None self.cmdline: Optional[str] = None self.arch: Optional[str] = None @@ -623,23 +620,15 @@ class RecordData(object): self.gen_addr_hit_map_in_record_info = False self.binary_finder = BinaryFinder(binary_cache_path, ReadElf(ndk_path)) - def load_record_file( - self, record_file: str, show_art_frames: bool, sample_filter: Optional[str]): + def load_record_file(self, record_file: str, report_lib_options: ReportLibOptions): lib = ReportLib() lib.SetRecordFile(record_file) # If not showing ip for unknown symbols, the percent of the unknown symbol may be # accumulated to very big, and ranks first in the sample table. lib.ShowIpForUnknownSymbol() - if show_art_frames: - lib.ShowArtFrames() if self.binary_cache_path: lib.SetSymfs(self.binary_cache_path) - for file_path in self.proguard_mapping_files or []: - lib.AddProguardMappingFile(file_path) - if self.trace_offcpu: - lib.SetTraceOffCpuMode(self.trace_offcpu) - if sample_filter: - lib.SetSampleFilter(sample_filter) + lib.SetReportOptions(report_lib_options) self.meta_info = lib.MetaInfo() self.cmdline = lib.GetRecordCmd() self.arch = lib.GetArch() @@ -987,16 +976,10 @@ def get_args() -> argparse.Namespace: help='Use multithreading to speed up disassembly and source code annotation.') parser.add_argument('--ndk_path', nargs=1, help='Find tools in the ndk path.') parser.add_argument('--no_browser', action='store_true', help="Don't open report in browser.") - parser.add_argument('--show_art_frames', action='store_true', - help='Show frames of internal methods in the ART Java interpreter.') parser.add_argument('--aggregate-by-thread-name', action='store_true', help="""aggregate samples by thread name instead of thread id. This is useful for showing multiple perf.data generated for the same app.""") - parser.add_argument( - '--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols') - parser.add_trace_offcpu_option() - parser.add_sample_filter_options() + parser.add_report_lib_options() return parser.parse_args() @@ -1022,10 +1005,9 @@ def main(): log_exit('Invalid --jobs option.') # 2. Produce record data. - record_data = RecordData(binary_cache_path, ndk_path, build_addr_hit_map, - args.proguard_mapping_file, args.trace_offcpu) + record_data = RecordData(binary_cache_path, ndk_path, build_addr_hit_map) for record_file in args.record_file: - record_data.load_record_file(record_file, args.show_art_frames, args.sample_filter) + record_data.load_record_file(record_file, args.report_lib_options) if args.aggregate_by_thread_name: record_data.aggregate_by_thread_name() record_data.limit_percents(args.min_func_percent, args.min_callchain_percent) diff --git a/simpleperf/scripts/report_sample.py b/simpleperf/scripts/report_sample.py index 7388442a..dc5c4e2b 100755 --- a/simpleperf/scripts/report_sample.py +++ b/simpleperf/scripts/report_sample.py @@ -19,7 +19,7 @@ """ from simpleperf_report_lib import ReportLib -from simpleperf_utils import BaseArgumentParser, flatten_arg_list +from simpleperf_utils import BaseArgumentParser, flatten_arg_list, ReportLibOptions from typing import List, Set, Optional @@ -28,27 +28,19 @@ def report_sample( symfs_dir: str, kallsyms_file: str, show_tracing_data: bool, - proguard_mapping_file: List[str], header: bool, - comm_filter: Set[str], - trace_offcpu: Optional[str], - sample_filter: Optional[str]): + report_lib_options: ReportLibOptions): """ read record_file, and print each sample""" lib = ReportLib() lib.ShowIpForUnknownSymbol() - for file_path in proguard_mapping_file: - lib.AddProguardMappingFile(file_path) if symfs_dir is not None: lib.SetSymfs(symfs_dir) if record_file is not None: lib.SetRecordFile(record_file) if kallsyms_file is not None: lib.SetKallsymsFile(kallsyms_file) - if trace_offcpu: - lib.SetTraceOffCpuMode(trace_offcpu) - if sample_filter: - lib.SetSampleFilter(sample_filter) + lib.SetReportOptions(report_lib_options) if header: print("# ========") @@ -64,9 +56,6 @@ def report_sample( if sample is None: lib.Close() break - if comm_filter: - if sample.thread_comm not in comm_filter: - continue event = lib.GetEventOfCurrentSample() symbol = lib.GetSymbolOfCurrentSample() callchain = lib.GetCallChainOfCurrentSample() @@ -97,27 +86,17 @@ def main(): parser.add_argument('-i', '--record_file', nargs='?', default='perf.data', help='Default is perf.data.') parser.add_argument('--show_tracing_data', action='store_true', help='print tracing data.') - parser.add_argument( - '--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols', - default=[]) parser.add_argument('--header', action='store_true', help='Show metadata header, like perf script --header') - parser.add_argument('--comm', nargs='+', action='append', help=""" - Use samples only in threads with selected names.""") - parser.add_trace_offcpu_option() - parser.add_sample_filter_options() + parser.add_report_lib_options() args = parser.parse_args() report_sample( record_file=args.record_file, symfs_dir=args.symfs, kallsyms_file=args.kallsyms, show_tracing_data=args.show_tracing_data, - proguard_mapping_file=args.proguard_mapping_file, header=args.header, - comm_filter=set(flatten_arg_list(args.comm)), - trace_offcpu=args.trace_offcpu, - sample_filter=args.sample_filter) + report_lib_options=args.report_lib_options) if __name__ == '__main__': diff --git a/simpleperf/scripts/simpleperf_report_lib.py b/simpleperf/scripts/simpleperf_report_lib.py index 5c2893c1..48b31199 100644 --- a/simpleperf/scripts/simpleperf_report_lib.py +++ b/simpleperf/scripts/simpleperf_report_lib.py @@ -26,7 +26,8 @@ from pathlib import Path import struct from typing import Any, Dict, List, Optional, Union -from simpleperf_utils import bytes_to_str, get_host_binary_path, is_windows, str_to_bytes +from simpleperf_utils import (bytes_to_str, get_host_binary_path, is_windows, str_to_bytes, + ReportLibOptions) def _is_null(p: Optional[ct._Pointer]) -> bool: @@ -297,6 +298,18 @@ class ReportLib(object): self._DestroyReportLibFunc(self._instance) self._instance = None + def SetReportOptions(self, options: ReportLibOptions): + """ Set report options in one call. """ + if options.proguard_mapping_files: + for file_path in options.proguard_mapping_files: + self.AddProguardMappingFile(file_path) + if options.show_art_frames: + self.ShowArtFrames(True) + if options.trace_offcpu: + self.SetTraceOffCpuMode(options.trace_offcpu) + if options.sample_filters: + self.SetSampleFilter(options.sample_filters) + def SetLogSeverity(self, log_level: str = 'info'): """ Set log severity of native lib, can be verbose,debug,info,error,fatal.""" cond: bool = self._SetLogSeverityFunc(self.getInstance(), _char_pt(log_level)) @@ -362,7 +375,7 @@ class ReportLib(object): res: bool = self._SetTraceOffCpuModeFunc(self.getInstance(), _char_pt(mode)) _check(res, f'Failed to call SetTraceOffCpuMode({mode})') - def SetSampleFilter(self, filter: str): + def SetSampleFilter(self, filters: List[str]): """ Set options used to filter samples. Available options are: --exclude-pid pid1,pid2,... Exclude samples for selected processes. --exclude-tid tid1,tid2,... Exclude samples for selected threads. @@ -381,8 +394,10 @@ class ReportLib(object): The filter argument should be a concatenation of options. """ - res: bool = self._SetSampleFilterFunc(self.getInstance(), _char_pt(filter)) - _check(res, f'Failed to call SetSampleFilter({filter})') + filter_array = (ct.c_char_p * len(filters))() + filter_array[:] = [_char_pt(f) for f in filters] + res: bool = self._SetSampleFilterFunc(self.getInstance(), filter_array, len(filters)) + _check(res, f'Failed to call SetSampleFilter({filters})') def GetNextSample(self) -> Optional[SampleStruct]: """ Return the next sample. If no more samples, return None. """ diff --git a/simpleperf/scripts/simpleperf_utils.py b/simpleperf/scripts/simpleperf_utils.py index 755919c0..2a7dfd35 100644 --- a/simpleperf/scripts/simpleperf_utils.py +++ b/simpleperf/scripts/simpleperf_utils.py @@ -21,6 +21,7 @@ from __future__ import annotations import argparse from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass import logging import os import os.path @@ -999,14 +1000,33 @@ class ArgParseFormatter( pass +@dataclass +class ReportLibOptions: + show_art_frames: bool + trace_offcpu: str + proguard_mapping_files: List[str] + sample_filters: List[str] + + class BaseArgumentParser(argparse.ArgumentParser): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs, formatter_class=ArgParseFormatter) self.has_sample_filter_options = False self.sample_filter_with_pid_shortcut = False - - def add_trace_offcpu_option(self, subparser: Optional[Any] = None): - parser = subparser if subparser else self + self.has_report_lib_options = False + + def add_report_lib_options(self, group: Optional[Any] = None, + default_show_art_frames: bool = False, + sample_filter_group: Optional[Any] = None, + sample_filter_with_pid_shortcut: bool = True): + self.has_report_lib_options = True + parser = group if group else self + parser.add_argument( + '--proguard-mapping-file', nargs='+', + help='Add proguard mapping file to de-obfuscate symbols') + parser.add_argument('--show-art-frames', '--show_art_frames', + action=argparse.BooleanOptionalAction, default=default_show_art_frames, + help='Show frames of internal methods in the ART Java interpreter.') parser.add_argument( '--trace-offcpu', choices=['on-cpu', 'off-cpu', 'on-off-cpu', 'mixed-on-off-cpu'], help="""Set report mode for profiles recorded with --trace-offcpu option. All possible @@ -1015,8 +1035,9 @@ class BaseArgumentParser(argparse.ArgumentParser): mixed-on-off-cpu (on-cpu and off-cpu samples using the same event name). If not set, mixed-on-off-cpu mode is used. """) + self._add_sample_filter_options(sample_filter_group, sample_filter_with_pid_shortcut) - def add_sample_filter_options( + def _add_sample_filter_options( self, group: Optional[Any] = None, with_pid_shortcut: bool = True): if not group: group = self.add_argument_group('Sample filter options') @@ -1044,7 +1065,8 @@ class BaseArgumentParser(argparse.ArgumentParser): '--include-process-name', metavar='process_name_regex', nargs='+', help='only include samples for processes with name containing the regular expression') group.add_argument( - '--include-thread-name', metavar='thread_name_regex', nargs='+', + '--comm', '--include-thread-name', metavar='thread_name_regex', + dest='include_thread_name', nargs='+', help='only include samples for threads with name containing the regular expression') group.add_argument( '--filter-file', metavar='file', @@ -1053,40 +1075,38 @@ class BaseArgumentParser(argparse.ArgumentParser): self.has_sample_filter_options = True self.sample_filter_with_pid_shortcut = with_pid_shortcut - def _build_sample_filter(self, args: argparse.Namespace) -> Optional[str]: - """ Convert sample filter options into a sample filter string, which can be passed to - ReportLib.SetSampleFilter(). - """ + def _build_sample_filter(self, args: argparse.Namespace) -> List[str]: + """ Build sample filters, which can be passed to ReportLib.SetSampleFilter(). """ filters = [] if args.exclude_pid: - filters.append('--exclude-pid ' + ','.join(str(pid) for pid in args.exclude_pid)) + filters.extend(['--exclude-pid', ','.join(str(pid) for pid in args.exclude_pid)]) if args.exclude_tid: - filters.append('--exclude-tid ' + ','.join(str(tid) for tid in args.exclude_tid)) + filters.extend(['--exclude-tid', ','.join(str(tid) for tid in args.exclude_tid)]) if args.exclude_process_name: for name in args.exclude_process_name: - filters.append('--exclude-process-name ' + name) + filters.extend(['--exclude-process-name', name]) if args.exclude_thread_name: for name in args.exclude_thread_name: - filters.append('--exclude-thread-name ' + name) + filters.extend(['--exclude-thread-name', name]) if args.include_pid: - filters.append('--include-pid ' + ','.join(str(pid) for pid in args.include_pid)) + filters.extend(['--include-pid', ','.join(str(pid) for pid in args.include_pid)]) if args.include_tid: - filters.append('--include-tid ' + ','.join(str(tid) for tid in args.include_tid)) + filters.extend(['--include-tid', ','.join(str(tid) for tid in args.include_tid)]) if self.sample_filter_with_pid_shortcut: if args.pid: - filters.append('--include-pid ' + ','.join(str(pid) for pid in args.pid)) + filters.extend(['--include-pid', ','.join(str(pid) for pid in args.pid)]) if args.tid: - filters.append('--include-tid ' + ','.join(str(pid) for pid in args.tid)) + filters.extend(['--include-tid', ','.join(str(pid) for pid in args.tid)]) if args.include_process_name: for name in args.include_process_name: - filters.append('--include-process-name ' + name) + filters.extend(['--include-process-name', name]) if args.include_thread_name: for name in args.include_thread_name: - filters.append('--include-thread-name ' + name) + filters.extend(['--include-thread-name', name]) if args.filter_file: - filters.append('--filter-file ' + args.filter_file) - return ' '.join(filters) + filters.extend(['--filter-file', args.filter_file]) + return filters def parse_known_args(self, *args, **kwargs): self.add_argument( @@ -1094,8 +1114,12 @@ class BaseArgumentParser(argparse.ArgumentParser): default='info', help='set log level') namespace, left_args = super().parse_known_args(*args, **kwargs) - if self.has_sample_filter_options: - setattr(namespace, 'sample_filter', self._build_sample_filter(namespace)) + if self.has_report_lib_options: + sample_filters = self._build_sample_filter(namespace) + report_lib_options = ReportLibOptions( + namespace.show_art_frames, namespace.trace_offcpu, namespace.proguard_mapping_file, + sample_filters) + setattr(namespace, 'report_lib_options', report_lib_options) if not Log.initialized: Log.init(namespace.log) diff --git a/simpleperf/scripts/stackcollapse.py b/simpleperf/scripts/stackcollapse.py index 2574843a..e0e1d86f 100755 --- a/simpleperf/scripts/stackcollapse.py +++ b/simpleperf/scripts/stackcollapse.py @@ -26,7 +26,7 @@ from collections import defaultdict from simpleperf_report_lib import ReportLib -from simpleperf_utils import BaseArgumentParser, flatten_arg_list +from simpleperf_utils import BaseArgumentParser, flatten_arg_list, ReportLibOptions from typing import DefaultDict, List, Optional, Set import logging @@ -37,30 +37,25 @@ def collapse_stacks( record_file: str, symfs_dir: str, kallsyms_file: str, - proguard_mapping_file: List[str], event_filter: str, include_pid: bool, include_tid: bool, annotate_kernel: bool, annotate_jit: bool, include_addrs: bool, - comm_filter: Set[str], - sample_filter: Optional[str]): + report_lib_options: ReportLibOptions): """read record_file, aggregate per-stack and print totals per-stack""" lib = ReportLib() if include_addrs: lib.ShowIpForUnknownSymbol() - for file_path in proguard_mapping_file: - lib.AddProguardMappingFile(file_path) if symfs_dir is not None: lib.SetSymfs(symfs_dir) if record_file is not None: lib.SetRecordFile(record_file) if kallsyms_file is not None: lib.SetKallsymsFile(kallsyms_file) - if sample_filter: - lib.SetSampleFilter(sample_filter) + lib.SetReportOptions(report_lib_options) stacks: DefaultDict[str, int] = defaultdict(int) event_defaulted = False @@ -70,9 +65,6 @@ def collapse_stacks( if sample is None: lib.Close() break - if comm_filter: - if sample.thread_comm not in comm_filter: - continue event = lib.GetEventOfCurrentSample() symbol = lib.GetSymbolOfCurrentSample() callchain = lib.GetCallChainOfCurrentSample() @@ -123,30 +115,23 @@ def main(): parser.add_argument('--jit', action='store_true', help='Annotate JIT functions with a _[j]') parser.add_argument('--addrs', action='store_true', help='include raw addresses where symbols can\'t be found') - parser.add_argument( - '--proguard-mapping-file', nargs='+', - help='Add proguard mapping file to de-obfuscate symbols', - default=[]) sample_filter_group = parser.add_argument_group('Sample filter options') - parser.add_sample_filter_options(sample_filter_group, False) sample_filter_group.add_argument('--event-filter', nargs='?', default='', help='Event type filter e.g. "cpu-cycles" or "instructions"') - sample_filter_group.add_argument('--comm', nargs='+', action='append', help=""" - Use samples only in threads with selected names.""") + parser.add_report_lib_options(sample_filter_group=sample_filter_group, + sample_filter_with_pid_shortcut=False) args = parser.parse_args() collapse_stacks( record_file=args.record_file, symfs_dir=args.symfs, kallsyms_file=args.kallsyms, - proguard_mapping_file=args.proguard_mapping_file, event_filter=args.event_filter, include_pid=args.pid, include_tid=args.tid, annotate_kernel=args.kernel, annotate_jit=args.jit, include_addrs=args.addrs, - comm_filter=set(flatten_arg_list(args.comm)), - sample_filter=args.sample_filter) + report_lib_options=args.report_lib_options) if __name__ == '__main__': diff --git a/simpleperf/scripts/test/annotate_test.py b/simpleperf/scripts/test/annotate_test.py index eef5133a..ce419f66 100644 --- a/simpleperf/scripts/test/annotate_test.py +++ b/simpleperf/scripts/test/annotate_test.py @@ -74,3 +74,10 @@ class TestAnnotate(TestBase): filter_file.flush() get_report('--filter-file ' + filter_file.name) os.unlink(filter_file.name) + + def test_show_art_frames(self): + self.run_cmd( + ['annotate.py', '-i', TestHelper.testdata_path('perf_with_interpreter_frames.data'), + '--show-art-frames']) + summary = Path('annotated_files') / 'summary' + self.check_strings_in_file(summary, 'total period: 9800649') diff --git a/simpleperf/scripts/test/do_test.py b/simpleperf/scripts/test/do_test.py index 2c67bb78..b95a1fef 100755 --- a/simpleperf/scripts/test/do_test.py +++ b/simpleperf/scripts/test/do_test.py @@ -32,6 +32,7 @@ import multiprocessing as mp import os from pathlib import Path import re +import subprocess import sys import time from tqdm import tqdm @@ -39,7 +40,7 @@ import types from typing import List, Optional import unittest -from simpleperf_utils import BaseArgumentParser, extant_dir, log_exit, remove +from simpleperf_utils import BaseArgumentParser, extant_dir, log_exit, remove, is_darwin from . api_profiler_test import * from . annotate_test import * @@ -517,6 +518,15 @@ def run_tests_in_child_process(tests: List[str], args: argparse.Namespace) -> bo return False +def sign_executables_on_darwin(): + """ Sign executables on M1 Mac, otherwise they can't run. """ + if not is_darwin(): + return + bin_dir = Path(__file__).resolve().parents[1] / 'bin' / 'darwin' / 'x86_64' + for path in bin_dir.iterdir(): + subprocess.run(f'codesign --force -s - {path}', shell=True, check=True) + + def main() -> bool: args = get_args() tests = get_host_tests() if args.only_host_test else get_all_tests() @@ -532,4 +542,5 @@ def main() -> bool: # Switch to the test dir. os.chdir(test_dir) build_testdata(Path('testdata')) + sign_executables_on_darwin() return run_tests_in_child_process(tests, args) diff --git a/simpleperf/scripts/test/gecko_profile_generator_test.py b/simpleperf/scripts/test/gecko_profile_generator_test.py index 54bde746..5a0d45ec 100644 --- a/simpleperf/scripts/test/gecko_profile_generator_test.py +++ b/simpleperf/scripts/test/gecko_profile_generator_test.py @@ -18,20 +18,22 @@ import json import os import re import tempfile -from typing import Set +from typing import List, Optional, Set from . test_utils import TestBase, TestHelper class TestGeckoProfileGenerator(TestBase): - def run_generator(self, testdata_file): + def run_generator(self, testdata_file: str, options: Optional[List[str]] = None) -> str: testdata_path = TestHelper.testdata_path(testdata_file) - gecko_profile_json = self.run_cmd( - ['gecko_profile_generator.py', '-i', testdata_path], return_output=True) - return json.loads(gecko_profile_json) + args = ['gecko_profile_generator.py', '-i', testdata_path] + if options: + args.extend(options) + return self.run_cmd(args, return_output=True) def test_golden(self): - got = self.run_generator('perf_with_interpreter_frames.data') + output = self.run_generator('perf_with_interpreter_frames.data') + got = json.loads(output) golden_path = TestHelper.testdata_path('perf_with_interpreter_frames.gecko.json') with open(golden_path) as f: want = json.load(f) @@ -41,8 +43,7 @@ class TestGeckoProfileGenerator(TestBase): def test_sample_filters(self): def get_threads_for_filter(filter: str) -> Set[int]: - report = self.run_cmd(['gecko_profile_generator.py', '-i', TestHelper.testdata_path( - 'perf_display_bitmaps.data')] + filter.split(), return_output=True) + report = self.run_generator('perf_display_bitmaps.data', filter.split()) pattern = re.compile(r'"tid":\s+(\d+),') threads = set() for m in re.finditer(pattern, report): @@ -71,3 +72,10 @@ class TestGeckoProfileGenerator(TestBase): self.assertIn(31881, threads) self.assertNotIn(31850, threads) os.unlink(filter_file.name) + + def test_show_art_frames(self): + art_frame_str = 'art::interpreter::DoCall' + report = self.run_generator('perf_with_interpreter_frames.data') + self.assertNotIn(art_frame_str, report) + report = self.run_generator('perf_with_interpreter_frames.data', ['--show-art-frames']) + self.assertIn(art_frame_str, report) diff --git a/simpleperf/scripts/test/inferno_test.py b/simpleperf/scripts/test/inferno_test.py index bddb2200..83fc7cf8 100644 --- a/simpleperf/scripts/test/inferno_test.py +++ b/simpleperf/scripts/test/inferno_test.py @@ -81,3 +81,12 @@ class TestInferno(TestBase): self.assertIn(31881, threads) self.assertNotIn(31850, threads) os.unlink(filter_file.name) + + def test_show_art_frames(self): + art_frame_str = 'art::interpreter::DoCall' + options = ['--record_file', + TestHelper.testdata_path('perf_with_interpreter_frames.data'), '-sc'] + report = self.get_report(options) + self.assertNotIn(art_frame_str, report) + report = self.get_report(options + ['--show-art-frames']) + self.assertIn(art_frame_str, report) diff --git a/simpleperf/scripts/test/pprof_proto_generator_test.py b/simpleperf/scripts/test/pprof_proto_generator_test.py index 0d964841..cbeb8d6a 100644 --- a/simpleperf/scripts/test/pprof_proto_generator_test.py +++ b/simpleperf/scripts/test/pprof_proto_generator_test.py @@ -24,6 +24,7 @@ from typing import List, Optional, Set from binary_cache_builder import BinaryCacheBuilder from pprof_proto_generator import load_pprof_profile, PprofProfileGenerator from . test_utils import TestBase, TestHelper +from simpleperf_utils import ReportLibOptions class TestPprofProtoGenerator(TestBase): @@ -216,7 +217,8 @@ class TestPprofProtoGenerator(TestBase): binary_cache_builder.build_binary_cache(testdata_file, [TestHelper.testdata_dir]) # Read recording file. - config = {'ndk_path': None, 'max_chain_length': 1000000, 'proguard_mapping_file': None} + config = {'ndk_path': None, 'max_chain_length': 1000000, + 'report_lib_options': ReportLibOptions(False, '', None, None)} generator = PprofProfileGenerator(config) generator.load_record_file(testdata_file) diff --git a/simpleperf/scripts/test/purgatorio_test.py b/simpleperf/scripts/test/purgatorio_test.py index e814c42d..ff862f4f 100644 --- a/simpleperf/scripts/test/purgatorio_test.py +++ b/simpleperf/scripts/test/purgatorio_test.py @@ -42,3 +42,11 @@ class TestPurgatorio(TestBase): # Show original method name with proguard mapping file. self.assertIn(original_methodname, self.get_report( ['--proguard-mapping-file', proguard_mapping_file, '-i', testdata_file])) + + def test_show_art_frames(self): + art_frame_str = 'art::interpreter::DoCall' + options = ['-i', TestHelper.testdata_path('perf_with_interpreter_frames.data')] + report = self.get_report(options) + self.assertIn(art_frame_str, report) + report = self.get_report(options + ['--no-show-art-frames']) + self.assertNotIn(art_frame_str, report) diff --git a/simpleperf/scripts/test/report_html_test.py b/simpleperf/scripts/test/report_html_test.py index 0ae99e95..250ad033 100644 --- a/simpleperf/scripts/test/report_html_test.py +++ b/simpleperf/scripts/test/report_html_test.py @@ -84,6 +84,10 @@ class TestReportHtml(TestBase): self.assertIn(original_methodname, json.dumps(record_data)) def get_record_data(self, options: List[str]) -> Dict[str, Any]: + json_data = self.get_record_data_string(options) + return json.loads(json_data) + + def get_record_data_string(self, options: List[str]) -> str: args = ['report_html.py'] + options if TestHelper.ndk_path: args += ['--ndk_path', TestHelper.ndk_path] @@ -99,8 +103,7 @@ class TestReportHtml(TestBase): start_pos += 1 end_pos = data.find(end_str, start_pos) self.assertNotEqual(end_pos, -1) - json_data = data[start_pos:end_pos] - return json.loads(json_data) + return data[start_pos:end_pos] def test_add_source_code(self): """ Test --add_source_code option. """ @@ -243,3 +246,11 @@ class TestReportHtml(TestBase): self.assertIn(31881, threads) self.assertNotIn(31850, threads) os.unlink(filter_file.name) + + def test_show_art_frames(self): + art_frame_str = 'art::interpreter::DoCall' + options = ['-i', TestHelper.testdata_path('perf_with_interpreter_frames.data')] + report = self.get_record_data_string(options) + self.assertNotIn(art_frame_str, report) + report = self.get_record_data_string(options + ['--show-art-frames']) + self.assertIn(art_frame_str, report) diff --git a/simpleperf/scripts/test/report_lib_test.py b/simpleperf/scripts/test/report_lib_test.py index ba7ae466..1447c5ae 100644 --- a/simpleperf/scripts/test/report_lib_test.py +++ b/simpleperf/scripts/test/report_lib_test.py @@ -16,7 +16,7 @@ import os import tempfile -from typing import Set +from typing import List, Set from simpleperf_report_lib import ReportLib from . test_utils import TestBase, TestHelper @@ -275,34 +275,39 @@ class TestReportLib(TestBase): def test_set_sample_filter(self): """ Test using ReportLib.SetSampleFilter(). """ - def get_threads_for_filter(filter: str) -> Set[int]: + def get_threads_for_filter(filters: List[str]) -> Set[int]: self.report_lib.Close() self.report_lib = ReportLib() self.report_lib.SetRecordFile(TestHelper.testdata_path('perf_display_bitmaps.data')) - self.report_lib.SetSampleFilter(filter) + self.report_lib.SetSampleFilter(filters) threads = set() while self.report_lib.GetNextSample(): sample = self.report_lib.GetCurrentSample() threads.add(sample.tid) return threads - self.assertNotIn(31850, get_threads_for_filter('--exclude-pid 31850')) - self.assertIn(31850, get_threads_for_filter('--include-pid 31850')) - self.assertNotIn(31881, get_threads_for_filter('--exclude-tid 31881')) - self.assertIn(31881, get_threads_for_filter('--include-tid 31881')) + self.assertNotIn(31850, get_threads_for_filter(['--exclude-pid', '31850'])) + self.assertIn(31850, get_threads_for_filter(['--include-pid', '31850'])) + self.assertNotIn(31881, get_threads_for_filter(['--exclude-tid', '31881'])) + self.assertIn(31881, get_threads_for_filter(['--include-tid', '31881'])) self.assertNotIn(31881, get_threads_for_filter( - '--exclude-process-name com.example.android.displayingbitmaps')) + ['--exclude-process-name', 'com.example.android.displayingbitmaps'])) self.assertIn(31881, get_threads_for_filter( - '--include-process-name com.example.android.displayingbitmaps')) + ['--include-process-name', 'com.example.android.displayingbitmaps'])) self.assertNotIn(31850, get_threads_for_filter( - '--exclude-thread-name com.example.android.displayingbitmaps')) + ['--exclude-thread-name', 'com.example.android.displayingbitmaps'])) self.assertIn(31850, get_threads_for_filter( - '--include-thread-name com.example.android.displayingbitmaps')) + ['--include-thread-name', 'com.example.android.displayingbitmaps'])) + + # Check that thread name can have space. + self.assertNotIn(31856, get_threads_for_filter( + ['--exclude-thread-name', 'Jit thread pool'])) + self.assertIn(31856, get_threads_for_filter(['--include-thread-name', 'Jit thread pool'])) with tempfile.NamedTemporaryFile('w', delete=False) as filter_file: filter_file.write('GLOBAL_BEGIN 684943449406175\nGLOBAL_END 684943449406176') filter_file.flush() - threads = get_threads_for_filter('--filter-file ' + filter_file.name) + threads = get_threads_for_filter(['--filter-file', filter_file.name]) self.assertIn(31881, threads) self.assertNotIn(31850, threads) os.unlink(filter_file.name) diff --git a/simpleperf/scripts/test/report_sample_test.py b/simpleperf/scripts/test/report_sample_test.py index 4b09f82f..5b6681cb 100644 --- a/simpleperf/scripts/test/report_sample_test.py +++ b/simpleperf/scripts/test/report_sample_test.py @@ -17,32 +17,27 @@ import os import re import tempfile -from typing import Set +from typing import List, Optional, Set from . test_utils import TestBase, TestHelper class TestReportSample(TestBase): + def get_record_data_string(self, record_file: str, options: Optional[List[str]] = None): + args = ['report_sample.py', '-i', TestHelper.testdata_path(record_file)] + if options: + args += options + report = self.run_cmd(args, return_output=True) + return report.replace('\r', '') def test_no_flags(self): - got = self.run_cmd( - ['report_sample.py', - '-i', - TestHelper.testdata_path('perf_display_bitmaps.data')], - return_output=True) - got = got.replace('\r', '') + got = self.get_record_data_string('perf_display_bitmaps.data') with open(TestHelper.testdata_path('perf_display_bitmaps.perf-script')) as f: want = f.read() self.assertEqual(got, want) def test_comm_filter_to_renderthread(self): - got = self.run_cmd( - ['report_sample.py', - '-i', - TestHelper.testdata_path('perf_display_bitmaps.data'), - '--comm', 'RenderThread'], - return_output=True) - got = got.replace('\r', '') + got = self.get_record_data_string('perf_display_bitmaps.data', ['--comm', 'RenderThread']) self.assertIn('RenderThread', got) self.assertNotIn('com.example.android.displayingbitmaps', got) @@ -51,13 +46,8 @@ class TestReportSample(TestBase): self.assertEqual(got, want) def test_comm_filter_to_ui_thread(self): - got = self.run_cmd( - ['report_sample.py', - '-i', - TestHelper.testdata_path('perf_display_bitmaps.data'), - '--comm', 'com.example.android.displayingbitmaps'], - return_output=True) - got = got.replace('\r', '') + got = self.get_record_data_string('perf_display_bitmaps.data', [ + '--comm', 'com.example.android.displayingbitmaps']) self.assertIn('com.example.android.displayingbitmaps', got) self.assertNotIn('RenderThread', got) with open(TestHelper.testdata_path('perf_display_bitmaps.UiThread.perf-script')) as f: @@ -65,29 +55,20 @@ class TestReportSample(TestBase): self.assertEqual(got, want) def test_header(self): - got = self.run_cmd( - ['report_sample.py', - '-i', - TestHelper.testdata_path('perf_display_bitmaps.data'), - '--header'], - return_output=True) - got = got.replace('\r', '') + got = self.get_record_data_string('perf_display_bitmaps.data', ['--header']) with open(TestHelper.testdata_path('perf_display_bitmaps.header.perf-script')) as f: want = f.read() self.assertEqual(got, want) def test_trace_offcpu(self): - got = self.run_cmd( - ['report_sample.py', '-i', TestHelper.testdata_path('perf_with_trace_offcpu_v2.data'), - '--trace-offcpu', 'on-cpu'], return_output=True) + got = self.get_record_data_string('perf_with_trace_offcpu_v2.data', [ + '--trace-offcpu', 'on-cpu']) self.assertIn('cpu-clock:u', got) self.assertNotIn('sched:sched_switch', got) def test_sample_filters(self): def get_threads_for_filter(filter: str) -> Set[int]: - report = self.run_cmd( - ['report_sample.py', '-i', TestHelper.testdata_path('perf_display_bitmaps.data')] + - filter.split(), return_output=True) + report = self.get_record_data_string('perf_display_bitmaps.data', filter.split()) pattern = re.compile(r'\s+31850/(\d+)\s+') threads = set() for m in re.finditer(pattern, report): @@ -116,3 +97,11 @@ class TestReportSample(TestBase): self.assertIn(31881, threads) self.assertNotIn(31850, threads) os.unlink(filter_file.name) + + def test_show_art_frames(self): + art_frame_str = 'art::interpreter::DoCall' + report = self.get_record_data_string('perf_with_interpreter_frames.data') + self.assertNotIn(art_frame_str, report) + report = self.get_record_data_string( + 'perf_with_interpreter_frames.data', ['--show-art-frames']) + self.assertIn(art_frame_str, report) diff --git a/simpleperf/scripts/test/stackcollapse_test.py b/simpleperf/scripts/test/stackcollapse_test.py index fee39222..4b9cdbed 100644 --- a/simpleperf/scripts/test/stackcollapse_test.py +++ b/simpleperf/scripts/test/stackcollapse_test.py @@ -19,90 +19,57 @@ import os from pathlib import Path import re import tempfile -from typing import Set +from typing import List, Optional, Set from . test_utils import TestBase, TestHelper class TestStackCollapse(TestBase): + def get_report(self, testdata_file: str, options: Optional[List[str]] = None) -> str: + args = ['stackcollapse.py', '-i', TestHelper.testdata_path(testdata_file)] + if options: + args.extend(options) + report = self.run_cmd(args, return_output=True) + return report.replace('\r', '') def test_jit_annotations(self): - got = self.run_cmd([ - 'stackcollapse.py', - '-i', TestHelper.testdata_path('perf_with_jit_symbol.data'), - '--jit', - ], return_output=True) - got = got.replace('\r', '') + got = self.get_report('perf_with_jit_symbol.data', ['--jit']) golden_path = TestHelper.testdata_path('perf_with_jit_symbol.foldedstack') self.assertEqual(got, Path(golden_path).read_text()) def test_kernel_annotations(self): - got = self.run_cmd([ - 'stackcollapse.py', - '-i', TestHelper.testdata_path('perf_with_jit_symbol.data'), - '--kernel', - ], return_output=True) - got = got.replace('\r', '') + got = self.get_report('perf_with_jit_symbol.data', ['--kernel']) golden_path = TestHelper.testdata_path('perf_with_jit_symbol.foldedstack_with_kernel') self.assertEqual(got, Path(golden_path).read_text()) def test_with_pid(self): - got = self.run_cmd([ - 'stackcollapse.py', - '-i', TestHelper.testdata_path('perf_with_jit_symbol.data'), - '--jit', - '--pid', - ], return_output=True) - got = got.replace('\r', '') + got = self.get_report('perf_with_jit_symbol.data', ['--jit', '--pid']) golden_path = TestHelper.testdata_path('perf_with_jit_symbol.foldedstack_with_pid') self.assertEqual(got, Path(golden_path).read_text()) def test_with_tid(self): - got = self.run_cmd([ - 'stackcollapse.py', - '-i', TestHelper.testdata_path('perf_with_jit_symbol.data'), - '--jit', - '--tid', - ], return_output=True) - got = got.replace('\r', '') + got = self.get_report('perf_with_jit_symbol.data', ['--jit', '--tid']) golden_path = TestHelper.testdata_path('perf_with_jit_symbol.foldedstack_with_tid') self.assertEqual(got, Path(golden_path).read_text()) def test_two_event_types_chooses_first(self): - got = self.run_cmd([ - 'stackcollapse.py', - '-i', TestHelper.testdata_path('perf_with_two_event_types.data'), - ], return_output=True) - got = got.replace('\r', '') + got = self.get_report('perf_with_two_event_types.data') golden_path = TestHelper.testdata_path('perf_with_two_event_types.foldedstack') self.assertEqual(got, Path(golden_path).read_text()) def test_two_event_types_chooses_with_event_filter(self): - got = self.run_cmd([ - 'stackcollapse.py', - '-i', TestHelper.testdata_path('perf_with_two_event_types.data'), - '--event-filter', 'cpu-clock', - ], return_output=True) - got = got.replace('\r', '') + got = self.get_report('perf_with_two_event_types.data', ['--event-filter', 'cpu-clock']) golden_path = TestHelper.testdata_path('perf_with_two_event_types.foldedstack_cpu_clock') self.assertEqual(got, Path(golden_path).read_text()) def test_unknown_symbol_addrs(self): - got = self.run_cmd([ - 'stackcollapse.py', - '-i', TestHelper.testdata_path('perf_with_jit_symbol.data'), - '--addrs', - ], return_output=True) - got = got.replace('\r', '') + got = self.get_report('perf_with_jit_symbol.data', ['--addrs']) golden_path = TestHelper.testdata_path('perf_with_jit_symbol.foldedstack_addrs') self.assertEqual(got, Path(golden_path).read_text()) def test_sample_filters(self): def get_threads_for_filter(filter: str) -> Set[int]: - report = self.run_cmd( - ['stackcollapse.py', '-i', TestHelper.testdata_path('perf_display_bitmaps.data'), - '--tid'] + filter.split(), - return_output=True) + report = self.get_report('perf_display_bitmaps.data', ['--tid'] + filter.split()) pattern = re.compile(r'-31850/(\d+);') threads = set() for m in re.finditer(pattern, report): @@ -129,3 +96,10 @@ class TestStackCollapse(TestBase): self.assertIn(31881, threads) self.assertNotIn(31850, threads) os.unlink(filter_file.name) + + def test_show_art_frames(self): + art_frame_str = 'art::interpreter::DoCall' + report = self.get_report('perf_with_interpreter_frames.data') + self.assertNotIn(art_frame_str, report) + report = self.get_report('perf_with_interpreter_frames.data', ['--show-art-frames']) + self.assertIn(art_frame_str, report) diff --git a/toolchain-extras/Android.bp b/toolchain-extras/Android.bp index 220f3e34..d48f081f 100644 --- a/toolchain-extras/Android.bp +++ b/toolchain-extras/Android.bp @@ -53,18 +53,18 @@ cc_defaults { sanitize: { blocklist: "libprofile_clang_extras_blocklist.txt", }, -} - -cc_library_static { - name: "libprofile-clang-extras", - defaults: ["libprofile-clang-defaults"], - native_bridge_supported: true, vendor_available: true, product_available: true, vndk: { enabled: true, }, +} + +cc_defaults { + name: "libprofile-clang-platform-defaults", + defaults: ["libprofile-clang-defaults"], + ramdisk_available: true, vendor_ramdisk_available: true, recovery_available: true, @@ -74,36 +74,70 @@ cc_library_static { header_libs: ["libc_headers"], } +// ----------------------------------------------------------------------------- +// libprofile-clang libraries for regular coverage. They also install a signal +// handler that writes coverage data to disk. +// ----------------------------------------------------------------------------- + +cc_library_static { + name: "libprofile-clang-extras", + defaults: ["libprofile-clang-platform-defaults"], +} + cc_library_static { name: "libprofile-clang-extras_ndk", defaults: ["libprofile-clang-defaults"], - native_bridge_supported: true, - vendor_available: true, - product_available: true, - vndk: { - enabled: true, - }, sdk_version: "minimum", } cc_library_static { name: "libprofile-clang-extras_cfi_support", - defaults: ["libprofile-clang-defaults"], + defaults: ["libprofile-clang-platform-defaults"], - native_bridge_supported: true, - vendor_available: true, - product_available: true, - vndk: { - enabled: true, + sanitize: { + cfi: true, + config: { + cfi_assembly_support: true, + }, }, - ramdisk_available: true, - vendor_ramdisk_available: true, - recovery_available: true, +} + +// ----------------------------------------------------------------------------- +// libprofile-clang libraries for continuous coverage. They install a no-op +// signal handler. +// ----------------------------------------------------------------------------- + +cc_defaults { + name: "profile-extras-continuous-mode", + cflags: ["-D__CONTINUOUS_COVERAGE_MODE__"], +} + +cc_library_static { + name: "libprofile-clang-extras_continuous", + defaults: [ + "libprofile-clang-platform-defaults", + "profile-extras-continuous-mode", + ], +} + +cc_library_static { + name: "libprofile-clang-extras_ndk_continuous", + defaults: [ + "libprofile-clang-platform-defaults", + "profile-extras-continuous-mode", + ], + + sdk_version: "minimum", +} + +cc_library_static { + name: "libprofile-clang-extras_cfi_support_continuous", + defaults: [ + "libprofile-clang-platform-defaults", + "profile-extras-continuous-mode", + ], - stl: "none", - system_shared_libs: [], - header_libs: ["libc_headers"], sanitize: { cfi: true, config: { @@ -112,6 +146,9 @@ cc_library_static { }, } +// ----------------------------------------------------------------------------- +// tests +// ----------------------------------------------------------------------------- cc_test { name: "libprofile-extras-test", srcs: [ diff --git a/toolchain-extras/profile-clang-extras.cpp b/toolchain-extras/profile-clang-extras.cpp index c45f9b39..6a7766ef 100644 --- a/toolchain-extras/profile-clang-extras.cpp +++ b/toolchain-extras/profile-clang-extras.cpp @@ -24,10 +24,16 @@ extern "C" { static sighandler_t chained_signal_handler = SIG_ERR; +#ifndef __CONTINUOUS_COVERAGE_MODE__ int __llvm_profile_write_file(void); +#endif // __CONTINUOUS_COVERAGE_MODE__ static void llvm_signal_handler(__unused int signum) { + // TODO(pirama) Only disable __llvm_profile_write_file call to begin with. + // After continuous mode is stable, stop registering the signal handler. +#ifndef __CONTINUOUS_COVERAGE_MODE__ __llvm_profile_write_file(); +#endif // __CONTINUOUS_COVERAGE_MODE__ if (chained_signal_handler != SIG_ERR && chained_signal_handler != SIG_IGN && chained_signal_handler != SIG_DFL) { diff --git a/verity/Android.bp b/verity/Android.bp index fc2b827c..ef09528c 100644 --- a/verity/Android.bp +++ b/verity/Android.bp @@ -154,13 +154,8 @@ python_binary_host { name: "build_verity_metadata", srcs: ["build_verity_metadata.py"], version: { - py2: { - enabled: true, - embedded_launcher: true, - }, py3: { - enabled: false, - embedded_launcher: false, + embedded_launcher: true, }, }, } diff --git a/verity/build_verity_metadata.py b/verity/build_verity_metadata.py index 5a7d7d27..a428f270 100644 --- a/verity/build_verity_metadata.py +++ b/verity/build_verity_metadata.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#!/usr/bin/env python3 # # Copyright (C) 2013 The Android Open Source Project # @@ -29,9 +29,9 @@ BLOCK_SIZE = 4096 METADATA_SIZE = BLOCK_SIZE * 8 def run(cmd): - p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, text=True) output, _ = p.communicate() - print output + print(output) if p.returncode: exit(-1) @@ -43,12 +43,12 @@ def build_metadata_block(verity_table, signature, verity_disable=False): magic = MAGIC_DISABLE if verity_disable else MAGIC_NUMBER block = struct.pack("II256sI", magic, VERSION, signature, table_len) block += verity_table - block = block.ljust(METADATA_SIZE, '\x00') + block = block.ljust(METADATA_SIZE, b'\x00') return block def sign_verity_table(table, signer_path, key_path, signer_args=None): - with tempfile.NamedTemporaryFile(suffix='.table') as table_file: - with tempfile.NamedTemporaryFile(suffix='.sig') as signature_file: + with tempfile.NamedTemporaryFile(mode='wb', suffix='.table') as table_file: + with tempfile.NamedTemporaryFile(mode='rb', suffix='.sig') as signature_file: table_file.write(table) table_file.flush() if signer_args is None: @@ -56,7 +56,7 @@ def sign_verity_table(table, signer_path, key_path, signer_args=None): else: args_list = shlex.split(signer_args) cmd = [signer_path] + args_list + [table_file.name, key_path, signature_file.name] - print cmd + print(cmd) run(cmd) return signature_file.read() @@ -70,7 +70,7 @@ def build_verity_table(block_device, data_blocks, root_hash, salt): data_blocks, root_hash, salt) - return table + return table.encode() def build_verity_metadata(data_blocks, metadata_image, root_hash, salt, block_device, signer_path, signing_key, signer_args=None, @@ -109,9 +109,9 @@ if __name__ == "__main__": args = parser.parse_args() if args.dest == 'size': - print get_verity_metadata_size(args.partition_size) + print(get_verity_metadata_size(args.partition_size)) else: - build_verity_metadata(args.blocks / 4096, args.metadata_image, + build_verity_metadata(args.blocks // 4096, args.metadata_image, args.root_hash, args.salt, args.block_device, args.signer_path, args.signing_key, args.signer_args, args.verity_disable) diff --git a/verity/fec/Android.bp b/verity/fec/Android.bp index 9ca45600..ede5fd19 100644 --- a/verity/fec/Android.bp +++ b/verity/fec/Android.bp @@ -25,6 +25,11 @@ cc_binary_host { misc_undefined: ["integer"], }, }, + linux_musl_x86_64: { + sanitize: { + misc_undefined: ["integer"], + }, + }, linux: { static_libs: [ "libavb", |