diff options
author | Android Build Coastguard Worker <android-build-coastguard-worker@google.com> | 2023-04-20 16:51:01 +0000 |
---|---|---|
committer | Gerrit Code Review <noreply-gerritcodereview@google.com> | 2023-04-20 16:51:01 +0000 |
commit | f0d2b53d1d887b61e635e81db72a8da23fbcb1fd (patch) | |
tree | 61b26890f41f801dff001c6f05fc2be9d3270037 | |
parent | 26a85afddc6972ca3f14f1ef66799bab7df9779a (diff) | |
parent | d3deec2c3111defe7bcbd0363095ab6ac749a03c (diff) | |
download | extras-platform-tools-34.0.3.tar.gz |
Merge "Snap for 9979206 from 905c121a888f4a22169c30c678b8e58c746ab92a to sdk-release" into sdk-releaseplatform-tools-34.0.3
71 files changed, 1205 insertions, 373 deletions
diff --git a/ext4_utils/Android.bp b/ext4_utils/Android.bp index ba2c8ac0..b28e84f3 100644 --- a/ext4_utils/Android.bp +++ b/ext4_utils/Android.bp @@ -32,6 +32,7 @@ cc_library { cflags: [ "-Werror", "-fno-strict-aliasing", + "-D_FILE_OFFSET_BITS=64", ], export_include_dirs: ["include"], shared_libs: [ diff --git a/ext4_utils/ext4_utils.cpp b/ext4_utils/ext4_utils.cpp index 5fce61bc..dde75903 100644 --- a/ext4_utils/ext4_utils.cpp +++ b/ext4_utils/ext4_utils.cpp @@ -83,12 +83,9 @@ int ext4_bg_has_super_block(int bg) { /* Function to read the primary superblock */ void read_sb(int fd, struct ext4_super_block* sb) { - off64_t ret; + if (lseek(fd, 1024, SEEK_SET) < 0) critical_error_errno("failed to seek to superblock"); - ret = lseek64(fd, 1024, SEEK_SET); - if (ret < 0) critical_error_errno("failed to seek to superblock"); - - ret = read(fd, sb, sizeof(*sb)); + ssize_t ret = read(fd, sb, sizeof(*sb)); if (ret < 0) critical_error_errno("failed to read superblock"); if (ret != sizeof(*sb)) critical_error("failed to read all of superblock"); } @@ -277,17 +274,17 @@ static void read_block_group_descriptors(int fd) { } int read_ext(int fd, int verbose) { - off64_t ret; + off_t ret; struct ext4_super_block sb; read_sb(fd, &sb); ext4_parse_sb_info(&sb); - ret = lseek64(fd, info.len, SEEK_SET); + ret = lseek(fd, info.len, SEEK_SET); if (ret < 0) critical_error_errno("failed to seek to end of input image"); - ret = lseek64(fd, info.block_size * (aux_info.first_data_block + 1), SEEK_SET); + ret = lseek(fd, info.block_size * (aux_info.first_data_block + 1), SEEK_SET); if (ret < 0) critical_error_errno("failed to seek to block group descriptors"); read_block_group_descriptors(fd); diff --git a/ext4_utils/include/ext4_utils/ext4_utils.h b/ext4_utils/include/ext4_utils/ext4_utils.h index 48f3ee78..d6bef68d 100644 --- a/ext4_utils/include/ext4_utils/ext4_utils.h +++ b/ext4_utils/include/ext4_utils/ext4_utils.h @@ -21,11 +21,6 @@ extern "C" { #endif -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif -#define _FILE_OFFSET_BITS 64 -#define _LARGEFILE64_SOURCE 1 #include <sys/types.h> #include <unistd.h> @@ -38,13 +33,6 @@ extern "C" { #include <string.h> #include <sys/types.h> -#if defined(__APPLE__) && defined(__MACH__) -#define lseek64 lseek -#define ftruncate64 ftruncate -#define mmap64 mmap -#define off64_t off_t -#endif - #include "ext4_sb.h" extern int force; diff --git a/f2fs_utils/mkf2fsuserimg.sh b/f2fs_utils/mkf2fsuserimg.sh index 59f9eea5..e95a0c51 100755 --- a/f2fs_utils/mkf2fsuserimg.sh +++ b/f2fs_utils/mkf2fsuserimg.sh @@ -163,13 +163,14 @@ function _build() SLOAD_F2FS_CMD="sload_f2fs $SLOAD_OPTS $OUTPUT_FILE" echo $SLOAD_F2FS_CMD - MB_SIZE=`$SLOAD_F2FS_CMD | grep "Max image size" | awk '{print $5}'` + SLOAD_LOG=`$SLOAD_F2FS_CMD` # allow 1: Filesystem errors corrected ret=$? if [ $ret -ne 0 ] && [ $ret -ne 1 ]; then rm -f $OUTPUT_FILE exit 4 fi + MB_SIZE=`echo "$SLOAD_LOG" | grep "Max image size" | awk '{print $5}'` SIZE=$(((MB_SIZE + 6) * 1024 * 1024)) } diff --git a/libfscrypt/tests/fscrypt_test.cpp b/libfscrypt/tests/fscrypt_test.cpp index 70eb1780..0cd79950 100644 --- a/libfscrypt/tests/fscrypt_test.cpp +++ b/libfscrypt/tests/fscrypt_test.cpp @@ -176,6 +176,14 @@ TEST(fscrypt, ParseOptions) { EXPECT_FALSE(ParseOptionsForApiLevel(30, "aes-256-xts:aes-256-cts:v2:foo", &dummy_options)); EXPECT_FALSE(ParseOptionsForApiLevel(30, "aes-256-xts:aes-256-cts:blah", &dummy_options)); EXPECT_FALSE(ParseOptionsForApiLevel(30, "aes-256-xts:aes-256-cts:vblah", &dummy_options)); + + { + TEST_STRING(34, ":aes-256-hctr2", "aes-256-xts:aes-256-hctr2:v2"); + EXPECT_EQ(2, options.version); + EXPECT_EQ(FSCRYPT_MODE_AES_256_XTS, options.contents_mode); + EXPECT_EQ(FSCRYPT_MODE_AES_256_HCTR2, options.filenames_mode); + EXPECT_EQ(FSCRYPT_POLICY_FLAGS_PAD_16, options.flags); + } } TEST(fscrypt, ComparePolicies) { diff --git a/memory_replay/Android.bp b/memory_replay/Android.bp index bdcde1b2..c7de1feb 100644 --- a/memory_replay/Android.bp +++ b/memory_replay/Android.bp @@ -75,7 +75,6 @@ cc_defaults { static_libs: [ "liballoc_parser", - "libasync_safe", ], } diff --git a/memory_replay/NativeInfo.cpp b/memory_replay/NativeInfo.cpp index 3439a29d..8493b682 100644 --- a/memory_replay/NativeInfo.cpp +++ b/memory_replay/NativeInfo.cpp @@ -27,23 +27,12 @@ #include <unistd.h> #include <android-base/unique_fd.h> -#include <async_safe/log.h> #include "NativeInfo.h" -void NativePrintf(const char* fmt, ...) { - va_list args; - va_start(args, fmt); - char buffer[512]; - int buffer_len = async_safe_format_buffer_va_list(buffer, sizeof(buffer), fmt, args); - va_end(args); - - (void)write(STDOUT_FILENO, buffer, buffer_len); -} - void NativeFormatFloat(char* buffer, size_t buffer_len, uint64_t value, uint64_t divisor) { uint64_t hundreds = ((((value % divisor) * 1000) / divisor) + 5) / 10; - async_safe_format_buffer(buffer, buffer_len, "%" PRIu64 ".%02" PRIu64, value / divisor, hundreds); + snprintf(buffer, buffer_len, "%" PRIu64 ".%02" PRIu64, value / divisor, hundreds); } // This function is not re-entrant since it uses a static buffer for @@ -114,7 +103,7 @@ void NativePrintInfo(const char* preamble) { // Avoid any allocations, so use special non-allocating printfs. char buffer[256]; NativeFormatFloat(buffer, sizeof(buffer), rss_bytes, 1024 * 1024); - NativePrintf("%sNative RSS: %zu bytes %sMB\n", preamble, rss_bytes, buffer); + dprintf(STDOUT_FILENO, "%sNative RSS: %zu bytes %sMB\n", preamble, rss_bytes, buffer); NativeFormatFloat(buffer, sizeof(buffer), va_bytes, 1024 * 1024); - NativePrintf("%sNative VA Space: %zu bytes %sMB\n", preamble, va_bytes, buffer); + dprintf(STDOUT_FILENO, "%sNative VA Space: %zu bytes %sMB\n", preamble, va_bytes, buffer); } diff --git a/memory_replay/NativeInfo.h b/memory_replay/NativeInfo.h index c91eec29..a33db027 100644 --- a/memory_replay/NativeInfo.h +++ b/memory_replay/NativeInfo.h @@ -20,8 +20,5 @@ void NativeGetInfo(int smaps_fd, size_t* rss_bytes, size_t* va_bytes); void NativePrintInfo(const char* preamble); -// Does not support any floating point specifiers. -void NativePrintf(const char* fmt, ...) __printflike(1, 2); - // Fill buffer as if %0.2f was chosen for value / divisor. void NativeFormatFloat(char* buffer, size_t buffer_len, uint64_t value, uint64_t divisor); diff --git a/memory_replay/main.cpp b/memory_replay/main.cpp index e610305e..2dd66bb3 100644 --- a/memory_replay/main.cpp +++ b/memory_replay/main.cpp @@ -78,15 +78,15 @@ static void ProcessDump(const AllocEntry* entries, size_t num_entries, size_t ma Pointers pointers(max_allocs); Threads threads(&pointers, max_threads); - NativePrintf("Maximum threads available: %zu\n", threads.max_threads()); - NativePrintf("Maximum allocations in dump: %zu\n", max_allocs); - NativePrintf("Total pointers available: %zu\n\n", pointers.max_pointers()); + dprintf(STDOUT_FILENO, "Maximum threads available: %zu\n", threads.max_threads()); + dprintf(STDOUT_FILENO, "Maximum allocations in dump: %zu\n", max_allocs); + dprintf(STDOUT_FILENO, "Total pointers available: %zu\n\n", pointers.max_pointers()); NativePrintInfo("Initial "); for (size_t i = 0; i < num_entries; i++) { if (((i + 1) % 100000) == 0) { - NativePrintf(" At line %zu:\n", i + 1); + dprintf(STDOUT_FILENO, " At line %zu:\n", i + 1); NativePrintInfo(" "); } const AllocEntry& entry = entries[i]; @@ -139,7 +139,7 @@ static void ProcessDump(const AllocEntry* entries, size_t num_entries, size_t ma char buffer[256]; uint64_t total_nsecs = threads.total_time_nsecs(); NativeFormatFloat(buffer, sizeof(buffer), total_nsecs, 1000000000); - NativePrintf("Total Allocation/Free Time: %" PRIu64 "ns %ss\n", total_nsecs, buffer); + dprintf(STDOUT_FILENO, "Total Allocation/Free Time: %" PRIu64 "ns %ss\n", total_nsecs, buffer); } int main(int argc, char** argv) { @@ -161,13 +161,13 @@ int main(int argc, char** argv) { } #if defined(__LP64__) - NativePrintf("64 bit environment.\n"); + dprintf(STDOUT_FILENO, "64 bit environment.\n"); #else - NativePrintf("32 bit environment.\n"); + dprintf(STDOUT_FILENO, "32 bit environment.\n"); #endif #if defined(__BIONIC__) - NativePrintf("Setting decay time to 1\n"); + dprintf(STDOUT_FILENO, "Setting decay time to 1\n"); mallopt(M_DECAY_TIME, 1); #endif @@ -180,7 +180,7 @@ int main(int argc, char** argv) { size_t num_entries; GetUnwindInfo(argv[1], &entries, &num_entries); - NativePrintf("Processing: %s\n", argv[1]); + dprintf(STDOUT_FILENO, "Processing: %s\n", argv[1]); ProcessDump(entries, num_entries, max_threads); diff --git a/mtectrl/mtectrl.rc b/mtectrl/mtectrl.rc index 699b56ec..a474cdcf 100644 --- a/mtectrl/mtectrl.rc +++ b/mtectrl/mtectrl.rc @@ -13,10 +13,10 @@ # limitations under the License. on property:arm64.memtag.bootctl=* - exec -- /system/bin/mtectrl ${arm64.memtag.bootctl:-none} ${persist.device_config.memory_safety_native_boot.bootloader_override:-default} + exec -- /system/bin/mtectrl ${arm64.memtag.bootctl:-none} ${persist.device_config.runtime_native_boot.bootloader_override:-default} -on property:persist.device_config.memory_safety_native_boot.bootloader_override=* - exec -- /system/bin/mtectrl ${arm64.memtag.bootctl:-none} ${persist.device_config.memory_safety_native_boot.bootloader_override:-default} +on property:persist.device_config.runtime_native_boot.bootloader_override=* + exec -- /system/bin/mtectrl ${arm64.memtag.bootctl:-none} ${persist.device_config.runtime_native_boot.bootloader_override:-default} # adbd gets initialized in init, so run before that. this makes sure that the # user does not change the value before we initialize it @@ -24,4 +24,4 @@ on early-init && property:ro.arm64.memtag.bootctl_supported=1 exec -- /system/bin/mtectrl -s arm64.memtag.bootctl on shutdown && property:ro.arm64.memtag.bootctl_supported=1 - exec -- /system/bin/mtectrl ${arm64.memtag.bootctl:-none} ${persist.device_config.memory_safety_native_boot.bootloader_override:-default} + exec -- /system/bin/mtectrl ${arm64.memtag.bootctl:-none} ${persist.device_config.runtime_native_boot.bootloader_override:-default} diff --git a/partition_tools/dynamic_partitions_device_info.proto b/partition_tools/dynamic_partitions_device_info.proto index e53b40e2..8800dac7 100644 --- a/partition_tools/dynamic_partitions_device_info.proto +++ b/partition_tools/dynamic_partitions_device_info.proto @@ -25,7 +25,7 @@ message DynamicPartitionsDeviceInfoProto { bool enabled = 1; bool retrofit = 2; - // Next: 7 + // Next: 8 message Partition { string name = 1; string group_name = 2 [json_name = "group_name"]; @@ -36,6 +36,8 @@ message DynamicPartitionsDeviceInfoProto { uint64 fs_size = 5 [json_name = "fs_size"]; /** Used space of the filesystem. */ uint64 fs_used = 6 [json_name = "fs_used"]; + /** Name of the filesystem. */ + string fs_type = 7 [json_name = "fs_type"]; } repeated Partition partitions = 3; diff --git a/partition_tools/lpdump.cc b/partition_tools/lpdump.cc index 047b5ee0..97682940 100644 --- a/partition_tools/lpdump.cc +++ b/partition_tools/lpdump.cc @@ -229,6 +229,13 @@ static bool MergeFsUsage(DynamicPartitionsDeviceInfoProto* proto, partition_proto->set_is_dynamic(false); } partition_proto->set_fs_size((uint64_t)vst.f_blocks * vst.f_frsize); + + if (!entry.fs_type.empty()) { + partition_proto->set_fs_type(entry.fs_type); + } else { + partition_proto->set_fs_type("UNKNOWN"); + } + if (vst.f_bavail <= vst.f_blocks) { partition_proto->set_fs_used((uint64_t)(vst.f_blocks - vst.f_bavail) * vst.f_frsize); } diff --git a/partition_tools/lpunpack.cc b/partition_tools/lpunpack.cc index 1f870c5d..b215c58e 100644 --- a/partition_tools/lpunpack.cc +++ b/partition_tools/lpunpack.cc @@ -34,11 +34,12 @@ using namespace android::fs_mgr; using android::base::unique_fd; +using android::base::borrowed_fd; using SparsePtr = std::unique_ptr<sparse_file, decltype(&sparse_file_destroy)>; class ImageExtractor final { public: - ImageExtractor(unique_fd&& image_fd, std::unique_ptr<LpMetadata>&& metadata, + ImageExtractor(std::vector<unique_fd>&& image_fds, std::unique_ptr<LpMetadata>&& metadata, std::unordered_set<std::string>&& partitions, const std::string& output_dir); bool Extract(); @@ -48,7 +49,7 @@ class ImageExtractor final { bool ExtractPartition(const LpMetadataPartition* partition); bool ExtractExtent(const LpMetadataExtent& extent, int output_fd); - unique_fd image_fd_; + std::vector<unique_fd> image_fds_; std::unique_ptr<LpMetadata> metadata_; std::unordered_set<std::string> partitions_; std::string output_dir_; @@ -59,16 +60,15 @@ class ImageExtractor final { // file format. class SparseWriter final { public: - SparseWriter(int output_fd, int image_fd, uint32_t block_size); + SparseWriter(borrowed_fd output_fd, uint32_t block_size); - bool WriteExtent(const LpMetadataExtent& extent); + bool WriteExtent(borrowed_fd image_fd, const LpMetadataExtent& extent); bool Finish(); private: bool WriteBlock(const uint8_t* data); - int output_fd_; - int image_fd_; + borrowed_fd output_fd_; uint32_t block_size_; off_t hole_size_ = 0; }; @@ -81,7 +81,14 @@ static int usage(int /* argc */, char* argv[]) { "Usage:\n" " %s [options...] SUPER_IMAGE [OUTPUT_DIR]\n" "\n" + "The SUPER_IMAGE argument is mandatory and expected to contain\n" + "the metadata. Additional super images are referenced with '-i' as needed to extract\n" + "the desired partition[s].\n" + "Default OUTPUT_DIR is '.'.\n" + "\n" "Options:\n" + " -i, --image=IMAGE Use the given file as an additional super image.\n" + " This can be specified multiple times.\n" " -p, --partition=NAME Extract the named partition. This can\n" " be specified multiple times.\n" " -S, --slot=NUM Slot number (default is 0).\n", @@ -92,6 +99,7 @@ static int usage(int /* argc */, char* argv[]) { int main(int argc, char* argv[]) { // clang-format off struct option options[] = { + { "image", required_argument, nullptr, 'i' }, { "partition", required_argument, nullptr, 'p' }, { "slot", required_argument, nullptr, 'S' }, { nullptr, 0, nullptr, 0 }, @@ -100,6 +108,7 @@ int main(int argc, char* argv[]) { uint32_t slot_num = 0; std::unordered_set<std::string> partitions; + std::vector<std::string> image_files; int rv, index; while ((rv = getopt_long_only(argc, argv, "+p:sh", options, &index)) != -1) { @@ -116,6 +125,9 @@ int main(int argc, char* argv[]) { return usage(argc, argv); } break; + case 'i': + image_files.push_back(optarg); + break; case 'p': partitions.emplace(optarg); break; @@ -126,56 +138,66 @@ int main(int argc, char* argv[]) { std::cerr << "Missing super image argument.\n"; return usage(argc, argv); } - std::string super_path = argv[optind++]; + image_files.emplace(image_files.begin(), argv[optind++]); std::string output_dir = "."; if (optind + 1 <= argc) { output_dir = argv[optind++]; } - if (optind < argc) { - std::cerr << "Unrecognized command-line arguments.\n"; - return usage(argc, argv); - } + std::unique_ptr<LpMetadata> metadata; + std::vector<unique_fd> fds; - // Done reading arguments; open super.img. PartitionOpener will decorate - // relative paths with /dev/block/by-name, so get an absolute path here. - std::string abs_super_path; - if (!android::base::Realpath(super_path, &abs_super_path)) { - std::cerr << "realpath failed: " << super_path << ": " << strerror(errno) << "\n"; - return EX_OSERR; - } + for (std::size_t index = 0; index < image_files.size(); ++index) { + std::string super_path = image_files[index]; - unique_fd fd(open(super_path.c_str(), O_RDONLY | O_CLOEXEC)); - if (fd < 0) { - std::cerr << "open failed: " << abs_super_path << ": " << strerror(errno) << "\n"; - return EX_OSERR; - } + // Done reading arguments; open super.img. PartitionOpener will decorate + // relative paths with /dev/block/by-name, so get an absolute path here. + std::string abs_super_path; + if (!android::base::Realpath(super_path, &abs_super_path)) { + std::cerr << "realpath failed: " << super_path << ": " << strerror(errno) << "\n"; + return EX_OSERR; + } + + unique_fd fd(open(super_path.c_str(), O_RDONLY | O_CLOEXEC)); + if (fd < 0) { + std::cerr << "open failed: " << abs_super_path << ": " << strerror(errno) << "\n"; + return EX_OSERR; + } - auto metadata = ReadMetadata(abs_super_path, slot_num); - if (!metadata) { SparsePtr ptr(sparse_file_import(fd, false, false), sparse_file_destroy); if (ptr) { - std::cerr << "This image appears to be a sparse image. It must be " - "unsparsed to be" - << " unpacked.\n"; + std::cerr << "The image file '" + << super_path + << "' appears to be a sparse image. It must be unsparsed to be unpacked.\n"; return EX_USAGE; } - std::cerr << "Image does not appear to be in super-partition format.\n"; - return EX_USAGE; + + if (!metadata) { + metadata = ReadMetadata(abs_super_path, slot_num); + if (!metadata) { + std::cerr << "Could not read metadata from the super image file '" + << super_path + << "'.\n"; + return EX_USAGE; + } + } + + fds.emplace_back(std::move(fd)); } - ImageExtractor extractor(std::move(fd), std::move(metadata), std::move(partitions), output_dir); + // Now do actual extraction. + ImageExtractor extractor(std::move(fds), std::move(metadata), std::move(partitions), output_dir); if (!extractor.Extract()) { return EX_SOFTWARE; } return EX_OK; } -ImageExtractor::ImageExtractor(unique_fd&& image_fd, std::unique_ptr<LpMetadata>&& metadata, +ImageExtractor::ImageExtractor(std::vector<unique_fd>&& image_fds, std::unique_ptr<LpMetadata>&& metadata, std::unordered_set<std::string>&& partitions, const std::string& output_dir) - : image_fd_(std::move(image_fd)), + : image_fds_(std::move(image_fds)), metadata_(std::move(metadata)), partitions_(std::move(partitions)), output_dir_(output_dir) {} @@ -186,6 +208,7 @@ bool ImageExtractor::Extract() { } for (const auto& [name, info] : partition_map_) { + std::cout << "Attempting to extract partition '" << name << "'...\n"; if (!ExtractPartition(info)) { return false; } @@ -217,13 +240,14 @@ bool ImageExtractor::ExtractPartition(const LpMetadataPartition* partition) { for (uint32_t i = 0; i < partition->num_extents; i++) { uint32_t index = partition->first_extent_index + i; const LpMetadataExtent& extent = metadata_->extents[index]; + std::cout << " Dealing with extent " << i << " from target source " << extent.target_source << "...\n"; if (extent.target_type != LP_TARGET_TYPE_LINEAR) { std::cerr << "Unsupported target type in extent: " << extent.target_type << "\n"; return false; } - if (extent.target_source != 0) { - std::cerr << "Split super devices are not supported.\n"; + if (extent.target_source >= image_fds_.size()) { + std::cerr << "Insufficient number of super images passed, need at least " << extent.target_source + 1 << ".\n"; return false; } total_size += extent.num_sectors * LP_SECTOR_SIZE; @@ -237,28 +261,28 @@ bool ImageExtractor::ExtractPartition(const LpMetadataPartition* partition) { return false; } - SparseWriter writer(output_fd, image_fd_, metadata_->geometry.logical_block_size); + SparseWriter writer(output_fd, metadata_->geometry.logical_block_size); // Extract each extent into output_fd. for (uint32_t i = 0; i < partition->num_extents; i++) { uint32_t index = partition->first_extent_index + i; const LpMetadataExtent& extent = metadata_->extents[index]; - if (!writer.WriteExtent(extent)) { + if (!writer.WriteExtent(image_fds_[extent.target_source], extent)) { return false; } } return writer.Finish(); } -SparseWriter::SparseWriter(int output_fd, int image_fd, uint32_t block_size) - : output_fd_(output_fd), image_fd_(image_fd), block_size_(block_size) {} +SparseWriter::SparseWriter(borrowed_fd output_fd, uint32_t block_size) + : output_fd_(output_fd), block_size_(block_size) {} -bool SparseWriter::WriteExtent(const LpMetadataExtent& extent) { +bool SparseWriter::WriteExtent(borrowed_fd image_fd, const LpMetadataExtent& extent) { auto buffer = std::make_unique<uint8_t[]>(block_size_); off_t super_offset = extent.target_data * LP_SECTOR_SIZE; - if (lseek(image_fd_, super_offset, SEEK_SET) < 0) { + if (lseek(image_fd.get(), super_offset, SEEK_SET) < 0) { std::cerr << "image lseek failed: " << strerror(errno) << "\n"; return false; } @@ -269,7 +293,7 @@ bool SparseWriter::WriteExtent(const LpMetadataExtent& extent) { std::cerr << "extent is not block-aligned\n"; return false; } - if (!android::base::ReadFully(image_fd_, buffer.get(), block_size_)) { + if (!android::base::ReadFully(image_fd, buffer.get(), block_size_)) { std::cerr << "read failed: " << strerror(errno) << "\n"; return false; } @@ -297,7 +321,7 @@ bool SparseWriter::WriteBlock(const uint8_t* data) { } if (hole_size_) { - if (lseek(output_fd_, hole_size_, SEEK_CUR) < 0) { + if (lseek(output_fd_.get(), hole_size_, SEEK_CUR) < 0) { std::cerr << "lseek failed: " << strerror(errno) << "\n"; return false; } @@ -312,12 +336,12 @@ bool SparseWriter::WriteBlock(const uint8_t* data) { bool SparseWriter::Finish() { if (hole_size_) { - off_t offset = lseek(output_fd_, 0, SEEK_CUR); + off_t offset = lseek(output_fd_.get(), 0, SEEK_CUR); if (offset < 0) { std::cerr << "lseek failed: " << strerror(errno) << "\n"; return false; } - if (ftruncate(output_fd_, offset + hole_size_) < 0) { + if (ftruncate(output_fd_.get(), offset + hole_size_) < 0) { std::cerr << "ftruncate failed: " << strerror(errno) << "\n"; return false; } diff --git a/profcollectd/libprofcollectd/report.rs b/profcollectd/libprofcollectd/report.rs index 22789bd8..6686ec62 100644 --- a/profcollectd/libprofcollectd/report.rs +++ b/profcollectd/libprofcollectd/report.rs @@ -79,14 +79,17 @@ fn get_report_filename(node_id: &MacAddr6) -> Result<String> { let since_epoch = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH)?; let ts = Timestamp::from_unix(&*UUID_CONTEXT, since_epoch.as_secs(), since_epoch.subsec_nanos()); - let uuid = Uuid::new_v1(ts, node_id.as_bytes())?; + let uuid = Uuid::new_v1( + ts, + node_id.as_bytes().try_into().expect("Invalid number of bytes in V1 UUID"), + ); Ok(uuid.to_string()) } /// Get report creation timestamp through its filename (version 1 UUID). pub fn get_report_ts(filename: &str) -> Result<SystemTime> { let uuid_ts = Uuid::parse_str(filename)? - .to_timestamp() + .get_timestamp() .ok_or_else(|| anyhow!("filename is not a valid V1 UUID."))? .to_unix(); Ok(SystemTime::UNIX_EPOCH + Duration::new(uuid_ts.0, uuid_ts.1)) diff --git a/simpleperf/JITDebugReader.cpp b/simpleperf/JITDebugReader.cpp index 28058940..2d4b132b 100644 --- a/simpleperf/JITDebugReader.cpp +++ b/simpleperf/JITDebugReader.cpp @@ -47,7 +47,7 @@ using android::base::StringPrintf; // If the size of a symfile is larger than EXPECTED_MAX_SYMFILE_SIZE, we don't want to read it // remotely. -static constexpr size_t MAX_JIT_SYMFILE_SIZE = 1024 * 1024u; +static constexpr size_t MAX_JIT_SYMFILE_SIZE = 1 * kMegabyte; // It takes about 30us-130us on Pixel (depending on the cpu frequency) to check if the descriptors // have been updated (most time spent in process_vm_preadv). We want to know if the JIT debug info diff --git a/simpleperf/RecordReadThread.cpp b/simpleperf/RecordReadThread.cpp index ae632d38..3e492ce2 100644 --- a/simpleperf/RecordReadThread.cpp +++ b/simpleperf/RecordReadThread.cpp @@ -29,8 +29,8 @@ namespace simpleperf { -static constexpr size_t kDefaultLowBufferLevel = 10 * 1024 * 1024u; -static constexpr size_t kDefaultCriticalBufferLevel = 5 * 1024 * 1024u; +static constexpr size_t kDefaultLowBufferLevel = 10 * kMegabyte; +static constexpr size_t kDefaultCriticalBufferLevel = 5 * kMegabyte; RecordBuffer::RecordBuffer(size_t buffer_size) : read_head_(0), write_head_(0), buffer_size_(buffer_size), buffer_(new char[buffer_size]) {} @@ -533,7 +533,7 @@ void RecordReadThread::PushRecordToRecordBuffer(KernelRecordReader* kernel_recor if (free_size < record_buffer_critical_level_) { // When the free size in record buffer is below critical level, drop sample records to save // space for more important records (like mmap or fork records). - stat_.lost_samples++; + stat_.userspace_lost_samples++; return; } size_t stack_size_limit = stack_size_in_sample_record_; @@ -580,10 +580,10 @@ void RecordReadThread::PushRecordToRecordBuffer(KernelRecordReader* kernel_recor memcpy(p + pos + new_stack_size, &new_stack_size, sizeof(uint64_t)); record_buffer_.FinishWrite(); if (new_stack_size < dyn_stack_size) { - stat_.cut_stack_samples++; + stat_.userspace_cut_stack_samples++; } } else { - stat_.lost_samples++; + stat_.userspace_lost_samples++; } return; } @@ -603,13 +603,18 @@ void RecordReadThread::PushRecordToRecordBuffer(KernelRecordReader* kernel_recor // only after we have collected the aux data. event_fds_disabled_by_kernel_.insert(kernel_record_reader->GetEventFd()); } + } else if (header.type == PERF_RECORD_LOST) { + LostRecord r; + if (r.Parse(attr_, p, p + header.size)) { + stat_.kernelspace_lost_records += static_cast<size_t>(r.lost); + } } record_buffer_.FinishWrite(); } else { if (header.type == PERF_RECORD_SAMPLE) { - stat_.lost_samples++; + stat_.userspace_lost_samples++; } else { - stat_.lost_non_samples++; + stat_.userspace_lost_non_samples++; } } } diff --git a/simpleperf/RecordReadThread.h b/simpleperf/RecordReadThread.h index 47dfbb20..658f9ea7 100644 --- a/simpleperf/RecordReadThread.h +++ b/simpleperf/RecordReadThread.h @@ -90,9 +90,10 @@ class RecordParser { }; struct RecordStat { - size_t lost_samples = 0; - size_t lost_non_samples = 0; - size_t cut_stack_samples = 0; + size_t kernelspace_lost_records = 0; + size_t userspace_lost_samples = 0; + size_t userspace_lost_non_samples = 0; + size_t userspace_cut_stack_samples = 0; uint64_t aux_data_size = 0; uint64_t lost_aux_data_size = 0; }; diff --git a/simpleperf/RecordReadThread_test.cpp b/simpleperf/RecordReadThread_test.cpp index 3c90e4be..9917c650 100644 --- a/simpleperf/RecordReadThread_test.cpp +++ b/simpleperf/RecordReadThread_test.cpp @@ -77,7 +77,7 @@ TEST(RecordParser, smoke) { std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(GetTestData(PERF_DATA_NO_UNWIND)); ASSERT_TRUE(reader); - RecordParser parser(*reader->AttrSection()[0].attr); + RecordParser parser(reader->AttrSection()[0].attr); auto process_record = [&](std::unique_ptr<Record> record) { if (record->type() == PERF_RECORD_MMAP || record->type() == PERF_RECORD_COMM || record->type() == PERF_RECORD_FORK || record->type() == PERF_RECORD_SAMPLE) { @@ -370,9 +370,9 @@ TEST_F(RecordReadThreadTest, process_sample_record) { thread.SetBufferLevels(record_buffer_size, record_buffer_size); read_record(r); ASSERT_FALSE(r); - ASSERT_EQ(thread.GetStat().lost_samples, 1u); - ASSERT_EQ(thread.GetStat().lost_non_samples, 0u); - ASSERT_EQ(thread.GetStat().cut_stack_samples, 1u); + ASSERT_EQ(thread.GetStat().userspace_lost_samples, 1u); + ASSERT_EQ(thread.GetStat().userspace_lost_non_samples, 0u); + ASSERT_EQ(thread.GetStat().userspace_cut_stack_samples, 1u); } // Test that the data notification exists until the RecordBuffer is empty. So we can read all @@ -421,9 +421,9 @@ TEST_F(RecordReadThreadTest, no_cut_samples) { received_samples++; } ASSERT_GT(received_samples, 0u); - ASSERT_GT(thread.GetStat().lost_samples, 0u); - ASSERT_EQ(thread.GetStat().lost_samples, total_samples - received_samples); - ASSERT_EQ(thread.GetStat().cut_stack_samples, 0u); + ASSERT_GT(thread.GetStat().userspace_lost_samples, 0u); + ASSERT_EQ(thread.GetStat().userspace_lost_samples, total_samples - received_samples); + ASSERT_EQ(thread.GetStat().userspace_cut_stack_samples, 0u); } TEST_F(RecordReadThreadTest, exclude_perf) { diff --git a/simpleperf/cmd_debug_unwind.cpp b/simpleperf/cmd_debug_unwind.cpp index 7af7aa38..52d3c96c 100644 --- a/simpleperf/cmd_debug_unwind.cpp +++ b/simpleperf/cmd_debug_unwind.cpp @@ -403,12 +403,13 @@ class TestFileGenerator : public RecordFileProcessor { bool WriteMapsForSample(const SampleRecord& r) { ThreadEntry* thread = thread_tree_.FindThread(r.tid_data.tid); if (thread != nullptr && thread->maps) { - auto attr = reader_->AttrSection()[0].attr; - auto event_id = reader_->AttrSection()[0].ids[0]; + const EventAttrIds& attrs = reader_->AttrSection(); + const perf_event_attr& attr = attrs[0].attr; + uint64_t event_id = attrs[0].ids[0]; for (const auto& p : thread->maps->maps) { const MapEntry* map = p.second; - Mmap2Record map_record(*attr, false, r.tid_data.pid, r.tid_data.tid, map->start_addr, + Mmap2Record map_record(attr, false, r.tid_data.pid, r.tid_data.tid, map->start_addr, map->len, map->pgoff, map->flags, map->dso->Path(), event_id, r.Timestamp()); if (!writer_->WriteRecord(map_record)) { @@ -425,7 +426,7 @@ class TestFileGenerator : public RecordFileProcessor { } std::unordered_set<int> feature_types_to_copy = { PerfFileFormat::FEAT_ARCH, PerfFileFormat::FEAT_CMDLINE, PerfFileFormat::FEAT_META_INFO}; - const size_t BUFFER_SIZE = 64 * 1024; + const size_t BUFFER_SIZE = 64 * kKilobyte; std::string buffer(BUFFER_SIZE, '\0'); for (const auto& p : reader_->FeatureSectionDescriptors()) { auto feat_type = p.first; diff --git a/simpleperf/cmd_dumprecord.cpp b/simpleperf/cmd_dumprecord.cpp index 89e56b67..9a71182f 100644 --- a/simpleperf/cmd_dumprecord.cpp +++ b/simpleperf/cmd_dumprecord.cpp @@ -306,11 +306,11 @@ void DumpRecordCommand::DumpFileHeader() { } void DumpRecordCommand::DumpAttrSection() { - std::vector<EventAttrWithId> attrs = record_file_reader_->AttrSection(); + const EventAttrIds& attrs = record_file_reader_->AttrSection(); for (size_t i = 0; i < attrs.size(); ++i) { const auto& attr = attrs[i]; printf("attr %zu:\n", i + 1); - DumpPerfEventAttr(*attr.attr, 1); + DumpPerfEventAttr(attr.attr, 1); if (!attr.ids.empty()) { printf(" ids:"); for (const auto& id : attr.ids) { @@ -440,16 +440,16 @@ bool DumpRecordCommand::ProcessTracingData(const TracingDataRecord& r) { if (!tracing) { return false; } - std::vector<EventAttrWithId> attrs = record_file_reader_->AttrSection(); + const EventAttrIds& attrs = record_file_reader_->AttrSection(); events_.resize(attrs.size()); for (size_t i = 0; i < attrs.size(); i++) { auto& attr = attrs[i].attr; auto& event = events_[i]; - if (attr->type != PERF_TYPE_TRACEPOINT) { + if (attr.type != PERF_TYPE_TRACEPOINT) { continue; } - TracingFormat format = tracing->GetTracingFormatHavingId(attr->config); + TracingFormat format = tracing->GetTracingFormatHavingId(attr.config); event.tp_fields = format.fields; // Decide dump function for each field. for (size_t j = 0; j < event.tp_fields.size(); j++) { diff --git a/simpleperf/cmd_inject.cpp b/simpleperf/cmd_inject.cpp index 43cfc728..e00f4826 100644 --- a/simpleperf/cmd_inject.cpp +++ b/simpleperf/cmd_inject.cpp @@ -618,6 +618,7 @@ class AutoFDOWriter { } // Write the binary path in comment. + fprintf(output_fp.get(), "// build_id: %s\n", key.build_id.ToString().c_str()); fprintf(output_fp.get(), "// %s\n\n", key.path.c_str()); } return true; diff --git a/simpleperf/cmd_kmem.cpp b/simpleperf/cmd_kmem.cpp index 81746ba5..febc6518 100644 --- a/simpleperf/cmd_kmem.cpp +++ b/simpleperf/cmd_kmem.cpp @@ -532,10 +532,9 @@ bool KmemCommand::PrepareToBuildSampleTree() { } void KmemCommand::ReadEventAttrsFromRecordFile() { - std::vector<EventAttrWithId> attrs = record_file_reader_->AttrSection(); - for (const auto& attr_with_id : attrs) { + for (const EventAttrWithId& attr_with_id : record_file_reader_->AttrSection()) { EventAttrWithName attr; - attr.attr = *attr_with_id.attr; + attr.attr = attr_with_id.attr; attr.event_ids = attr_with_id.ids; attr.name = GetEventNameByAttr(attr.attr); event_attrs_.push_back(attr); diff --git a/simpleperf/cmd_merge.cpp b/simpleperf/cmd_merge.cpp index 6f9bc20c..1bf9833f 100644 --- a/simpleperf/cmd_merge.cpp +++ b/simpleperf/cmd_merge.cpp @@ -255,16 +255,16 @@ class MergeCommand : public Command { // Check attr sections to know if recorded event types are the same. bool CheckAttrSection() { - std::vector<EventAttrWithId> attrs0 = readers_[0]->AttrSection(); + const EventAttrIds& attrs0 = readers_[0]->AttrSection(); for (size_t i = 1; i < readers_.size(); i++) { - std::vector<EventAttrWithId> attrs = readers_[i]->AttrSection(); + const EventAttrIds& attrs = readers_[i]->AttrSection(); if (attrs.size() != attrs0.size()) { LOG(ERROR) << input_files_[0] << " and " << input_files_[i] << " are not mergeable for recording different event types"; return false; } for (size_t attr_id = 0; attr_id < attrs.size(); attr_id++) { - if (memcmp(attrs[attr_id].attr, attrs0[attr_id].attr, sizeof(perf_event_attr)) != 0) { + if (attrs[attr_id].attr != attrs0[attr_id].attr) { LOG(ERROR) << input_files_[0] << " and " << input_files_[i] << " are not mergeable for recording different event types"; return false; @@ -300,7 +300,7 @@ class MergeCommand : public Command { // map event_ids in readers_[next_read_id] to event attrs. The map info is put into an // EventIdRecord. const std::unordered_map<uint64_t, size_t>& cur_map = readers_[prev_reader_id]->EventIdMap(); - std::vector<EventAttrWithId> attrs = readers_[next_reader_id]->AttrSection(); + const EventAttrIds& attrs = readers_[next_reader_id]->AttrSection(); std::vector<uint64_t> event_id_data; for (size_t attr_id = 0; attr_id < attrs.size(); attr_id++) { for (size_t event_id : attrs[attr_id].ids) { diff --git a/simpleperf/cmd_monitor.cpp b/simpleperf/cmd_monitor.cpp index d81ccfea..2ef9cbca 100644 --- a/simpleperf/cmd_monitor.cpp +++ b/simpleperf/cmd_monitor.cpp @@ -80,8 +80,8 @@ constexpr size_t DESIRED_PAGES_IN_MAPPED_BUFFER = 1024; // buffer size on a 8 core system. For system-wide recording, it is 8K pages * // 4K page_size * 8 cores = 256MB. For non system-wide recording, it is 1K pages // * 4K page_size * 8 cores = 64MB. -static constexpr size_t kRecordBufferSize = 64 * 1024 * 1024; -static constexpr size_t kSystemWideRecordBufferSize = 256 * 1024 * 1024; +static constexpr size_t kRecordBufferSize = 64 * kMegabyte; +static constexpr size_t kSystemWideRecordBufferSize = 256 * kMegabyte; class MonitorCommand : public Command { public: @@ -252,7 +252,7 @@ bool MonitorCommand::PrepareMonitoring() { // Use first perf_event_attr and first event id to dump mmap and comm records. EventAttrWithId dumping_attr_id = event_selection_set_.GetEventAttrWithId()[0]; - map_record_reader_.emplace(*dumping_attr_id.attr, dumping_attr_id.ids[0], + map_record_reader_.emplace(dumping_attr_id.attr, dumping_attr_id.ids[0], event_selection_set_.RecordNotExecutableMaps()); map_record_reader_->SetCallback([this](Record* r) { return ProcessRecord(r); }); diff --git a/simpleperf/cmd_record.cpp b/simpleperf/cmd_record.cpp index 514f9061..c9ae6591 100644 --- a/simpleperf/cmd_record.cpp +++ b/simpleperf/cmd_record.cpp @@ -98,23 +98,17 @@ static std::unordered_map<std::string, int> clockid_map = { // The max size of records dumped by kernel is 65535, and dump stack size // should be a multiply of 8, so MAX_DUMP_STACK_SIZE is 65528. -constexpr uint32_t MAX_DUMP_STACK_SIZE = 65528; +static constexpr uint32_t MAX_DUMP_STACK_SIZE = 65528; // The max allowed pages in mapped buffer is decided by rlimit(RLIMIT_MEMLOCK). // Here 1024 is a desired value for pages in mapped buffer. If mapped // successfully, the buffer size = 1024 * 4K (page size) = 4M. -constexpr size_t DESIRED_PAGES_IN_MAPPED_BUFFER = 1024; +static constexpr size_t DESIRED_PAGES_IN_MAPPED_BUFFER = 1024; // Cache size used by CallChainJoiner to cache call chains in memory. -constexpr size_t DEFAULT_CALL_CHAIN_JOINER_CACHE_SIZE = 8 * 1024 * 1024; +static constexpr size_t DEFAULT_CALL_CHAIN_JOINER_CACHE_SIZE = 8 * kMegabyte; -// Currently, the record buffer size in user-space is set to match the kernel buffer size on a -// 8 core system. For system-wide recording, it is 8K pages * 4K page_size * 8 cores = 256MB. -// For non system-wide recording, it is 1K pages * 4K page_size * 8 cores = 64MB. -static constexpr size_t kDefaultRecordBufferSize = 64 * 1024 * 1024; -static constexpr size_t kDefaultSystemWideRecordBufferSize = 256 * 1024 * 1024; - -static constexpr size_t kDefaultAuxBufferSize = 4 * 1024 * 1024; +static constexpr size_t kDefaultAuxBufferSize = 4 * kMegabyte; // On Pixel 3, it takes about 1ms to enable ETM, and 16-40ms to disable ETM and copy 4M ETM data. // So make default period to 100ms. @@ -128,6 +122,30 @@ struct TimeStat { uint64_t post_process_time = 0; }; +std::optional<size_t> GetDefaultRecordBufferSize(bool system_wide_recording) { + // Currently, the record buffer size in user-space is set to match the kernel buffer size on a + // 8 core system. For system-wide recording, it is 8K pages * 4K page_size * 8 cores = 256MB. + // For non system-wide recording, it is 1K pages * 4K page_size * 8 cores = 64MB. + // But on devices with memory >= 4GB, we increase buffer size to 256MB. This reduces the chance + // of cutting samples, which can cause broken callchains. + static constexpr size_t kLowMemoryRecordBufferSize = 64 * kMegabyte; + static constexpr size_t kHighMemoryRecordBufferSize = 256 * kMegabyte; + static constexpr size_t kSystemWideRecordBufferSize = 256 * kMegabyte; + // Ideally we can use >= 4GB here. But the memory size shown in /proc/meminfo is like to be 3.x GB + // on a device with 4GB memory. So we have to use <= 3GB. + static constexpr uint64_t kLowMemoryLimit = 3 * kGigabyte; + + if (system_wide_recording) { + return kSystemWideRecordBufferSize; + } + auto device_memory = GetMemorySize(); + if (!device_memory.has_value()) { + return std::nullopt; + } + return device_memory.value() <= kLowMemoryLimit ? kLowMemoryRecordBufferSize + : kHighMemoryRecordBufferSize; +} + class RecordCommand : public Command { public: RecordCommand() @@ -220,8 +238,7 @@ class RecordCommand : public Command { " It should be a power of 2. If not set, the max possible value <= 1024\n" " will be used.\n" "--user-buffer-size <buffer_size> Set buffer size in userspace to cache sample data.\n" -" By default, it is 64M for process recording and 256M\n" -" for system wide recording.\n" +" By default, it is %s.\n" "--aux-buffer-size <buffer_size> Set aux buffer size, only used in cs-etm event type.\n" " Need to be power of 2 and page size aligned.\n" " Used memory size is (buffer_size * (cpu_count + 1).\n" @@ -323,7 +340,6 @@ RECORD_FILTER_OPTION_HELP_MSG_FOR_RECORDING mmap_page_range_(std::make_pair(1, DESIRED_PAGES_IN_MAPPED_BUFFER)), record_filename_("perf.data"), sample_record_count_(0), - lost_record_count_(0), in_app_context_(false), trace_offcpu_(false), exclude_kernel_callchain_(false), @@ -338,6 +354,7 @@ RECORD_FILTER_OPTION_HELP_MSG_FOR_RECORDING signal(SIGPIPE, SIG_IGN); } + std::string LongHelpString() const override; void Run(const std::vector<std::string>& args, int* exit_code) override; bool Run(const std::vector<std::string>& args) override { int exit_code; @@ -356,8 +373,8 @@ RECORD_FILTER_OPTION_HELP_MSG_FOR_RECORDING bool TraceOffCpu(); bool SetEventSelectionFlags(); bool CreateAndInitRecordFile(); - std::unique_ptr<RecordFileWriter> CreateRecordFile( - const std::string& filename, const std::vector<EventAttrWithId>& override_attrs); + std::unique_ptr<RecordFileWriter> CreateRecordFile(const std::string& filename, + const EventAttrIds& attrs); bool DumpKernelSymbol(); bool DumpTracingData(); bool DumpMaps(); @@ -419,7 +436,6 @@ RECORD_FILTER_OPTION_HELP_MSG_FOR_RECORDING android::base::unique_fd stop_signal_fd_; uint64_t sample_record_count_; - uint64_t lost_record_count_; android::base::unique_fd start_profiling_fd_; bool stdio_controls_profiling_ = false; @@ -454,6 +470,27 @@ RECORD_FILTER_OPTION_HELP_MSG_FOR_RECORDING std::vector<std::string> add_counters_; }; +std::string RecordCommand::LongHelpString() const { + uint64_t process_buffer_size = 0; + uint64_t system_wide_buffer_size = 0; + if (auto size = GetDefaultRecordBufferSize(false); size) { + process_buffer_size = size.value() / kMegabyte; + } + if (auto size = GetDefaultRecordBufferSize(true); size) { + system_wide_buffer_size = size.value() / kMegabyte; + } + std::string buffer_size_str; + if (process_buffer_size == system_wide_buffer_size) { + buffer_size_str = android::base::StringPrintf("%" PRIu64 "M", process_buffer_size); + } else { + buffer_size_str = + android::base::StringPrintf("%" PRIu64 "M for process recording and %" PRIu64 + "M\n for system wide recording", + process_buffer_size, system_wide_buffer_size); + } + return android::base::StringPrintf(long_help_string_.c_str(), buffer_size_str.c_str()); +} + void RecordCommand::Run(const std::vector<std::string>& args, int* exit_code) { *exit_code = 1; time_stat_.prepare_recording_time = GetSystemClock(); @@ -616,8 +653,11 @@ bool RecordCommand::PrepareRecording(Workload* workload) { if (user_buffer_size_.has_value()) { record_buffer_size = user_buffer_size_.value(); } else { - record_buffer_size = - system_wide_collection_ ? kDefaultSystemWideRecordBufferSize : kDefaultRecordBufferSize; + auto default_size = GetDefaultRecordBufferSize(system_wide_collection_); + if (!default_size.has_value()) { + return false; + } + record_buffer_size = default_size.value(); } if (!event_selection_set_.MmapEventFiles(mmap_page_range_.first, mmap_page_range_.second, aux_buffer_size_, record_buffer_size, @@ -811,23 +851,55 @@ bool RecordCommand::PostProcessRecording(const std::vector<std::string>& args) { LOG(INFO) << "Aux data lost in user space: " << record_stat.lost_aux_data_size; } } else { - std::string cut_samples; - if (record_stat.cut_stack_samples > 0) { - cut_samples = android::base::StringPrintf(" (cut %zu)", record_stat.cut_stack_samples); - } - lost_record_count_ += record_stat.lost_samples + record_stat.lost_non_samples; - LOG(INFO) << "Samples recorded: " << sample_record_count_ << cut_samples - << ". Samples lost: " << lost_record_count_ << "."; - LOG(DEBUG) << "In user space, dropped " << record_stat.lost_samples << " samples, " - << record_stat.lost_non_samples << " non samples, cut stack of " - << record_stat.cut_stack_samples << " samples."; - if (sample_record_count_ + lost_record_count_ != 0) { - double lost_percent = - static_cast<double>(lost_record_count_) / (lost_record_count_ + sample_record_count_); - constexpr double LOST_PERCENT_WARNING_BAR = 0.1; - if (lost_percent >= LOST_PERCENT_WARNING_BAR) { - LOG(WARNING) << "Lost " << (lost_percent * 100) << "% of samples, " - << "consider increasing mmap_pages(-m), " + // Here we report all lost records as samples. This isn't accurate. Because records like + // MmapRecords are not samples. But It's easier for users to understand. + size_t userspace_lost_samples = + record_stat.userspace_lost_samples + record_stat.userspace_lost_non_samples; + size_t lost_samples = record_stat.kernelspace_lost_records + userspace_lost_samples; + + std::stringstream os; + os << "Samples recorded: " << sample_record_count_; + if (record_stat.userspace_cut_stack_samples > 0) { + os << " (cut " << record_stat.userspace_cut_stack_samples << ")"; + } + os << ". Samples lost: " << lost_samples; + if (lost_samples != 0) { + os << " (kernelspace: " << record_stat.kernelspace_lost_records + << ", userspace: " << userspace_lost_samples << ")"; + } + os << "."; + LOG(INFO) << os.str(); + + LOG(DEBUG) << "Record stat: kernelspace_lost_records=" << record_stat.kernelspace_lost_records + << ", userspace_lost_samples=" << record_stat.userspace_lost_samples + << ", userspace_lost_non_samples=" << record_stat.userspace_lost_non_samples + << ", userspace_cut_stack_samples=" << record_stat.userspace_cut_stack_samples; + + if (sample_record_count_ + record_stat.kernelspace_lost_records != 0) { + double kernelspace_lost_percent = + static_cast<double>(record_stat.kernelspace_lost_records) / + (record_stat.kernelspace_lost_records + sample_record_count_); + constexpr double KERNELSPACE_LOST_PERCENT_WARNING_BAR = 0.1; + if (kernelspace_lost_percent >= KERNELSPACE_LOST_PERCENT_WARNING_BAR) { + LOG(WARNING) << "Lost " << (kernelspace_lost_percent * 100) + << "% of samples in kernel space, " + << "consider increasing kernel buffer size(-m), " + << "or decreasing sample frequency(-f), " + << "or increasing sample period(-c)."; + } + } + size_t userspace_lost_cut_samples = + userspace_lost_samples + record_stat.userspace_cut_stack_samples; + size_t userspace_complete_samples = + sample_record_count_ - record_stat.userspace_cut_stack_samples; + if (userspace_complete_samples + userspace_lost_cut_samples != 0) { + double userspace_lost_percent = static_cast<double>(userspace_lost_cut_samples) / + (userspace_complete_samples + userspace_lost_cut_samples); + constexpr double USERSPACE_LOST_PERCENT_WARNING_BAR = 0.1; + if (userspace_lost_percent >= USERSPACE_LOST_PERCENT_WARNING_BAR) { + LOG(WARNING) << "Lost/Cut " << (userspace_lost_percent * 100) + << "% of samples in user space, " + << "consider increasing userspace buffer size(--user-buffer-size), " << "or decreasing sample frequency(-f), " << "or increasing sample period(-c)."; } @@ -1300,32 +1372,35 @@ bool RecordCommand::SetEventSelectionFlags() { } bool RecordCommand::CreateAndInitRecordFile() { - record_file_writer_ = - CreateRecordFile(record_filename_, event_selection_set_.GetEventAttrWithId()); + EventAttrIds attrs = event_selection_set_.GetEventAttrWithId(); + bool remove_regs_and_stacks = unwind_dwarf_callchain_ && !post_unwind_; + if (remove_regs_and_stacks) { + for (auto& attr : attrs) { + ReplaceRegAndStackWithCallChain(attr.attr); + } + } + record_file_writer_ = CreateRecordFile(record_filename_, attrs); if (record_file_writer_ == nullptr) { return false; } // Use first perf_event_attr and first event id to dump mmap and comm records. - dumping_attr_id_ = event_selection_set_.GetEventAttrWithId()[0]; + CHECK(!attrs.empty()); + dumping_attr_id_ = attrs[0]; CHECK(!dumping_attr_id_.ids.empty()); - map_record_reader_.emplace(*dumping_attr_id_.attr, dumping_attr_id_.ids[0], + map_record_reader_.emplace(dumping_attr_id_.attr, dumping_attr_id_.ids[0], event_selection_set_.RecordNotExecutableMaps()); map_record_reader_->SetCallback([this](Record* r) { return ProcessRecord(r); }); return DumpKernelSymbol() && DumpTracingData() && DumpMaps() && DumpAuxTraceInfo(); } -std::unique_ptr<RecordFileWriter> RecordCommand::CreateRecordFile( - const std::string& filename, const std::vector<EventAttrWithId>& attrs) { +std::unique_ptr<RecordFileWriter> RecordCommand::CreateRecordFile(const std::string& filename, + const EventAttrIds& attrs) { std::unique_ptr<RecordFileWriter> writer = RecordFileWriter::CreateInstance(filename); - if (writer == nullptr) { - return nullptr; - } - - if (!writer->WriteAttrSection(attrs)) { - return nullptr; + if (writer != nullptr && writer->WriteAttrSection(attrs)) { + return writer; } - return writer; + return nullptr; } bool RecordCommand::DumpKernelSymbol() { @@ -1506,8 +1581,6 @@ bool RecordCommand::SaveRecordAfterUnwinding(Record* record) { return true; } sample_record_count_++; - } else if (record->type() == PERF_RECORD_LOST) { - lost_record_count_ += static_cast<LostRecord*>(record)->lost; } else { thread_tree_.Update(*record); } @@ -1525,8 +1598,6 @@ bool RecordCommand::SaveRecordWithoutUnwinding(Record* record) { return true; } sample_record_count_++; - } else if (record->type() == PERF_RECORD_LOST) { - lost_record_count_ += static_cast<LostRecord*>(record)->lost; } return record_file_writer_->WriteRecord(*record); } @@ -1537,7 +1608,7 @@ bool RecordCommand::ProcessJITDebugInfo(const std::vector<JITDebugInfo>& debug_i if (info.type == JITDebugInfo::JIT_DEBUG_JIT_CODE) { uint64_t timestamp = jit_debug_reader_->SyncWithRecords() ? info.timestamp : last_record_timestamp_; - Mmap2Record record(*dumping_attr_id_.attr, false, info.pid, info.pid, info.jit_code_addr, + Mmap2Record record(dumping_attr_id_.attr, false, info.pid, info.pid, info.jit_code_addr, info.jit_code_len, info.file_offset, map_flags::PROT_JIT_SYMFILE_MAP, info.file_path, dumping_attr_id_.ids[0], timestamp); if (!ProcessRecord(&record)) { @@ -1548,7 +1619,7 @@ bool RecordCommand::ProcessJITDebugInfo(const std::vector<JITDebugInfo>& debug_i ThreadMmap& map = *info.extracted_dex_file_map; uint64_t timestamp = jit_debug_reader_->SyncWithRecords() ? info.timestamp : last_record_timestamp_; - Mmap2Record record(*dumping_attr_id_.attr, false, info.pid, info.pid, map.start_addr, + Mmap2Record record(dumping_attr_id_.attr, false, info.pid, info.pid, map.start_addr, map.len, map.pgoff, map.prot, map.name, dumping_attr_id_.ids[0], timestamp); if (!ProcessRecord(&record)) { @@ -1711,11 +1782,15 @@ std::unique_ptr<RecordFileReader> RecordCommand::MoveRecordFile(const std::strin return nullptr; } record_file_writer_.reset(); - { - std::error_code ec; - std::filesystem::rename(record_filename_, old_filename, ec); - if (ec) { - LOG(ERROR) << "Failed to rename: " << ec.message(); + std::error_code ec; + std::filesystem::rename(record_filename_, old_filename, ec); + if (ec) { + LOG(DEBUG) << "Failed to rename: " << ec.message(); + // rename() fails on Android N x86 emulator, which uses kernel 3.10. Because rename() in bionic + // uses renameat2 syscall, which isn't support on kernel < 3.15. So add a fallback to mv + // command. The mv command can also work with other situations when rename() doesn't work. + // So we'd like to keep it as a fallback to rename(). + if (!Workload::RunCmd({"mv", record_filename_, old_filename})) { return nullptr; } } @@ -1774,8 +1849,16 @@ bool RecordCommand::PostUnwindRecords() { if (!reader) { return false; } + // Write new event attrs without regs and stacks fields. + EventAttrIds attrs = reader->AttrSection(); + for (auto& attr : attrs) { + ReplaceRegAndStackWithCallChain(attr.attr); + } + if (!record_file_writer_->WriteAttrSection(attrs)) { + return false; + } + sample_record_count_ = 0; - lost_record_count_ = 0; auto callback = [this](std::unique_ptr<Record> record) { return SaveRecordAfterUnwinding(record.get()); }; @@ -2032,6 +2115,15 @@ bool RecordCommand::DumpMetaInfoFeature(bool kernel_symbols_available) { if (dwarf_callchain_sampling_ && !unwind_dwarf_callchain_) { OfflineUnwinder::CollectMetaInfo(&info_map); } + auto record_stat = event_selection_set_.GetRecordStat(); + info_map["record_stat"] = android::base::StringPrintf( + "sample_record_count=%" PRIu64 + ",kernelspace_lost_records=%zu,userspace_lost_samples=%zu," + "userspace_lost_non_samples=%zu,userspace_cut_stack_samples=%zu", + sample_record_count_, record_stat.kernelspace_lost_records, + record_stat.userspace_lost_samples, record_stat.userspace_lost_non_samples, + record_stat.userspace_cut_stack_samples); + return record_file_writer_->WriteMetaInfoFeature(info_map); } diff --git a/simpleperf/cmd_record_test.cpp b/simpleperf/cmd_record_test.cpp index a2409fac..395bfd19 100644 --- a/simpleperf/cmd_record_test.cpp +++ b/simpleperf/cmd_record_test.cpp @@ -98,15 +98,15 @@ static void CheckEventType(const std::string& record_file, const std::string& ev ASSERT_TRUE(type != nullptr); std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(record_file); ASSERT_TRUE(reader); - std::vector<EventAttrWithId> attrs = reader->AttrSection(); - for (auto& attr : attrs) { - if (attr.attr->type == type->type && attr.attr->config == type->config) { - if (attr.attr->freq == 0) { - ASSERT_EQ(sample_period, attr.attr->sample_period); + for (const auto& attr_with_id : reader->AttrSection()) { + const perf_event_attr& attr = attr_with_id.attr; + if (attr.type == type->type && attr.config == type->config) { + if (attr.freq == 0) { + ASSERT_EQ(sample_period, attr.sample_period); ASSERT_EQ(sample_freq, 0u); } else { ASSERT_EQ(sample_period, 0u); - ASSERT_EQ(sample_freq, attr.attr->sample_freq); + ASSERT_EQ(sample_freq, attr.sample_freq); } return; } @@ -204,10 +204,10 @@ TEST(record_cmd, rN_event) { ASSERT_TRUE(RunRecordCmd({"-e", event_name}, tmpfile.path)); std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(tmpfile.path); ASSERT_TRUE(reader); - std::vector<EventAttrWithId> attrs = reader->AttrSection(); + const EventAttrIds& attrs = reader->AttrSection(); ASSERT_EQ(1u, attrs.size()); - ASSERT_EQ(PERF_TYPE_RAW, attrs[0].attr->type); - ASSERT_EQ(event_number, attrs[0].attr->config); + ASSERT_EQ(PERF_TYPE_RAW, attrs[0].attr.type); + ASSERT_EQ(event_number, attrs[0].attr.config); } TEST(record_cmd, branch_sampling) { @@ -257,7 +257,16 @@ TEST(record_cmd, dwarf_callchain_sampling) { ASSERT_TRUE(RunRecordCmd({"-p", pid, "--call-graph", "dwarf"})); ASSERT_TRUE(RunRecordCmd({"-p", pid, "--call-graph", "dwarf,16384"})); ASSERT_FALSE(RunRecordCmd({"-p", pid, "--call-graph", "dwarf,65536"})); - ASSERT_TRUE(RunRecordCmd({"-p", pid, "-g"})); + TemporaryFile tmpfile; + ASSERT_TRUE(RunRecordCmd({"-p", pid, "-g"}, tmpfile.path)); + auto reader = RecordFileReader::CreateInstance(tmpfile.path); + ASSERT_TRUE(reader); + const EventAttrIds& attrs = reader->AttrSection(); + ASSERT_GT(attrs.size(), 0); + // Check that reg and stack fields are removed after unwinding. + for (const auto& attr : attrs) { + ASSERT_EQ(attr.attr.sample_type & (PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER), 0); + } } TEST(record_cmd, system_wide_dwarf_callchain_sampling) { @@ -512,6 +521,7 @@ TEST(record_cmd, record_meta_info_feature) { auto& info_map = reader->GetMetaInfoFeature(); ASSERT_NE(info_map.find("simpleperf_version"), info_map.end()); ASSERT_NE(info_map.find("timestamp"), info_map.end()); + ASSERT_NE(info_map.find("record_stat"), info_map.end()); #if defined(__ANDROID__) ASSERT_NE(info_map.find("product_props"), info_map.end()); ASSERT_NE(info_map.find("android_version"), info_map.end()); @@ -551,7 +561,7 @@ TEST(record_cmd, trace_offcpu_option) { auto info_map = reader->GetMetaInfoFeature(); ASSERT_EQ(info_map["trace_offcpu"], "true"); if (IsSwitchRecordSupported()) { - ASSERT_EQ(reader->AttrSection()[0].attr->context_switch, 1); + ASSERT_EQ(reader->AttrSection()[0].attr.context_switch, 1); } // Release recording environment in perf.data, to avoid affecting tests below. reader.reset(); @@ -943,9 +953,9 @@ TEST(record_cmd, cs_etm_event) { // cs-etm uses sample period instead of sample freq. ASSERT_EQ(reader->AttrSection().size(), 1u); - const perf_event_attr* attr = reader->AttrSection()[0].attr; - ASSERT_EQ(attr->freq, 0); - ASSERT_EQ(attr->sample_period, 1); + const perf_event_attr& attr = reader->AttrSection()[0].attr; + ASSERT_EQ(attr.freq, 0); + ASSERT_EQ(attr.sample_period, 1); bool has_auxtrace_info = false; bool has_auxtrace = false; @@ -1252,3 +1262,21 @@ TEST(record_cmd, add_counter_option) { TEST(record_cmd, user_buffer_size_option) { ASSERT_TRUE(RunRecordCmd({"--user-buffer-size", "256M"})); } + +TEST(record_cmd, record_process_name) { + TemporaryFile tmpfile; + ASSERT_TRUE(RecordCmd()->Run({"-e", GetDefaultEvent(), "-o", tmpfile.path, "sleep", SLEEP_SEC})); + std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(tmpfile.path); + ASSERT_TRUE(reader); + bool has_comm = false; + ASSERT_TRUE(reader->ReadDataSection([&](std::unique_ptr<Record> r) { + if (r->type() == PERF_RECORD_COMM) { + CommRecord* cr = static_cast<CommRecord*>(r.get()); + if (strcmp(cr->comm, "sleep") == 0) { + has_comm = true; + } + } + return true; + })); + ASSERT_TRUE(has_comm); +} diff --git a/simpleperf/cmd_report.cpp b/simpleperf/cmd_report.cpp index 2f2dab10..2ab98548 100644 --- a/simpleperf/cmd_report.cpp +++ b/simpleperf/cmd_report.cpp @@ -856,9 +856,8 @@ bool ReportCommand::ReadMetaInfoFromRecordFile() { } bool ReportCommand::ReadEventAttrFromRecordFile() { - std::vector<EventAttrWithId> attrs = record_file_reader_->AttrSection(); - for (const auto& attr_with_id : attrs) { - const perf_event_attr& attr = *attr_with_id.attr; + for (const EventAttrWithId& attr_with_id : record_file_reader_->AttrSection()) { + const perf_event_attr& attr = attr_with_id.attr; attr_names_.emplace_back(GetEventNameByAttr(attr)); // There are no samples for events added by --add-counter. So skip them. diff --git a/simpleperf/cmd_report_sample.cpp b/simpleperf/cmd_report_sample.cpp index 1026dbbe..12e3ed61 100644 --- a/simpleperf/cmd_report_sample.cpp +++ b/simpleperf/cmd_report_sample.cpp @@ -522,7 +522,7 @@ bool ReportSampleCommand::OpenRecordFile() { if (auto it = meta_info.find("trace_offcpu"); it != meta_info.end()) { trace_offcpu_ = it->second == "true"; if (trace_offcpu_) { - std::string event_name = GetEventNameByAttr(*record_file_reader_->AttrSection()[0].attr); + std::string event_name = GetEventNameByAttr(record_file_reader_->AttrSection()[0].attr); if (!android::base::StartsWith(event_name, "cpu-clock") && !android::base::StartsWith(event_name, "task-clock")) { LOG(ERROR) << "Recording file " << record_filename_ << " is no longer supported. " @@ -537,8 +537,8 @@ bool ReportSampleCommand::OpenRecordFile() { if (!record_filter_.CheckClock(record_file_reader_->GetClockId())) { return false; } - for (EventAttrWithId& attr : record_file_reader_->AttrSection()) { - event_types_.push_back(GetEventNameByAttr(*attr.attr)); + for (const EventAttrWithId& attr : record_file_reader_->AttrSection()) { + event_types_.push_back(GetEventNameByAttr(attr.attr)); } return true; } diff --git a/simpleperf/cmd_stat_test.cpp b/simpleperf/cmd_stat_test.cpp index dff42a21..53ff0d48 100644 --- a/simpleperf/cmd_stat_test.cpp +++ b/simpleperf/cmd_stat_test.cpp @@ -233,12 +233,12 @@ TEST(stat_cmd, sample_speed_should_be_zero) { ASSERT_TRUE(set.AddEventType("cpu-cycles")); set.AddMonitoredProcesses({getpid()}); ASSERT_TRUE(set.OpenEventFiles({-1})); - std::vector<EventAttrWithId> attrs = set.GetEventAttrWithId(); + const EventAttrIds& attrs = set.GetEventAttrWithId(); ASSERT_GT(attrs.size(), 0u); for (auto& attr : attrs) { - ASSERT_EQ(attr.attr->sample_period, 0u); - ASSERT_EQ(attr.attr->sample_freq, 0u); - ASSERT_EQ(attr.attr->freq, 0u); + ASSERT_EQ(attr.attr.sample_period, 0u); + ASSERT_EQ(attr.attr.sample_freq, 0u); + ASSERT_EQ(attr.attr.freq, 0u); } } diff --git a/simpleperf/cmd_trace_sched.cpp b/simpleperf/cmd_trace_sched.cpp index 607fb2a4..d02495a9 100644 --- a/simpleperf/cmd_trace_sched.cpp +++ b/simpleperf/cmd_trace_sched.cpp @@ -192,9 +192,9 @@ bool TraceSchedCommand::ParseSchedEvents(const std::string& record_file_path) { return false; } const EventType* event = FindEventTypeByName("sched:sched_stat_runtime"); - std::vector<EventAttrWithId> attrs = reader->AttrSection(); - if (attrs.size() != 1u || attrs[0].attr->type != event->type || - attrs[0].attr->config != event->config) { + const EventAttrIds& attrs = reader->AttrSection(); + if (attrs.size() != 1u || attrs[0].attr.type != event->type || + attrs[0].attr.config != event->config) { LOG(ERROR) << "sched:sched_stat_runtime isn't recorded in " << record_file_path; return false; } diff --git a/simpleperf/command.h b/simpleperf/command.h index 1616bc40..35db8469 100644 --- a/simpleperf/command.h +++ b/simpleperf/command.h @@ -169,7 +169,7 @@ class Command { const std::string& ShortHelpString() const { return short_help_string_; } - const std::string LongHelpString() const { return long_help_string_; } + virtual std::string LongHelpString() const { return long_help_string_; } virtual bool Run(const std::vector<std::string>&) { return false; } virtual void Run(const std::vector<std::string>& args, int* exit_code) { @@ -203,7 +203,6 @@ class Command { bool NextArgumentOrError(const std::vector<std::string>& args, size_t* pi); void ReportUnknownOption(const std::vector<std::string>& args, size_t i); - private: const std::string name_; const std::string short_help_string_; const std::string long_help_string_; diff --git a/simpleperf/cpu_hotplug_test.cpp b/simpleperf/cpu_hotplug_test.cpp index 9b78c644..526ab87a 100644 --- a/simpleperf/cpu_hotplug_test.cpp +++ b/simpleperf/cpu_hotplug_test.cpp @@ -497,7 +497,7 @@ int main(int argc, char** argv) { verbose_mode = true; } } - android::base::InitLogging(argv, android::base::StderrLogger); testing::InitGoogleTest(&argc, argv); + android::base::InitLogging(argv, android::base::StderrLogger); return RUN_ALL_TESTS(); } diff --git a/simpleperf/doc/README.md b/simpleperf/doc/README.md index f989b785..d1ecbcce 100644 --- a/simpleperf/doc/README.md +++ b/simpleperf/doc/README.md @@ -106,83 +106,100 @@ See [view_the_profile.md](./view_the_profile.md). ## Answers to common issues -### Why we suggest profiling on Android >= N devices? - -1. Running on a device reflects a real running situation, so we suggest - profiling on real devices instead of emulators. -2. To profile Java code, we need ART running in oat mode, which is only - available >= L for rooted devices, and >= N for non-rooted devices. -3. Old Android versions are likely to be shipped with old kernels (< 3.18), - which may not support profiling features like recording dwarf based call graphs. -4. Old Android versions are likely to be shipped with Arm32 chips. In Arm32 - mode, recording stack frame based call graphs doesn't work well. - -### Suggestions about recording call graphs - -Below is our experiences of dwarf based call graphs and stack frame based call graphs. - -dwarf based call graphs: -1. Need support of debug information in binaries. -2. Behave normally well on both ARM and ARM64, for both fully compiled Java code and C++ code. -3. Can only unwind 64K stack for each sample. So usually can't show complete flamegraph. But - probably is enough for users to identify hot places. -4. Take more CPU time than stack frame based call graphs. So the sample frequency is suggested - to be 1000 Hz. Thus at most 1000 samples per second. - -stack frame based call graphs: -1. Need support of stack frame registers. -2. Don't work well on ARM. Because ARM is short of registers, and ARM and THUMB code have different - stack frame registers. So the kernel can't unwind user stack containing both ARM/THUMB code. -3. Also don't work well on fully compiled Java code on ARM64. Because the ART compiler doesn't - reserve stack frame registers. -4. Work well when profiling native programs on ARM64. One example is profiling surfacelinger. And +### Support on different Android versions + +On Android < N, the kernel may be too old (< 3.18) to support features like recording DWARF +based call graphs. +On Android M - O, we can only profile C++ code and fully compiled Java code. +On Android >= P, the ART interpreter supports DWARF based unwinding. So we can profile Java code. +On Android >= Q, we can used simpleperf shipped on device to profile released Android apps, with + `<profileable android:shell="true" />`. + + +### Comparing DWARF based and stack frame based call graphs + +Simpleperf supports two ways recording call stacks with samples. One is DWARF based call graph, +the other is stack frame based call graph. Below is their comparison: + +Recording DWARF based call graph: +1. Needs support of debug information in binaries. +2. Behaves normally well on both ARM and ARM64, for both Java code and C++ code. +3. Can only unwind 64K stack for each sample. So it isn't always possible to unwind to the bottom. + However, this is alleviated in simpleperf, as explained in the next section. +4. Takes more CPU time than stack frame based call graphs. So it has higher overhead, and can't + sample at very high frequency (usually <= 4000 Hz). + +Recording stack frame based call graph: +1. Needs support of stack frame registers. +2. Doesn't work well on ARM. Because ARM is short of registers, and ARM and THUMB code have + different stack frame registers. So the kernel can't unwind user stack containing both ARM and + THUMB code. +3. Also doesn't work well on Java code. Because the ART compiler doesn't reserve stack frame + registers. And it can't get frames for interpreted Java code. +4. Works well when profiling native programs on ARM64. One example is profiling surfacelinger. And usually shows complete flamegraph when it works well. -5. Take less CPU time than dwarf based call graphs. So the sample frequency can be 4000 Hz or +5. Takes much less CPU time than DWARF based call graphs. So the sample frequency can be 10000 Hz or higher. -So if you need to profile code on ARM or profile fully compiled Java code, dwarf based call graphs -may be better. If you need to profile C++ code on ARM64, stack frame based call graphs may be -better. After all, you can always try dwarf based call graph first, because it always produces -reasonable results when given unstripped binaries properly. If it doesn't work well enough, then -try stack frame based call graphs instead. +So if you need to profile code on ARM or profile Java code, DWARF based call graph is better. If you +need to profile C++ code on ARM64, stack frame based call graphs may be better. After all, you can +fisrt try DWARF based call graph, which is also the default option when `-g` is used. Because it +always produces reasonable results. If it doesn't work well enough, then try stack frame based call +graph instead. -Simpleperf may need unstripped native binaries on the device to generate good dwarf based call -graphs. It can be supported by downloading unstripped native libraries on device, as [here](#fix-broken-callchain-stopped-at-c-functions). -### Why we can't always get complete DWARF-based call graphs? +### Fix broken DWARF based call graph -DWARF-based call graphs are generated by unwinding thread stacks. When a sample is generated, up to -64KB stack data is dumped by the kernel. By unwinding the stack based on dwarf information, we get -a callchain. But the thread stack can be much longer than 64KB. In that case, we can't unwind to -the thread start point. +A DWARF-based call graph is generated by unwinding thread stacks. When a sample is recorded, a +kernel dumps up to 64 kilobytes of stack data. By unwinding the stack based on DWARF information, +we can get a call stack. -To alleviate the problem, simpleperf joins callchains after recording them. If two callchains of -a thread have an entry containing the same ip and sp address, then simpleperf tries to join them to -make the callchains longer. In that case, the longer we run, the more samples we get. This makes it -more likely to get complete callchains, but it's still not guaranteed to get complete call graphs. +Two reasons may cause a broken call stack: +1. The kernel can only dump up to 64 kilobytes of stack data for each sample, but a thread can have + much larger stack. In this case, we can't unwind to the thread start point. -### How to solve missing symbols in report? +2. We need binaries containing DWARF call frame information to unwind stack frames. The binary + should have one of the following sections: .eh_frame, .debug_frame, .ARM.exidx or .gnu_debugdata. -The simpleperf record command collects symbols on device in perf.data. But if the native libraries -you use on device are stripped, this will result in a lot of unknown symbols in the report. A -solution is to build binary_cache on host. +To mitigate these problems, + + +For the missing stack data problem: +1. To alleviate it, simpleperf joins callchains (call stacks) after recording. If two callchains of + a thread have an entry containing the same ip and sp address, then simpleperf tries to join them + to make the callchains longer. So we can get more complete callchains by recording longer and + joining more samples. This doesn't guarantee to get complete call graphs. But it usually works + well. + +2. Simpleperf stores samples in a buffer before unwinding them. If the bufer is low in free space, + simpleperf may decide to cut stack data for a sample to 1K. Hopefully, this can be recovered by + callchain joiner. But when a high percentage of samples are cut, many callchains can be broken. + We can tell if many samples are cut in the record command output, like: ```sh -# Collect binaries needed by perf.data in binary_cache/. -$ ./binary_cache_builder.py -lib NATIVE_LIB_DIR,... +$ simpleperf record ... +simpleperf I cmd_record.cpp:809] Samples recorded: 105584 (cut 86291). Samples lost: 6501. ``` -The NATIVE_LIB_DIRs passed in -lib option are the directories containing unstripped native -libraries on host. After running it, the native libraries containing symbol tables are collected -in binary_cache/ for use when reporting. + There are two ways to avoid cutting samples. One is increasing the buffer size, like + `--user-buffer-size 1G`. But `--user-buffer-size` is only available on latest simpleperf. If that + option isn't available, we can use `--no-cut-samples` to disable cutting samples. -```sh -$ ./report.py --symfs binary_cache +For the missing DWARF call frame info problem: +1. Most C++ code generates binaries containing call frame info, in .eh_frame or .ARM.exidx sections. + These sections are not stripped, and are usually enough for stack unwinding. + +2. For C code and a small percentage of C++ code that the compiler is sure will not generate + exceptions, the call frame info is generated in .debug_frame section. .debug_frame section is + usually stripped with other debug sections. One way to fix it, is to download unstripped binaries + on device, as [here](#fix-broken-callchain-stopped-at-c-functions). + +3. The compiler doesn't generate unwind instructions for function prologue and epilogue. Because + they operates stack frames and will not generate exceptions. But profiling may hit these + instructions, and fails to unwind them. This usually doesn't matter in a frame graph. But in a + time based Stack Chart (like in Android Studio and Firefox profiler), this causes stack gaps once + in a while. We can remove stack gaps via `--remove-gaps`, which is already enabled by default. -# report_html.py searches binary_cache/ automatically, so you don't need to -# pass it any argument. -$ ./report_html.py -``` ### Fix broken callchain stopped at C functions @@ -207,6 +224,31 @@ To use app_profiler.py: $ ./app_profiler.py -lib <unstripped_dir> ``` + +### How to solve missing symbols in report? + +The simpleperf record command collects symbols on device in perf.data. But if the native libraries +you use on device are stripped, this will result in a lot of unknown symbols in the report. A +solution is to build binary_cache on host. + +```sh +# Collect binaries needed by perf.data in binary_cache/. +$ ./binary_cache_builder.py -lib NATIVE_LIB_DIR,... +``` + +The NATIVE_LIB_DIRs passed in -lib option are the directories containing unstripped native +libraries on host. After running it, the native libraries containing symbol tables are collected +in binary_cache/ for use when reporting. + +```sh +$ ./report.py --symfs binary_cache + +# report_html.py searches binary_cache/ automatically, so you don't need to +# pass it any argument. +$ ./report_html.py +``` + + ### Show annotated source code and disassembly To show hot places at source code and instruction level, we need to show source code and diff --git a/simpleperf/doc/collect_etm_data_for_autofdo.md b/simpleperf/doc/collect_etm_data_for_autofdo.md index 5313c4e6..2c001016 100644 --- a/simpleperf/doc/collect_etm_data_for_autofdo.md +++ b/simpleperf/doc/collect_etm_data_for_autofdo.md @@ -157,6 +157,25 @@ Step 4: Use AutoFDO data to build optimized binary. ```sh (host) <AOSP>$ mkdir toolchain/pgo-profiles/sampling/ (host) <AOSP>$ cp etm_test_loop.afdo toolchain/pgo-profiles/sampling/ +(host) <AOSP>$ vi toolchain/pgo-profiles/sampling/Android.bp +# edit Android.bp to add a fdo_profile module +# soong_namespace {} +# +# fdo_profile { +# name: "etm_test_loop_afdo", +# profile: ["etm_test_loop.afdo"], +# } +``` + +`soong_namespace` is added to support fdo_profile modules with the same name + +In a product config mk file, update `PRODUCT_AFDO_PROFILES` with + +```make +PRODUCT_AFDO_PROFILES += etm_test_loop://toolchain/pgo-profiles/sampling:etm_test_loop_afdo +``` + +```sh (host) <AOSP>$ vi system/extras/simpleperf/runtest/Android.bp # edit Android.bp to enable afdo for etm_test_loop. # cc_binary { diff --git a/simpleperf/environment.cpp b/simpleperf/environment.cpp index 8c4b9576..39803289 100644 --- a/simpleperf/environment.cpp +++ b/simpleperf/environment.cpp @@ -241,6 +241,16 @@ bool CanRecordRawData() { #endif } +std::optional<uint64_t> GetMemorySize() { + std::unique_ptr<FILE, decltype(&fclose)> fp(fopen("/proc/meminfo", "r"), fclose); + uint64_t size; + if (fp && fscanf(fp.get(), "MemTotal:%" PRIu64 " k", &size) == 1) { + return size * kKilobyte; + } + PLOG(ERROR) << "failed to get memory size"; + return std::nullopt; +} + static const char* GetLimitLevelDescription(int limit_level) { switch (limit_level) { case -1: @@ -955,7 +965,9 @@ std::string GetCompleteProcessName(pid_t pid) { if (pos != std::string::npos) { argv0.resize(pos); } - return android::base::Basename(argv0); + // argv0 can be empty if the process is in zombie state. In that case, we don't want to pass argv0 + // to Basename(), which returns ".". + return argv0.empty() ? std::string() : android::base::Basename(argv0); } const char* GetTraceFsDir() { diff --git a/simpleperf/environment.h b/simpleperf/environment.h index 34ce7fa5..a69abaa9 100644 --- a/simpleperf/environment.h +++ b/simpleperf/environment.h @@ -85,6 +85,7 @@ bool SetCpuTimeMaxPercent(size_t percent); bool GetPerfEventMlockKb(uint64_t* mlock_kb); bool SetPerfEventMlockKb(uint64_t mlock_kb); bool CanRecordRawData(); +std::optional<uint64_t> GetMemorySize(); ArchType GetMachineArch(); void PrepareVdsoFile(); diff --git a/simpleperf/environment_test.cpp b/simpleperf/environment_test.cpp index a95caca8..511f1844 100644 --- a/simpleperf/environment_test.cpp +++ b/simpleperf/environment_test.cpp @@ -136,3 +136,9 @@ TEST(environment, GetAppType) { ASSERT_EQ(GetAppType("com.android.simpleperf.profileable"), "profileable"); ASSERT_EQ(GetAppType("com.android.simpleperf.app_not_exist"), "not_exist"); } + +TEST(environment, GetMemorySize) { + auto value = GetMemorySize(); + ASSERT_TRUE(value); + ASSERT_GT(value.value(), 0); +} diff --git a/simpleperf/event_attr.cpp b/simpleperf/event_attr.cpp index dea77a94..b8acc341 100644 --- a/simpleperf/event_attr.cpp +++ b/simpleperf/event_attr.cpp @@ -145,7 +145,7 @@ void DumpPerfEventAttr(const perf_event_attr& attr, size_t indent) { PrintIndented(indent + 1, "sample_stack_user 0x%" PRIx64 "\n", attr.sample_stack_user); } -bool GetCommonEventIdPositionsForAttrs(std::vector<perf_event_attr>& attrs, +bool GetCommonEventIdPositionsForAttrs(const EventAttrIds& attrs, size_t* event_id_pos_in_sample_records, size_t* event_id_reverse_pos_in_non_sample_records) { // When there are more than one perf_event_attrs, we need to read event id @@ -153,7 +153,7 @@ bool GetCommonEventIdPositionsForAttrs(std::vector<perf_event_attr>& attrs, // we need to determine the event id position in a record here. std::vector<uint64_t> sample_types; for (const auto& attr : attrs) { - sample_types.push_back(attr.sample_type); + sample_types.push_back(attr.attr.sample_type); } // First determine event_id_pos_in_sample_records. // If PERF_SAMPLE_IDENTIFIER is enabled, it is just after perf_event_header. @@ -192,7 +192,7 @@ bool GetCommonEventIdPositionsForAttrs(std::vector<perf_event_attr>& attrs, // also be the same. bool sample_id_all_enabled = true; for (const auto& attr : attrs) { - if (attr.sample_id_all == 0) { + if (attr.attr.sample_id_all == 0) { sample_id_all_enabled = false; } } @@ -254,4 +254,11 @@ std::string GetEventNameByAttr(const perf_event_attr& attr) { return name; } +void ReplaceRegAndStackWithCallChain(perf_event_attr& attr) { + attr.sample_type &= ~(PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER); + attr.exclude_callchain_user = 0; + attr.sample_regs_user = 0; + attr.sample_stack_user = 0; +} + } // namespace simpleperf diff --git a/simpleperf/event_attr.h b/simpleperf/event_attr.h index a9d8bba7..5521f328 100644 --- a/simpleperf/event_attr.h +++ b/simpleperf/event_attr.h @@ -18,6 +18,7 @@ #define SIMPLE_PERF_EVENT_ATTR_H_ #include <stddef.h> +#include <string.h> #include <string> #include <vector> @@ -29,15 +30,17 @@ namespace simpleperf { struct EventType; struct EventAttrWithId { - const perf_event_attr* attr; + perf_event_attr attr; std::vector<uint64_t> ids; }; +using EventAttrIds = std::vector<EventAttrWithId>; + inline constexpr uint64_t INFINITE_SAMPLE_PERIOD = 1ULL << 62; perf_event_attr CreateDefaultPerfEventAttr(const EventType& event_type); void DumpPerfEventAttr(const perf_event_attr& attr, size_t indent = 0); -bool GetCommonEventIdPositionsForAttrs(std::vector<perf_event_attr>& attrs, +bool GetCommonEventIdPositionsForAttrs(const EventAttrIds& attrs, size_t* event_id_pos_in_sample_records, size_t* event_id_reverse_pos_in_non_sample_records); bool IsTimestampSupported(const perf_event_attr& attr); @@ -45,6 +48,15 @@ bool IsCpuSupported(const perf_event_attr& attr); // Return event name with modifier if the event is found, otherwise return "unknown". // This function is slow for using linear search, so only used when reporting. std::string GetEventNameByAttr(const perf_event_attr& attr); +void ReplaceRegAndStackWithCallChain(perf_event_attr& attr); + +inline bool operator==(const perf_event_attr& attr1, const perf_event_attr& attr2) { + return memcmp(&attr1, &attr2, sizeof(perf_event_attr)) == 0; +} + +inline bool operator!=(const perf_event_attr& attr1, const perf_event_attr& attr2) { + return !(attr1 == attr2); +} } // namespace simpleperf diff --git a/simpleperf/event_selection_set.cpp b/simpleperf/event_selection_set.cpp index 58b4b956..2fdb88f5 100644 --- a/simpleperf/event_selection_set.cpp +++ b/simpleperf/event_selection_set.cpp @@ -367,16 +367,17 @@ bool EventSelectionSet::ExcludeKernel() const { return true; } -std::vector<EventAttrWithId> EventSelectionSet::GetEventAttrWithId() const { - std::vector<EventAttrWithId> result; +EventAttrIds EventSelectionSet::GetEventAttrWithId() const { + EventAttrIds result; for (const auto& group : groups_) { for (const auto& selection : group) { - EventAttrWithId attr_id; - attr_id.attr = &selection.event_attr; + std::vector<uint64_t> ids; for (const auto& fd : selection.event_fds) { - attr_id.ids.push_back(fd->Id()); + ids.push_back(fd->Id()); } - result.push_back(attr_id); + result.resize(result.size() + 1); + result.back().attr = selection.event_attr; + result.back().ids = std::move(ids); } } return result; diff --git a/simpleperf/event_selection_set.h b/simpleperf/event_selection_set.h index 07754fd2..757034dc 100644 --- a/simpleperf/event_selection_set.h +++ b/simpleperf/event_selection_set.h @@ -115,7 +115,7 @@ class EventSelectionSet { std::vector<const EventType*> GetTracepointEvents() const; bool ExcludeKernel() const; bool HasAuxTrace() const { return has_aux_trace_; } - std::vector<EventAttrWithId> GetEventAttrWithId() const; + EventAttrIds GetEventAttrWithId() const; std::unordered_map<uint64_t, std::string> GetEventNamesById() const; void SetEnableOnExec(bool enable); diff --git a/simpleperf/gtest_main.cpp b/simpleperf/gtest_main.cpp index 3fce9857..c8942b69 100644 --- a/simpleperf/gtest_main.cpp +++ b/simpleperf/gtest_main.cpp @@ -76,6 +76,7 @@ int main(int argc, char** argv) { return RunSimpleperfCmd(argc, argv) ? 0 : 1; } + testing::InitGoogleTest(&argc, argv); android::base::InitLogging(argv, android::base::StderrLogger); android::base::LogSeverity log_severity = android::base::WARNING; testdata_dir = std::string(dirname(argv[0])) + "/testdata"; @@ -104,7 +105,6 @@ int main(int argc, char** argv) { ScopedEnablingPerf scoped_enabling_perf; #endif - testing::InitGoogleTest(&argc, argv); if (!::testing::GTEST_FLAG(list_tests)) { if (!IsDir(testdata_dir)) { LOG(ERROR) << "testdata wasn't found. Use \"" << argv[0] << " -t <testdata_dir>\""; diff --git a/simpleperf/record.cpp b/simpleperf/record.cpp index 40d3287a..c364b2fc 100644 --- a/simpleperf/record.cpp +++ b/simpleperf/record.cpp @@ -713,8 +713,10 @@ SampleRecord::SampleRecord(const perf_event_attr& attr, uint64_t id, uint64_t ip void SampleRecord::ReplaceRegAndStackWithCallChain(const std::vector<uint64_t>& ips) { uint32_t size_added_in_callchain = sizeof(uint64_t) * (ips.size() + 1); uint32_t size_reduced_in_reg_stack = - regs_user_data.reg_nr * sizeof(uint64_t) + stack_user_data.size + sizeof(uint64_t); + (regs_user_data.reg_nr + 1) * sizeof(uint64_t) + stack_user_data.size + sizeof(uint64_t) * 2; + uint32_t new_size = size() + size_added_in_callchain - size_reduced_in_reg_stack; + sample_type &= ~(PERF_SAMPLE_STACK_USER | PERF_SAMPLE_REGS_USER); BuildBinaryWithNewCallChain(new_size, ips); } diff --git a/simpleperf/record_file.h b/simpleperf/record_file.h index 6daae288..51c7b245 100644 --- a/simpleperf/record_file.h +++ b/simpleperf/record_file.h @@ -81,7 +81,7 @@ class RecordFileWriter { RecordFileWriter(const std::string& filename, FILE* fp, bool own_fp); ~RecordFileWriter(); - bool WriteAttrSection(const std::vector<EventAttrWithId>& attr_ids); + bool WriteAttrSection(const EventAttrIds& attr_ids); bool WriteRecord(const Record& record); bool WriteData(const void* buf, size_t len); @@ -141,14 +141,7 @@ class RecordFileReader { const PerfFileFormat::FileHeader& FileHeader() const { return header_; } - std::vector<EventAttrWithId> AttrSection() const { - std::vector<EventAttrWithId> result(file_attrs_.size()); - for (size_t i = 0; i < file_attrs_.size(); ++i) { - result[i].attr = &file_attrs_[i].attr; - result[i].ids = event_ids_for_file_attrs_[i]; - } - return result; - } + const EventAttrIds& AttrSection() const { return event_attrs_; } const std::unordered_map<uint64_t, size_t>& EventIdMap() const { return event_id_to_attr_map_; } @@ -208,7 +201,7 @@ class RecordFileReader { bool CheckSectionDesc(const PerfFileFormat::SectionDesc& desc, uint64_t min_offset, uint64_t alignment = 1); bool ReadAttrSection(); - bool ReadIdsForAttr(const PerfFileFormat::FileAttr& attr, std::vector<uint64_t>* ids); + bool ReadIdSection(const PerfFileFormat::SectionDesc& section, std::vector<uint64_t>* ids); bool ReadFeatureSectionDescriptors(); bool ReadFileV1Feature(uint64_t& read_pos, uint64_t max_size, FileFeature& file); bool ReadFileV2Feature(uint64_t& read_pos, uint64_t max_size, FileFeature& file); @@ -224,8 +217,7 @@ class RecordFileReader { uint64_t file_size_; PerfFileFormat::FileHeader header_; - std::vector<PerfFileFormat::FileAttr> file_attrs_; - std::vector<std::vector<uint64_t>> event_ids_for_file_attrs_; + EventAttrIds event_attrs_; std::unordered_map<uint64_t, size_t> event_id_to_attr_map_; std::map<int, PerfFileFormat::SectionDesc> feature_section_descriptors_; diff --git a/simpleperf/record_file_reader.cpp b/simpleperf/record_file_reader.cpp index 166917c8..d7f1d5eb 100644 --- a/simpleperf/record_file_reader.cpp +++ b/simpleperf/record_file_reader.cpp @@ -166,42 +166,42 @@ bool RecordFileReader::ReadAttrSection() { PLOG(ERROR) << "fseek() failed"; return false; } + event_attrs_.resize(attr_count); + std::vector<SectionDesc> id_sections(attr_count); + size_t attr_size_in_file = header_.attr_size - sizeof(SectionDesc); for (size_t i = 0; i < attr_count; ++i) { std::vector<char> buf(header_.attr_size); if (!Read(buf.data(), buf.size())) { return false; } - // The size of perf_event_attr is changing between different linux kernel versions. - // Make sure we copy correct data to memory. - FileAttr attr; - memset(&attr, 0, sizeof(attr)); - size_t section_desc_size = sizeof(attr.ids); - size_t perf_event_attr_size = header_.attr_size - section_desc_size; - memcpy(&attr.attr, &buf[0], std::min(sizeof(attr.attr), perf_event_attr_size)); - memcpy(&attr.ids, &buf[perf_event_attr_size], section_desc_size); - if (!CheckSectionDesc(attr.ids, 0, sizeof(uint64_t))) { + // The struct perf_event_attr is defined in a Linux header file. It can be extended in newer + // kernel versions with more fields and a bigger size. To disable these extensions, set their + // values to zero. So to copy perf_event_attr from file to memory safely, ensure the copy + // doesn't overflow the file or memory, and set the values of any extra fields in memory to + // zero. + if (attr_size_in_file >= sizeof(perf_event_attr)) { + memcpy(&event_attrs_[i].attr, &buf[0], sizeof(perf_event_attr)); + } else { + memset(&event_attrs_[i].attr, 0, sizeof(perf_event_attr)); + memcpy(&event_attrs_[i].attr, &buf[0], attr_size_in_file); + } + memcpy(&id_sections[i], &buf[attr_size_in_file], sizeof(SectionDesc)); + if (!CheckSectionDesc(id_sections[i], 0, sizeof(uint64_t))) { LOG(ERROR) << "invalid attr section in " << filename_; return false; } - file_attrs_.push_back(attr); } - if (file_attrs_.size() > 1) { - std::vector<perf_event_attr> attrs; - for (const auto& file_attr : file_attrs_) { - attrs.push_back(file_attr.attr); - } - if (!GetCommonEventIdPositionsForAttrs(attrs, &event_id_pos_in_sample_records_, + if (event_attrs_.size() > 1) { + if (!GetCommonEventIdPositionsForAttrs(event_attrs_, &event_id_pos_in_sample_records_, &event_id_reverse_pos_in_non_sample_records_)) { return false; } } - for (size_t i = 0; i < file_attrs_.size(); ++i) { - std::vector<uint64_t> ids; - if (!ReadIdsForAttr(file_attrs_[i], &ids)) { + for (size_t i = 0; i < attr_count; ++i) { + if (!ReadIdSection(id_sections[i], &event_attrs_[i].ids)) { return false; } - event_ids_for_file_attrs_.push_back(ids); - for (auto id : ids) { + for (auto id : event_attrs_[i].ids) { event_id_to_attr_map_[id] = i; } } @@ -237,14 +237,14 @@ bool RecordFileReader::ReadFeatureSectionDescriptors() { return true; } -bool RecordFileReader::ReadIdsForAttr(const FileAttr& attr, std::vector<uint64_t>* ids) { - size_t id_count = attr.ids.size / sizeof(uint64_t); - if (fseek(record_fp_, attr.ids.offset, SEEK_SET) != 0) { +bool RecordFileReader::ReadIdSection(const SectionDesc& section, std::vector<uint64_t>* ids) { + size_t id_count = section.size / sizeof(uint64_t); + if (fseek(record_fp_, section.offset, SEEK_SET) != 0) { PLOG(ERROR) << "fseek() failed"; return false; } ids->resize(id_count); - if (!Read(ids->data(), attr.ids.size)) { + if (!Read(ids->data(), section.size)) { return false; } return true; @@ -342,8 +342,8 @@ std::unique_ptr<Record> RecordFileReader::ReadRecord() { read_record_size_ += header.size; } - const perf_event_attr* attr = &file_attrs_[0].attr; - if (file_attrs_.size() > 1 && header.type < PERF_RECORD_USER_DEFINED_TYPE_START) { + const perf_event_attr* attr = &event_attrs_[0].attr; + if (event_attrs_.size() > 1 && header.type < PERF_RECORD_USER_DEFINED_TYPE_START) { bool has_event_id = false; uint64_t event_id; if (header.type == PERF_RECORD_SAMPLE) { @@ -361,7 +361,7 @@ std::unique_ptr<Record> RecordFileReader::ReadRecord() { if (has_event_id) { auto it = event_id_to_attr_map_.find(event_id); if (it != event_id_to_attr_map_.end()) { - attr = &file_attrs_[it->second].attr; + attr = &event_attrs_[it->second].attr; } } } @@ -401,8 +401,9 @@ bool RecordFileReader::ReadAtOffset(uint64_t offset, void* buf, size_t len) { void RecordFileReader::ProcessEventIdRecord(const EventIdRecord& r) { for (size_t i = 0; i < r.count; ++i) { - event_ids_for_file_attrs_[r.data[i].attr_id].push_back(r.data[i].event_id); - event_id_to_attr_map_[r.data[i].event_id] = r.data[i].attr_id; + const auto& data = r.data[i]; + event_attrs_[data.attr_id].ids.push_back(data.event_id); + event_id_to_attr_map_[data.event_id] = data.attr_id; } } @@ -489,7 +490,7 @@ std::vector<BuildIdRecord> RecordFileReader::ReadBuildIdFeature() { memcpy(binary.get(), p, header->size); p += header->size; BuildIdRecord record; - if (!record.Parse(file_attrs_[0].attr, binary.get(), binary.get() + header->size)) { + if (!record.Parse(event_attrs_[0].attr, binary.get(), binary.get() + header->size)) { return {}; } binary.release(); @@ -811,7 +812,7 @@ bool RecordFileReader::BuildAuxDataLocation() { return false; } AuxTraceRecord auxtrace; - if (!auxtrace.Parse(file_attrs_[0].attr, buf.get(), buf.get() + AuxTraceRecord::Size())) { + if (!auxtrace.Parse(event_attrs_[0].attr, buf.get(), buf.get() + AuxTraceRecord::Size())) { return false; } AuxDataLocation location(auxtrace.data->offset, auxtrace.data->aux_size, diff --git a/simpleperf/record_file_test.cpp b/simpleperf/record_file_test.cpp index 395b6603..e43e4bdf 100644 --- a/simpleperf/record_file_test.cpp +++ b/simpleperf/record_file_test.cpp @@ -40,20 +40,18 @@ class RecordFileTest : public ::testing::Test { void SetUp() override { close(tmpfile_.release()); } void AddEventType(const std::string& event_type_str) { + uint64_t fake_id = attr_ids_.size(); + attr_ids_.resize(attr_ids_.size() + 1); + EventAttrWithId& attr_id = attr_ids_.back(); std::unique_ptr<EventTypeAndModifier> event_type_modifier = ParseEventType(event_type_str); ASSERT_TRUE(event_type_modifier != nullptr); - perf_event_attr attr = CreateDefaultPerfEventAttr(event_type_modifier->event_type); - attr.sample_id_all = 1; - attrs_.push_back(std::unique_ptr<perf_event_attr>(new perf_event_attr(attr))); - EventAttrWithId attr_id; - attr_id.attr = attrs_.back().get(); - attr_id.ids.push_back(attrs_.size()); // Fake id. - attr_ids_.push_back(attr_id); + attr_id.attr = CreateDefaultPerfEventAttr(event_type_modifier->event_type); + attr_id.attr.sample_id_all = 1; + attr_id.ids.push_back(fake_id); } TemporaryFile tmpfile_; - std::vector<std::unique_ptr<perf_event_attr>> attrs_; - std::vector<EventAttrWithId> attr_ids_; + EventAttrIds attr_ids_; }; TEST_F(RecordFileTest, smoke) { @@ -66,7 +64,7 @@ TEST_F(RecordFileTest, smoke) { ASSERT_TRUE(writer->WriteAttrSection(attr_ids_)); // Write data section. - MmapRecord mmap_record(*(attr_ids_[0].attr), true, 1, 1, 0x1000, 0x2000, 0x3000, + MmapRecord mmap_record(attr_ids_[0].attr, true, 1, 1, 0x1000, 0x2000, 0x3000, "mmap_record_example", attr_ids_[0].ids[0]); ASSERT_TRUE(writer->WriteRecord(mmap_record)); @@ -86,9 +84,9 @@ TEST_F(RecordFileTest, smoke) { // Read from a record file. std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(tmpfile_.path); ASSERT_TRUE(reader != nullptr); - std::vector<EventAttrWithId> attrs = reader->AttrSection(); + const EventAttrIds& attrs = reader->AttrSection(); ASSERT_EQ(1u, attrs.size()); - ASSERT_EQ(0, memcmp(attrs[0].attr, attr_ids_[0].attr, sizeof(perf_event_attr))); + ASSERT_EQ(0, memcmp(&attrs[0].attr, &attr_ids_[0].attr, sizeof(perf_event_attr))); ASSERT_EQ(attrs[0].ids, attr_ids_[0].ids); // Read and check data section. @@ -120,10 +118,10 @@ TEST_F(RecordFileTest, record_more_than_one_attr) { // Read from a record file. std::unique_ptr<RecordFileReader> reader = RecordFileReader::CreateInstance(tmpfile_.path); ASSERT_TRUE(reader != nullptr); - std::vector<EventAttrWithId> attrs = reader->AttrSection(); + const EventAttrIds& attrs = reader->AttrSection(); ASSERT_EQ(3u, attrs.size()); for (size_t i = 0; i < attrs.size(); ++i) { - ASSERT_EQ(0, memcmp(attrs[i].attr, attr_ids_[i].attr, sizeof(perf_event_attr))); + ASSERT_EQ(0, memcmp(&attrs[i].attr, &attr_ids_[i].attr, sizeof(perf_event_attr))); ASSERT_EQ(attrs[i].ids, attr_ids_[i].ids); } } diff --git a/simpleperf/record_file_writer.cpp b/simpleperf/record_file_writer.cpp index bcc26a33..ed94e398 100644 --- a/simpleperf/record_file_writer.cpp +++ b/simpleperf/record_file_writer.cpp @@ -74,7 +74,7 @@ RecordFileWriter::~RecordFileWriter() { } } -bool RecordFileWriter::WriteAttrSection(const std::vector<EventAttrWithId>& attr_ids) { +bool RecordFileWriter::WriteAttrSection(const EventAttrIds& attr_ids) { if (attr_ids.empty()) { return false; } @@ -102,7 +102,7 @@ bool RecordFileWriter::WriteAttrSection(const std::vector<EventAttrWithId>& attr } for (auto& attr_id : attr_ids) { FileAttr file_attr; - file_attr.attr = *attr_id.attr; + file_attr.attr = attr_id.attr; file_attr.ids.offset = id_section_offset; file_attr.ids.size = attr_id.ids.size() * sizeof(uint64_t); id_section_offset += file_attr.ids.size; @@ -121,7 +121,7 @@ bool RecordFileWriter::WriteAttrSection(const std::vector<EventAttrWithId>& attr data_section_offset_ = data_section_offset; // Save event_attr for use when reading records. - event_attr_ = *attr_ids[0].attr; + event_attr_ = attr_ids[0].attr; return true; } diff --git a/simpleperf/record_test.cpp b/simpleperf/record_test.cpp index ddecc9bd..8d9d151a 100644 --- a/simpleperf/record_test.cpp +++ b/simpleperf/record_test.cpp @@ -104,11 +104,13 @@ TEST_F(RecordTest, SampleRecord_exclude_kernel_callchain) { } TEST_F(RecordTest, SampleRecord_ReplaceRegAndStackWithCallChain) { - event_attr.sample_type |= PERF_SAMPLE_CALLCHAIN | PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER; + event_attr.sample_type |= PERF_SAMPLE_CALLCHAIN; SampleRecord expected(event_attr, 0, 1, 2, 3, 4, 5, 6, {}, {1, PERF_CONTEXT_USER, 2, 3, 4, 5}, {}, 0); for (size_t stack_size : {8, 1024}) { + event_attr.sample_type |= PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER; SampleRecord r(event_attr, 0, 1, 2, 3, 4, 5, 6, {}, {1}, std::vector<char>(stack_size), 10); + event_attr.sample_type &= ~(PERF_SAMPLE_REGS_USER | PERF_SAMPLE_STACK_USER); r.ReplaceRegAndStackWithCallChain({2, 3, 4, 5}); CheckRecordMatchBinary(r); CheckRecordEqual(r, expected); diff --git a/simpleperf/report_lib_interface.cpp b/simpleperf/report_lib_interface.cpp index d94d6a76..3654c57e 100644 --- a/simpleperf/report_lib_interface.cpp +++ b/simpleperf/report_lib_interface.cpp @@ -332,7 +332,7 @@ bool ReportLib::OpenRecordFileIfNecessary() { auto& meta_info = record_file_reader_->GetMetaInfoFeature(); if (auto it = meta_info.find("trace_offcpu"); it != meta_info.end() && it->second == "true") { // If recorded with --trace-offcpu, default is to report on-off-cpu samples. - std::string event_name = GetEventNameByAttr(*record_file_reader_->AttrSection()[0].attr); + std::string event_name = GetEventNameByAttr(record_file_reader_->AttrSection()[0].attr); if (!android::base::StartsWith(event_name, "cpu-clock") && !android::base::StartsWith(event_name, "task-clock")) { LOG(ERROR) << "Recording file " << record_filename_ << " is no longer supported. " @@ -517,10 +517,10 @@ const EventInfo* ReportLib::FindEventOfCurrentSample() { } void ReportLib::CreateEvents() { - std::vector<EventAttrWithId> attrs = record_file_reader_->AttrSection(); + const EventAttrIds& attrs = record_file_reader_->AttrSection(); events_.resize(attrs.size()); for (size_t i = 0; i < attrs.size(); ++i) { - events_[i].attr = *attrs[i].attr; + events_[i].attr = attrs[i].attr; events_[i].name = GetEventNameByAttr(events_[i].attr); EventInfo::TracingInfo& tracing_info = events_[i].tracing_info; if (events_[i].attr.type == PERF_TYPE_TRACEPOINT && tracing_) { diff --git a/simpleperf/scripts/app_profiler.py b/simpleperf/scripts/app_profiler.py index 808e3e9c..a6399ef1 100755 --- a/simpleperf/scripts/app_profiler.py +++ b/simpleperf/scripts/app_profiler.py @@ -233,7 +233,7 @@ class ProfilerBase(object): raise NotImplementedError def start_profiling(self, target_args): - """Start simpleperf reocrd process on device.""" + """Start simpleperf record process on device.""" args = ['/data/local/tmp/simpleperf', 'record', '-o', '/data/local/tmp/perf.data', self.args.record_options] if self.adb.run(['shell', 'ls', NATIVE_LIBS_DIR_ON_DEVICE]): diff --git a/simpleperf/scripts/bin/android/arm/simpleperf b/simpleperf/scripts/bin/android/arm/simpleperf Binary files differindex 9b35b2a2..bf6b0563 100755 --- a/simpleperf/scripts/bin/android/arm/simpleperf +++ b/simpleperf/scripts/bin/android/arm/simpleperf diff --git a/simpleperf/scripts/bin/android/arm64/simpleperf b/simpleperf/scripts/bin/android/arm64/simpleperf Binary files differindex b3ff0b4d..a81a14b0 100755 --- a/simpleperf/scripts/bin/android/arm64/simpleperf +++ b/simpleperf/scripts/bin/android/arm64/simpleperf diff --git a/simpleperf/scripts/bin/android/x86/simpleperf b/simpleperf/scripts/bin/android/x86/simpleperf Binary files differindex de9efc16..32fc9197 100755 --- a/simpleperf/scripts/bin/android/x86/simpleperf +++ b/simpleperf/scripts/bin/android/x86/simpleperf diff --git a/simpleperf/scripts/bin/android/x86_64/simpleperf b/simpleperf/scripts/bin/android/x86_64/simpleperf Binary files differindex 16416c0f..cb7f7bf8 100755 --- a/simpleperf/scripts/bin/android/x86_64/simpleperf +++ b/simpleperf/scripts/bin/android/x86_64/simpleperf diff --git a/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib b/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib Binary files differindex da84ab3f..cd829eb4 100755 --- a/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib +++ b/simpleperf/scripts/bin/darwin/x86_64/libsimpleperf_report.dylib diff --git a/simpleperf/scripts/bin/darwin/x86_64/simpleperf b/simpleperf/scripts/bin/darwin/x86_64/simpleperf Binary files differindex fffdec9f..4a4396d7 100755 --- a/simpleperf/scripts/bin/darwin/x86_64/simpleperf +++ b/simpleperf/scripts/bin/darwin/x86_64/simpleperf diff --git a/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so b/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so Binary files differindex ab4066b3..1f9f13fb 100755 --- a/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so +++ b/simpleperf/scripts/bin/linux/x86_64/libsimpleperf_report.so diff --git a/simpleperf/scripts/bin/linux/x86_64/simpleperf b/simpleperf/scripts/bin/linux/x86_64/simpleperf Binary files differindex 587d1e87..d8dd8400 100755 --- a/simpleperf/scripts/bin/linux/x86_64/simpleperf +++ b/simpleperf/scripts/bin/linux/x86_64/simpleperf diff --git a/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll b/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll Binary files differindex 099ddfb2..4282b4c0 100755 --- a/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll +++ b/simpleperf/scripts/bin/windows/x86_64/libsimpleperf_report.dll diff --git a/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe b/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe Binary files differindex 3756745e..47fcd20f 100755 --- a/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe +++ b/simpleperf/scripts/bin/windows/x86_64/simpleperf.exe diff --git a/simpleperf/scripts/binary_cache_builder.py b/simpleperf/scripts/binary_cache_builder.py index 1b09e7ad..46c8532a 100755 --- a/simpleperf/scripts/binary_cache_builder.py +++ b/simpleperf/scripts/binary_cache_builder.py @@ -255,6 +255,7 @@ class BinaryCacheBuilder: self.binaries = {} def build_binary_cache(self, perf_data_path: str, symfs_dirs: List[Union[Path, str]]) -> bool: + self.binary_cache_dir.mkdir(exist_ok=True) self.collect_used_binaries(perf_data_path) if not self.copy_binaries_from_symfs_dirs(symfs_dirs): return False diff --git a/simpleperf/scripts/gecko_profile_generator.py b/simpleperf/scripts/gecko_profile_generator.py index 9fe9ad3d..741841c2 100755 --- a/simpleperf/scripts/gecko_profile_generator.py +++ b/simpleperf/scripts/gecko_profile_generator.py @@ -114,6 +114,13 @@ CATEGORIES = [ "color": 'green', "subcategories": ['Other'] }, + { + "name": 'Off-CPU', + # Follow Brendan Gregg's Flamegraph convention: blue for off-CPU + # https://github.com/brendangregg/FlameGraph/blob/810687f180f3c4929b5d965f54817a5218c9d89b/flamegraph.pl#L470 + "color": 'blue', + "subcategories": ['Other'] + }, # Not used by this exporter yet, but some Firefox Profiler code assumes # there is an 'Other' category by searching for a category with # color=grey, so include this. @@ -197,12 +204,19 @@ class Thread: # Heuristic: kernel code contains "kallsyms" as the library name. if "kallsyms" in frame_str or ".ko" in frame_str: category = 1 + if frame_str.startswith("__schedule "): + category = 5 elif ".so" in frame_str: category = 2 elif ".vdex" in frame_str: category = 3 elif ".oat" in frame_str: category = 4 + # Heuristic: empirically, off-CPU profiles mostly measure off-CPU time + # accounted to the linux kernel __schedule function, which handles + # blocking. This only works if we have kernel symbol (kallsyms) + # access though. + # https://cs.android.com/android/kernel/superproject/+/common-android-mainline:common/kernel/sched/core.c;l=6593;drc=0c99414a07ddaa18d8eb4be90b551d2687cbde2f self.frameTable.append(Frame( string_id=string_id, diff --git a/simpleperf/scripts/test/app_test.py b/simpleperf/scripts/test/app_test.py index 4b31e4d4..a13200af 100644 --- a/simpleperf/scripts/test/app_test.py +++ b/simpleperf/scripts/test/app_test.py @@ -62,6 +62,9 @@ class TestExampleBase(TestBase): def setUp(self): super(TestExampleBase, self).setUp() + if TestHelper.android_version == 8 and ( + 'ExampleJava' in self.id() or 'ExampleKotlin' in self.id()): + self.skipTest('Profiling java code needs wrap.sh on Android O (8.x)') if 'TraceOffCpu' in self.id() and not TestHelper.is_trace_offcpu_supported(): self.skipTest('trace-offcpu is not supported on device') # Use testcase_dir to share a common perf.data for reporting. So we don't need to diff --git a/simpleperf/scripts/test/gecko_profile_generator_test.py b/simpleperf/scripts/test/gecko_profile_generator_test.py index 97cedd6f..b96dfc85 100644 --- a/simpleperf/scripts/test/gecko_profile_generator_test.py +++ b/simpleperf/scripts/test/gecko_profile_generator_test.py @@ -41,6 +41,24 @@ class TestGeckoProfileGenerator(TestBase): golden_path = TestHelper.testdata_path('perf_with_interpreter_frames.gecko.json') with open(golden_path) as f: want = json.load(f) + # Golden data is formatted with `jq` tool (https://stedolan.github.io/jq/). + # Regenerate golden data by running: + # $ apt install jq + # $ ./gecko_profile_generator.py --remove-gaps 0 -i ../testdata/perf_with_interpreter_frames.data | jq > test/script_testdata/perf_with_interpreter_frames.gecko.json + self.assertEqual( + json.dumps(got, sort_keys=True, indent=2), + json.dumps(want, sort_keys=True, indent=2)) + + def test_golden_offcpu(self): + output = self.run_generator('perf_with_tracepoint_event.data', ['--remove-gaps', '0']) + got = json.loads(output) + golden_path = TestHelper.testdata_path('perf_with_tracepoint_event.gecko.json') + with open(golden_path) as f: + want = json.load(f) + # Golden data is formatted with `jq` tool (https://stedolan.github.io/jq/). + # Regenerate golden data by running: + # $ apt install jq + # $ ./gecko_profile_generator.py --remove-gaps 0 -i ../testdata/perf_with_tracepoint_event.data | jq > test/script_testdata/perf_with_tracepoint_event.gecko.json self.assertEqual( json.dumps(got, sort_keys=True, indent=2), json.dumps(want, sort_keys=True, indent=2)) diff --git a/simpleperf/scripts/test/java_app_test.py b/simpleperf/scripts/test/java_app_test.py index efd17461..5f74c643 100644 --- a/simpleperf/scripts/test/java_app_test.py +++ b/simpleperf/scripts/test/java_app_test.py @@ -227,7 +227,7 @@ class TestExampleJavaTraceOffCpu(TestExampleBase): ("RunFunction", 20, 20), ("SleepFunction", 20, 0), ("line 24", 1, 0), - ("line 32", 20, 0)]) + ("line 31", 20, 0)]) self.run_cmd([INFERNO_SCRIPT, "-sc"]) self.check_inferno_report_html( [('simpleperf.example.java.SleepActivity$1.run', 80), diff --git a/simpleperf/scripts/test/kotlin_app_test.py b/simpleperf/scripts/test/kotlin_app_test.py index 08d207ab..74f50229 100644 --- a/simpleperf/scripts/test/kotlin_app_test.py +++ b/simpleperf/scripts/test/kotlin_app_test.py @@ -140,8 +140,8 @@ class TestExampleKotlinTraceOffCpu(TestExampleBase): ("run", 80, 0), ("RunFunction", 20, 20), ("SleepFunction", 20, 0), - ("line 24", 20, 0), - ("line 32", 20, 0)]) + ("line 23", 20, 0), + ("line 31", 20, 0)]) self.run_cmd([INFERNO_SCRIPT, "-sc"]) self.check_inferno_report_html([ diff --git a/simpleperf/scripts/test/script_testdata/perf_with_interpreter_frames.gecko.json b/simpleperf/scripts/test/script_testdata/perf_with_interpreter_frames.gecko.json index 1903451a..e423687d 100644 --- a/simpleperf/scripts/test/script_testdata/perf_with_interpreter_frames.gecko.json +++ b/simpleperf/scripts/test/script_testdata/perf_with_interpreter_frames.gecko.json @@ -40,6 +40,13 @@ ] }, { + "color": "blue", + "name": "Off-CPU", + "subcategories": [ + "Other" + ] + }, + { "color": "grey", "name": "Other", "subcategories": [ diff --git a/simpleperf/scripts/test/script_testdata/perf_with_tracepoint_event.gecko.json b/simpleperf/scripts/test/script_testdata/perf_with_tracepoint_event.gecko.json new file mode 100644 index 00000000..75442324 --- /dev/null +++ b/simpleperf/scripts/test/script_testdata/perf_with_tracepoint_event.gecko.json @@ -0,0 +1,542 @@ +{ + "libs": [], + "meta": { + "abi": "aarch64", + "asyncstack": 1, + "categories": [ + { + "color": "yellow", + "name": "User", + "subcategories": [ + "Other" + ] + }, + { + "color": "orange", + "name": "Kernel", + "subcategories": [ + "Other" + ] + }, + { + "color": "yellow", + "name": "Native", + "subcategories": [ + "Other" + ] + }, + { + "color": "green", + "name": "DEX", + "subcategories": [ + "Other" + ] + }, + { + "color": "green", + "name": "OAT", + "subcategories": [ + "Other" + ] + }, + { + "color": "blue", + "name": "Off-CPU", + "subcategories": [ + "Other" + ] + }, + { + "color": "grey", + "name": "Other", + "subcategories": [ + "Other" + ] + } + ], + "debug": 0, + "device": "Google:Pixel 2:walleye", + "gcpoison": 0, + "interval": 1, + "markerSchema": [], + "oscpu": null, + "platform": null, + "presymbolicated": true, + "processType": 0, + "product": "/data/local/tmp/simpleperf record -e sched:sched_switch -e cpu-cycles sleep 1", + "shutdownTime": null, + "stackwalk": 1, + "startTime": 1512941771000, + "version": 24 + }, + "pausedRanges": [], + "processes": [], + "threads": [ + { + "frameTable": { + "data": [ + [ + 0, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 1, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 2, + false, + 0, + null, + null, + null, + null, + 5, + 0 + ], + [ + 3, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 4, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 5, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 6, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 7, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 8, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 9, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ], + [ + 10, + false, + 0, + null, + null, + null, + null, + 1, + 0 + ] + ], + "schema": { + "category": 7, + "column": 6, + "implementation": 3, + "innerWindowID": 2, + "line": 5, + "location": 0, + "optimizations": 4, + "relevantForJS": 1, + "subcategory": 8 + } + }, + "markers": { + "data": [], + "schema": { + "category": 4, + "data": 5, + "endTime": 2, + "name": 0, + "phase": 3, + "startTime": 1 + } + }, + "name": "sleep", + "pid": 9896, + "processType": "default", + "registerTime": 0, + "samples": { + "data": [ + [ + 0, + 536732415.65594, + 0 + ], + [ + 1, + 536732415.663024, + 0 + ], + [ + 2, + 536732415.855628, + 0 + ], + [ + 3, + 536732416.562399, + 0 + ], + [ + 2, + 536732416.922451, + 0 + ], + [ + 4, + 536732418.120784, + 0 + ], + [ + 2, + 536732418.371201, + 0 + ], + [ + 5, + 536732419.142503, + 0 + ], + [ + 5, + 536732419.14969, + 0 + ], + [ + 2, + 536732419.966826, + 0 + ], + [ + 6, + 536732420.235419, + 0 + ], + [ + 2, + 536732420.924794, + 0 + ], + [ + 7, + 536732421.171461, + 0 + ], + [ + 2, + 536732421.250211, + 0 + ], + [ + 4, + 536732421.445836, + 0 + ], + [ + 2, + 536732421.544899, + 0 + ], + [ + 7, + 536732421.800888, + 0 + ], + [ + 8, + 536732421.836878, + 0 + ], + [ + 9, + 536732421.841878, + 0 + ], + [ + 2, + 536732421.87318, + 0 + ], + [ + 2, + 536732421.900315, + 0 + ], + [ + 5, + 536732422.094638, + 0 + ], + [ + 2, + 536732422.188128, + 0 + ], + [ + 2, + 536732422.43094, + 0 + ], + [ + 2, + 536732422.702972, + 0 + ], + [ + 2, + 536732423.006357, + 0 + ], + [ + 2, + 536732423.138076, + 0 + ], + [ + 2, + 536732423.500628, + 0 + ], + [ + 2, + 536732423.76943, + 0 + ], + [ + 2, + 536732425.676201, + 0 + ], + [ + 2, + 536732426.086722, + 0 + ], + [ + 2, + 536732426.35693, + 0 + ], + [ + 2, + 536732426.801513, + 0 + ], + [ + 2, + 536732426.916774, + 0 + ], + [ + 2, + 536732427.020263, + 0 + ], + [ + 2, + 536732427.455524, + 0 + ], + [ + 2, + 536732428.163961, + 0 + ], + [ + 2, + 536732428.383753, + 0 + ], + [ + 2, + 536732428.868284, + 0 + ], + [ + 2, + 536732429.336982, + 0 + ], + [ + 2, + 536732429.894222, + 0 + ], + [ + 2, + 536732430.261357, + 0 + ], + [ + 10, + 536732430.486097, + 0 + ], + [ + 2, + 536732433.741565, + 0 + ], + [ + 2, + 536732434.293857, + 0 + ] + ], + "schema": { + "responsiveness": 2, + "stack": 0, + "time": 1 + } + }, + "stackTable": { + "data": [ + [ + null, + 0, + 0 + ], + [ + null, + 1, + 0 + ], + [ + null, + 2, + 0 + ], + [ + null, + 3, + 0 + ], + [ + null, + 4, + 0 + ], + [ + null, + 5, + 0 + ], + [ + null, + 6, + 0 + ], + [ + null, + 7, + 0 + ], + [ + null, + 8, + 0 + ], + [ + null, + 9, + 0 + ], + [ + null, + 10, + 0 + ] + ], + "schema": { + "category": 2, + "frame": 1, + "prefix": 0 + } + }, + "stringTable": [ + "perf_event_exec (in [kernel.kallsyms])", + "memcpy (in [kernel.kallsyms])", + "__schedule (in [kernel.kallsyms])", + "schedule_timeout (in [kernel.kallsyms])", + "filemap_fault (in [kernel.kallsyms])", + "_raw_spin_unlock_irq (in [kernel.kallsyms])", + "__wait_on_bit_lock (in [kernel.kallsyms])", + "generic_file_read_iter (in [kernel.kallsyms])", + "__do_softirq (in [kernel.kallsyms])", + "_raw_spin_unlock_irqrestore (in [kernel.kallsyms])", + "__clean_dcache_area_pou (in [kernel.kallsyms])" + ], + "tid": 9896, + "unregisterTime": null + } + ] +} diff --git a/simpleperf/testdata/etm/perf_inject.data b/simpleperf/testdata/etm/perf_inject.data index 4c121a32..9f83430b 100644 --- a/simpleperf/testdata/etm/perf_inject.data +++ b/simpleperf/testdata/etm/perf_inject.data @@ -20,5 +20,6 @@ 10a0->1054:1 10b0->0:1 10ec->0:1 +// build_id: 0x0c9a20bf9c009d0e4e8bbf9fad0300ae00000000 // /data/local/tmp/etm_test_loop diff --git a/simpleperf/utils.h b/simpleperf/utils.h index cd169889..c4ea377f 100644 --- a/simpleperf/utils.h +++ b/simpleperf/utils.h @@ -37,6 +37,10 @@ namespace simpleperf { +static constexpr size_t kKilobyte = 1024; +static constexpr size_t kMegabyte = 1024 * kKilobyte; +static constexpr uint64_t kGigabyte = 1024 * kMegabyte; + static inline uint64_t AlignDown(uint64_t value, uint64_t alignment) { return value & ~(alignment - 1); } diff --git a/tests/kernel.config/OWNERS b/tests/kernel.config/OWNERS new file mode 100644 index 00000000..65d27f47 --- /dev/null +++ b/tests/kernel.config/OWNERS @@ -0,0 +1,6 @@ +# Bug component: 119452 + +# TODO: we likely want to delete or factor out these tests +# b/261015480#comment6 + +smoreland@google.com |